+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: binnedcorr2
+"""
+
+import math
+import numpy as np
+import sys
+import coord
+import itertools
+import collections
+
+from . import _lib
+from .config import merge_config, setup_logger, get
+from .util import parse_metric, metric_enum, coord_enum, set_omp_threads, lazy_property
+from .util import depr_pos_kwargs
+
+class Namespace(object):
+ pass
+
+[docs]class BinnedCorr2(object):
+ """This class stores the results of a 2-point correlation calculation, along with some
+ ancillary data.
+
+ This is a base class that is not intended to be constructed directly. But it has a few
+ helper functions that derived classes can use to help perform their calculations. See
+ the derived classes for more details:
+
+ - `GGCorrelation` handles shear-shear correlation functions
+ - `NNCorrelation` handles count-count correlation functions
+ - `KKCorrelation` handles kappa-kappa correlation functions
+ - `NGCorrelation` handles count-shear correlation functions
+ - `NKCorrelation` handles count-kappa correlation functions
+ - `KGCorrelation` handles kappa-shear correlation functions
+
+ .. note::
+
+ When we refer to kappa in the correlation functions, that is because TreeCorr was
+ originally designed for weak lensing applications. But in fact any scalar quantity
+ may be used here. CMB temperature fluctuations for example.
+
+ The constructor for all derived classes take a config dict as the first argument,
+ since this is often how we keep track of parameters, but if you don't want to
+ use one or if you want to change some parameters from what are in a config dict,
+ then you can use normal kwargs, which take precedence over anything in the config dict.
+
+ There are a number of possible definitions for the distance between two points, which
+ are appropriate for different use cases. These are specified by the ``metric`` parameter.
+ The possible options are:
+
+ - 'Euclidean' = straight line Euclidean distance between two points.
+ - 'FisherRperp' = the perpendicular component of the distance, following the
+ definitions in Fisher et al, 1994 (MNRAS, 267, 927).
+ - 'OldRperp' = the perpendicular component of the distance using the definition
+ of Rperp from TreeCorr v3.x.
+ - 'Rperp' = an alias for FisherRperp. You can change it to be an alias for
+ OldRperp if you want by setting ``treecorr.Rperp_alias = 'OldRperp'`` before
+ using it.
+ - 'Rlens' = the distance from the first object (taken to be a lens) to the line
+ connecting Earth and the second object (taken to be a lensed source).
+ - 'Arc' = the true great circle distance for spherical coordinates.
+ - 'Periodic' = Like Euclidean, but with periodic boundaries.
+
+ See `Metrics` for more information about these various metric options.
+
+ There are also a few different possibile binning prescriptions to define the range of
+ distances, which should be placed into each bin.
+
+ - 'Log' - logarithmic binning in the distance. The bin steps will be uniform in
+ log(r) from log(min_sep) .. log(max_sep).
+ - 'Linear' - linear binning in the distance. The bin steps will be uniform in r
+ from min_sep .. max_sep.
+ - 'TwoD' = 2-dimensional binning from x = (-max_sep .. max_sep) and
+ y = (-max_sep .. max_sep). The bin steps will be uniform in both x and y.
+ (i.e. linear in x,y)
+
+ See `Binning` for more information about the different binning options.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in the below kwargs if
+ desired. This dict is allowed to have addition entries in addition
+ to those listed below, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+
+ nbins (int): How many bins to use. (Exactly three of nbins, bin_size, min_sep,
+ max_sep are required. If nbins is not given or set to None, it will be
+ calculated from the values of the other three, rounding up to the next
+ highest integer. In this case, bin_size will be readjusted to account
+ for this rounding up.)
+ bin_size (float): The width of the bins in log(separation). (Exactly three of nbins,
+ bin_size, min_sep, max_sep are required. If bin_size is not given or
+ set to None, it will be calculated from the values of the other three.)
+ min_sep (float): The minimum separation in units of sep_units, if relevant. (Exactly
+ three of nbins, bin_size, min_sep, max_sep are required. If min_sep is
+ not given or set to None, it will be calculated from the values of the
+ other three.)
+ max_sep (float): The maximum separation in units of sep_units, if relevant. (Exactly
+ three of nbins, bin_size, min_sep, max_sep are required. If max_sep is
+ not given or set to None, it will be calculated from the values of the
+ other three.)
+
+ sep_units (str): The units to use for the separation values, given as a string. This
+ includes both min_sep and max_sep above, as well as the units of the
+ output distance values. Valid options are arcsec, arcmin, degrees,
+ hours, radians. (default: radians if angular units make sense, but for
+ 3-d or flat 2-d positions, the default will just match the units of
+ x,y[,z] coordinates)
+ bin_slop (float): How much slop to allow in the placement of pairs in the bins.
+ If bin_slop = 1, then the bin into which a particular pair is placed
+ may be incorrect by at most 1.0 bin widths. (default: None, which
+ means to use a bin_slop that gives a maximum error of 10% on any bin,
+ which has been found to yield good results for most application.
+ brute (bool): Whether to use the "brute force" algorithm. (default: False) Options
+ are:
+
+ - False (the default): Stop at non-leaf cells whenever the error in
+ the separation is compatible with the given bin_slop.
+ - True: Go to the leaves for both catalogs.
+ - 1: Always go to the leaves for cat1, but stop at non-leaf cells of
+ cat2 when the error is compatible with the given bin_slop.
+ - 2: Always go to the leaves for cat2, but stop at non-leaf cells of
+ cat1 when the error is compatible with the given bin_slop.
+
+ verbose (int): If no logger is provided, this will optionally specify a logging level
+ to use:
+
+ - 0 means no logging output
+ - 1 means to output warnings only (default)
+ - 2 means to output various progress information
+ - 3 means to output extensive debugging information
+
+ log_file (str): If no logger is provided, this will specify a file to write the logging
+ output. (default: None; i.e. output to standard output)
+ output_dots (bool): Whether to output progress dots during the calcualtion of the
+ correlation function. (default: False unless verbose is given and >= 2,
+ in which case True)
+
+ split_method (str): How to split the cells in the tree when building the tree structure.
+ Options are:
+
+ - mean = Use the arithmetic mean of the coordinate being split.
+ (default)
+ - median = Use the median of the coordinate being split.
+ - middle = Use the middle of the range; i.e. the average of the minimum
+ and maximum value.
+ - random: Use a random point somewhere in the middle two quartiles of
+ the range.
+
+ min_top (int): The minimum number of top layers to use when setting up the field.
+ (default: :math:`\\max(3, \\log_2(N_{\\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the field.
+ The top-level cells are where each calculation job starts. There will
+ typically be of order :math:`2^{\\rm max\\_top}` top-level cells.
+ (default: 10)
+ precision (int): The precision to use for the output values. This specifies how many
+ digits to write. (default: 4)
+ pairwise (bool): Whether to use a different kind of calculation for cross correlations
+ whereby corresponding items in the two catalogs are correlated pairwise
+ rather than the usual case of every item in one catalog being correlated
+ with every item in the other catalog. (default: False) (DEPRECATED)
+ m2_uform (str): The default functional form to use for aperture mass calculations.
+ see `calculateMapSq` for more details. (default: 'Crittenden')
+
+ metric (str): Which metric to use for distance measurements. Options are listed
+ above. (default: 'Euclidean')
+ bin_type (str): What type of binning should be used. Options are listed above.
+ (default: 'Log')
+ min_rpar (float): The minimum difference in Rparallel to allow for pairs being included
+ in the correlation function. (default: None)
+ max_rpar (float): The maximum difference in Rparallel to allow for pairs being included
+ in the correlation function. (default: None)
+ period (float): For the 'Periodic' metric, the period to use in all directions.
+ (default: None)
+ xperiod (float): For the 'Periodic' metric, the period to use in the x direction.
+ (default: period)
+ yperiod (float): For the 'Periodic' metric, the period to use in the y direction.
+ (default: period)
+ zperiod (float): For the 'Periodic' metric, the period to use in the z direction.
+ (default: period)
+
+ var_method (str): Which method to use for estimating the variance. Options are:
+ 'shot', 'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'.
+ (default: 'shot')
+ num_bootstrap (int): How many bootstrap samples to use for the 'bootstrap' and
+ 'marked_bootstrap' var_methods. (default: 500)
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for bootstrap
+ random number generation. (default: None)
+
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores)
+
+ .. note::
+
+ This won't work if the system's C compiler cannot use OpenMP
+ (e.g. clang prior to version 3.7.)
+ """
+ _valid_params = {
+ 'nbins' : (int, False, None, None,
+ 'The number of output bins to use.'),
+ 'bin_size' : (float, False, None, None,
+ 'The size of the output bins in log(sep).'),
+ 'min_sep' : (float, False, None, None,
+ 'The minimum separation to include in the output.'),
+ 'max_sep' : (float, False, None, None,
+ 'The maximum separation to include in the output.'),
+ 'sep_units' : (str, False, None, coord.AngleUnit.valid_names,
+ 'The units to use for min_sep and max_sep. '
+ 'Also the units of the output distances'),
+ 'bin_slop' : (float, False, None, None,
+ 'The fraction of a bin width by which it is ok to let the pairs miss the correct '
+ 'bin.',
+ 'The default is to use 1 if bin_size <= 0.1, or 0.1/bin_size if bin_size > 0.1.'),
+ 'brute' : (bool, False, False, [False, True, 1, 2],
+ 'Whether to use brute-force algorithm'),
+ 'verbose' : (int, False, 1, [0, 1, 2, 3],
+ 'How verbose the code should be during processing. ',
+ '0 = Errors Only, 1 = Warnings, 2 = Progress, 3 = Debugging'),
+ 'log_file' : (str, False, None, None,
+ 'If desired, an output file for the logging output.',
+ 'The default is to write the output to stdout.'),
+ 'output_dots' : (bool, False, None, None,
+ 'Whether to output dots to the stdout during the C++-level computation.',
+ 'The default is True if verbose >= 2 and there is no log_file. Else False.'),
+ 'split_method' : (str, False, 'mean', ['mean', 'median', 'middle', 'random'],
+ 'Which method to use for splitting cells.'),
+ 'min_top' : (int, False, None, None,
+ 'The minimum number of top layers to use when setting up the field.'),
+ 'max_top' : (int, False, 10, None,
+ 'The maximum number of top layers to use when setting up the field.'),
+ 'precision' : (int, False, 4, None,
+ 'The number of digits after the decimal in the output.'),
+ 'pairwise' : (bool, True, False, None,
+ 'Whether to do a pair-wise cross-correlation. (DEPRECATED)'),
+ 'm2_uform' : (str, False, 'Crittenden', ['Crittenden', 'Schneider'],
+ 'The function form of the mass aperture.'),
+ 'metric': (str, False, 'Euclidean', ['Euclidean', 'Rperp', 'FisherRperp', 'OldRperp',
+ 'Rlens', 'Arc', 'Periodic'],
+ 'Which metric to use for the distance measurements'),
+ 'bin_type': (str, False, 'Log', ['Log', 'Linear', 'TwoD'],
+ 'Which type of binning should be used'),
+ 'min_rpar': (float, False, None, None,
+ 'The minimum difference in Rparallel for pairs to include'),
+ 'max_rpar': (float, False, None, None,
+ 'The maximum difference in Rparallel for pairs to include'),
+ 'period': (float, False, None, None,
+ 'The period to use for all directions for the Periodic metric'),
+ 'xperiod': (float, False, None, None,
+ 'The period to use for the x direction for the Periodic metric'),
+ 'yperiod': (float, False, None, None,
+ 'The period to use for the y direction for the Periodic metric'),
+ 'zperiod': (float, False, None, None,
+ 'The period to use for the z direction for the Periodic metric'),
+
+ 'var_method': (str, False, 'shot',
+ ['shot', 'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'],
+ 'The method to use for estimating the variance'),
+ 'num_bootstrap': (int, False, 500, None,
+ 'How many bootstrap samples to use for the var_method=bootstrap and '
+ 'marked_bootstrap'),
+ 'num_threads' : (int, False, None, None,
+ 'How many threads should be used. num_threads <= 0 means auto based on num cores.'),
+ }
+
+ @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, rng=None, **kwargs):
+ self._corr = None # Do this first to make sure we always have it for __del__
+ self.config = merge_config(config,kwargs,BinnedCorr2._valid_params)
+ if logger is None:
+ self.logger = setup_logger(get(self.config,'verbose',int,1),
+ self.config.get('log_file',None))
+ else:
+ self.logger = logger
+
+ # We'll make a bunch of attributes here, which we put into a namespace called _ro.
+ # These are the core attributes that won't ever be changed after construction.
+ # This is an efficiency optimization (both memory and flops), since it will allow
+ # copy() to just copy a pointer to the _ro namespace without having to copy each
+ # individual attribute separately.
+ # The access of these attributes are all via read-only properties.
+ self._ro = Namespace()
+
+ if 'output_dots' in self.config:
+ self._ro.output_dots = get(self.config,'output_dots',bool)
+ else:
+ self._ro.output_dots = get(self.config,'verbose',int,1) >= 2
+
+ self._ro.bin_type = self.config.get('bin_type', None)
+
+ self._ro.sep_units = self.config.get('sep_units','')
+ self._ro._sep_units = get(self.config,'sep_units',str,'radians')
+ self._ro._log_sep_units = math.log(self._sep_units)
+ if self.config.get('nbins', None) is None:
+ if self.config.get('max_sep', None) is None:
+ raise TypeError("Missing required parameter max_sep")
+ if self.config.get('min_sep', None) is None and self.bin_type != 'TwoD':
+ raise TypeError("Missing required parameter min_sep")
+ if self.config.get('bin_size', None) is None:
+ raise TypeError("Missing required parameter bin_size")
+ self._ro.min_sep = float(self.config.get('min_sep',0))
+ self._ro.max_sep = float(self.config['max_sep'])
+ if self.min_sep >= self.max_sep:
+ raise ValueError("max_sep must be larger than min_sep")
+ self._ro.bin_size = float(self.config['bin_size'])
+ self._ro.nbins = None
+ elif self.config.get('bin_size', None) is None:
+ if self.config.get('max_sep', None) is None:
+ raise TypeError("Missing required parameter max_sep")
+ if self.config.get('min_sep', None) is None and self.bin_type != 'TwoD':
+ raise TypeError("Missing required parameter min_sep")
+ self._ro.min_sep = float(self.config.get('min_sep',0))
+ self._ro.max_sep = float(self.config['max_sep'])
+ if self.min_sep >= self.max_sep:
+ raise ValueError("max_sep must be larger than min_sep")
+ self._ro.nbins = int(self.config['nbins'])
+ self._ro.bin_size = None
+ elif self.config.get('max_sep', None) is None:
+ if self.config.get('min_sep', None) is None and self.bin_type != 'TwoD':
+ raise TypeError("Missing required parameter min_sep")
+ self._ro.min_sep = float(self.config.get('min_sep',0))
+ self._ro.nbins = int(self.config['nbins'])
+ self._ro.bin_size = float(self.config['bin_size'])
+ self._ro.max_sep = None
+ else:
+ if self.bin_type == 'TwoD':
+ raise TypeError("Only 2 of max_sep, bin_size, nbins are allowed "
+ "for bin_type='TwoD'.")
+ if self.config.get('min_sep', None) is not None:
+ raise TypeError("Only 3 of min_sep, max_sep, bin_size, nbins are allowed.")
+ self._ro.max_sep = float(self.config['max_sep'])
+ self._ro.nbins = int(self.config['nbins'])
+ self._ro.bin_size = float(self.config['bin_size'])
+ self._ro.min_sep = None
+
+ if self.bin_type == 'Log':
+ if self.nbins is None:
+ self._ro.nbins = int(math.ceil(math.log(self.max_sep/self.min_sep)/self.bin_size))
+ # Update bin_size given this value of nbins
+ self._ro.bin_size = math.log(self.max_sep/self.min_sep)/self.nbins
+ elif self.bin_size is None:
+ self._ro.bin_size = math.log(self.max_sep/self.min_sep)/self.nbins
+ elif self.max_sep is None:
+ self._ro.max_sep = math.exp(self.nbins*self.bin_size)*self.min_sep
+ else:
+ self._ro.min_sep = self.max_sep*math.exp(-self.nbins*self.bin_size)
+
+ # This makes nbins evenly spaced entries in log(r) starting with 0 with step bin_size
+ self._ro.logr = np.linspace(0, self.nbins*self.bin_size, self.nbins, endpoint=False,
+ dtype=float)
+ # Offset by the position of the center of the first bin.
+ self._ro.logr += math.log(self.min_sep) + 0.5*self.bin_size
+ self._ro.rnom = np.exp(self.logr)
+ half_bin = np.exp(0.5*self.bin_size)
+ self._ro.left_edges = self.rnom / half_bin
+ self._ro.right_edges = self.rnom * half_bin
+ self._ro._nbins = self.nbins
+ self._ro._bintype = _lib.Log
+ min_log_bin_size = self.bin_size
+ max_log_bin_size = self.bin_size
+ max_good_slop = 0.1 / self.bin_size
+ elif self.bin_type == 'Linear':
+ if self.nbins is None:
+ self._ro.nbins = int(math.ceil((self.max_sep-self.min_sep)/self.bin_size))
+ # Update bin_size given this value of nbins
+ self._ro.bin_size = (self.max_sep-self.min_sep)/self.nbins
+ elif self.bin_size is None:
+ self._ro.bin_size = (self.max_sep-self.min_sep)/self.nbins
+ elif self.max_sep is None:
+ self._ro.max_sep = self.min_sep + self.nbins*self.bin_size
+ else:
+ self._ro.min_sep = self.max_sep - self.nbins*self.bin_size
+
+ self._ro.rnom = np.linspace(self.min_sep, self.max_sep, self.nbins, endpoint=False,
+ dtype=float)
+ # Offset by the position of the center of the first bin.
+ self._ro.rnom += 0.5*self.bin_size
+ self._ro.left_edges = self.rnom - 0.5*self.bin_size
+ self._ro.right_edges = self.rnom + 0.5*self.bin_size
+ self._ro.logr = np.log(self.rnom)
+ self._ro._nbins = self.nbins
+ self._ro._bintype = _lib.Linear
+ min_log_bin_size = self.bin_size / self.max_sep
+ max_log_bin_size = self.bin_size / (self.min_sep + self.bin_size/2)
+ max_good_slop = 0.1 / max_log_bin_size
+ elif self.bin_type == 'TwoD':
+ if self.nbins is None:
+ self._ro.nbins = int(math.ceil(2.*self.max_sep / self.bin_size))
+ self._ro.bin_size = 2.*self.max_sep/self.nbins
+ elif self.bin_size is None:
+ self._ro.bin_size = 2.*self.max_sep/self.nbins
+ else:
+ self._ro.max_sep = self.nbins * self.bin_size / 2.
+
+ sep = np.linspace(-self.max_sep, self.max_sep, self.nbins, endpoint=False,
+ dtype=float)
+ sep += 0.5 * self.bin_size
+ dx, dy = np.meshgrid(sep, sep)
+ self._ro.left_edges = dx - 0.5*self.bin_size
+ self._ro.right_edges = dx + 0.5*self.bin_size
+ self._ro.bottom_edges = dy - 0.5*self.bin_size
+ self._ro.top_edges = dy + 0.5*self.bin_size
+ self._ro.rnom = np.sqrt(dx**2 + dy**2)
+ self._ro.logr = np.zeros_like(self.rnom)
+ np.log(self.rnom, out=self._ro.logr, where=self.rnom > 0)
+ self._ro.logr[self.rnom==0.] = -np.inf
+ self._ro._nbins = self.nbins**2
+ self._ro._bintype = _lib.TwoD
+ min_log_bin_size = self.bin_size / self.max_sep
+ max_log_bin_size = self.bin_size / (self.min_sep + self.bin_size/2)
+ max_good_slop = 0.1 / max_log_bin_size
+ else: # pragma: no cover (Already checked by config layer)
+ raise ValueError("Invalid bin_type %s"%self.bin_type)
+
+ if self.sep_units == '':
+ self.logger.info("nbins = %d, min,max sep = %g..%g, bin_size = %g",
+ self.nbins, self.min_sep, self.max_sep, self.bin_size)
+ else:
+ self.logger.info("nbins = %d, min,max sep = %g..%g %s, bin_size = %g",
+ self.nbins, self.min_sep, self.max_sep, self.sep_units,
+ self.bin_size)
+ # The underscore-prefixed names are in natural units (radians for angles)
+ self._ro._min_sep = self.min_sep * self._sep_units
+ self._ro._max_sep = self.max_sep * self._sep_units
+ if self.bin_type in ['Linear', 'TwoD']:
+ self._ro._bin_size = self.bin_size * self._sep_units
+ min_log_bin_size *= self._sep_units
+ else:
+ self._ro._bin_size = self.bin_size
+
+ self._ro.split_method = self.config.get('split_method','mean')
+ self.logger.debug("Using split_method = %s",self.split_method)
+
+ self._ro.min_top = get(self.config,'min_top',int,None)
+ self._ro.max_top = get(self.config,'max_top',int,10)
+
+ self._ro.bin_slop = get(self.config,'bin_slop',float,-1.0)
+ if self.bin_slop < 0.0:
+ self._ro.bin_slop = min(max_good_slop, 1.0)
+ self._ro.b = min_log_bin_size * self.bin_slop
+ if self.bin_slop > max_good_slop + 0.0001: # Add some numerical slop
+ self.logger.warning(
+ "Using bin_slop = %g, bin_size = %g, b = %g\n"%(self.bin_slop,self.bin_size,self.b)+
+ "It is recommended to use bin_slop <= %s in this case.\n"%max_good_slop+
+ "Larger values of bin_slop (and hence b) may result in significant inaccuracies.")
+ else:
+ self.logger.debug("Using bin_slop = %g, b = %g",self.bin_slop,self.b)
+
+ self._ro.brute = get(self.config,'brute',bool,False)
+ if self.brute:
+ self.logger.info("Doing brute force calculation%s.",
+ self.brute is True and "" or
+ self.brute == 1 and " for first field" or
+ " for second field")
+ self.coords = None
+ self.metric = None
+ self._ro.min_rpar = get(self.config,'min_rpar',float,-sys.float_info.max)
+ self._ro.max_rpar = get(self.config,'max_rpar',float,sys.float_info.max)
+ if self.min_rpar > self.max_rpar:
+ raise ValueError("min_rpar must be <= max_rpar")
+ period = get(self.config,'period',float,0)
+ self._ro.xperiod = get(self.config,'xperiod',float,period)
+ self._ro.yperiod = get(self.config,'yperiod',float,period)
+ self._ro.zperiod = get(self.config,'zperiod',float,period)
+
+ self._ro.var_method = get(self.config,'var_method',str,'shot')
+ self._ro.num_bootstrap = get(self.config,'num_bootstrap',int,500)
+ self.results = {} # for jackknife, etc. store the results of each pair of patches.
+ self.npatch1 = self.npatch2 = 1
+ self._rng = rng
+
+ @property
+ def rng(self):
+ if self._rng is None:
+ self._rng = np.random.RandomState()
+ return self._rng
+
+ # Properties for all the read-only attributes ("ro" stands for "read-only")
+ @property
+ def output_dots(self): return self._ro.output_dots
+ @property
+ def bin_type(self): return self._ro.bin_type
+ @property
+ def sep_units(self): return self._ro.sep_units
+ @property
+ def _sep_units(self): return self._ro._sep_units
+ @property
+ def _log_sep_units(self): return self._ro._log_sep_units
+ @property
+ def min_sep(self): return self._ro.min_sep
+ @property
+ def max_sep(self): return self._ro.max_sep
+ @property
+ def bin_size(self): return self._ro.bin_size
+ @property
+ def nbins(self): return self._ro.nbins
+ @property
+ def logr(self): return self._ro.logr
+ @property
+ def rnom(self): return self._ro.rnom
+ @property
+ def left_edges(self): return self._ro.left_edges
+ @property
+ def right_edges(self): return self._ro.right_edges
+ @property
+ def top_edges(self): return self._ro.top_edges
+ @property
+ def bottom_edges(self): return self._ro.bottom_edges
+ @property
+ def _bintype(self): return self._ro._bintype
+ @property
+ def _nbins(self): return self._ro._nbins
+ @property
+ def _min_sep(self): return self._ro._min_sep
+ @property
+ def _max_sep(self): return self._ro._max_sep
+ @property
+ def _bin_size(self): return self._ro._bin_size
+ @property
+ def split_method(self): return self._ro.split_method
+ @property
+ def min_top(self): return self._ro.min_top
+ @property
+ def max_top(self): return self._ro.max_top
+ @property
+ def bin_slop(self): return self._ro.bin_slop
+ @property
+ def b(self): return self._ro.b
+ @property
+ def brute(self): return self._ro.brute
+ @property
+ def min_rpar(self): return self._ro.min_rpar
+ @property
+ def max_rpar(self): return self._ro.max_rpar
+ @property
+ def xperiod(self): return self._ro.xperiod
+ @property
+ def yperiod(self): return self._ro.yperiod
+ @property
+ def zperiod(self): return self._ro.zperiod
+ @property
+ def var_method(self): return self._ro.var_method
+ @property
+ def num_bootstrap(self): return self._ro.num_bootstrap
+ @property
+ def _d1(self): return self._ro._d1
+ @property
+ def _d2(self): return self._ro._d2
+
+ def __getstate__(self):
+ d = self.__dict__.copy()
+ d.pop('_corr',None)
+ d.pop('_ok',None) # Remake this as needed.
+ d.pop('logger',None) # Oh well. This is just lost in the copy. Can't be pickled.
+ return d
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ self._corr = None
+ self.logger = setup_logger(get(self.config,'verbose',int,1),
+ self.config.get('log_file',None))
+
+[docs] def clear(self):
+ """Clear all data vectors, the results dict, and any related values.
+ """
+ self._clear()
+ self.results = {}
+ self.npatch1 = self.npatch2 = 1
+ self.__dict__.pop('_ok',None)
+
+ @property
+ def nonzero(self):
+ """Return if there are any values accumulated yet. (i.e. npairs > 0)
+ """
+ return np.any(self.npairs)
+
+ def _add_tot(self, i, j, c1, c2):
+ # No op for all but NNCorrelation, which needs to add the tot value
+ pass
+
+ def _trivially_zero(self, c1, c2, metric):
+ # For now, ignore the metric. Just be conservative about how much space we need.
+ x1,y1,z1,s1 = c1._get_center_size()
+ x2,y2,z2,s2 = c2._get_center_size()
+ return _lib.TriviallyZero(self.corr, self._d1, self._d2, self._bintype,
+ self._metric, self._coords,
+ x1, y1, z1, s1, x2, y2, z2, s2)
+
+ def _process_all_auto(self, cat1, metric, num_threads, comm, low_mem):
+
+ def is_my_job(my_indices, i, j, n):
+ # Helper function to figure out if a given (i,j) job should be done on the
+ # current process.
+
+ # Always my job if not using MPI.
+ if my_indices is None:
+ return True
+
+ # Now the tricky part. If using MPI, we need to divide up the jobs smartly.
+ # The first point is to divvy up the auto jobs evenly. This is where most of the
+ # work is done, so we want those to be spreads as evenly as possibly across procs.
+ # Therefore, if both indices are mine, then do the job.
+ # This reduces the number of catalogs this machine needs to load up.
+ # If the auto i,i and j,j are both my job, then i and j are already being loaded
+ # on this machine, so also do that job.
+ if i in my_indices and j in my_indices:
+ self.logger.info("Rank %d: Job (%d,%d) is mine.",rank,i,j)
+ return True
+
+ # If neither index is mine, then it's not my job.
+ if i not in my_indices and j not in my_indices:
+ return False
+
+ # For the other jobs, we want to minimize how many other catalogs need to be
+ # loaded. Unfortunately, the nature of pairs is such that we can't reduce this
+ # too much. For the set of jobs i,j where i belongs to proc 1 and j belongs to proc 2,
+ # half of these pairs need to be assigned to each proc.
+ # The best I could figure for this is to give even i to proc 1 and odd i to proc 2.
+ # This means proc 1 has to load all the j catalogs, but proc 2 can skip half the i
+ # catalogs. This would naively have the result that procs with lower indices
+ # have to load more catalogs than those with higher indices, since i < j.
+ # So we reverse the procedure when j-i > n/2 to spread out the I/O more.
+ if j-i < n//2:
+ ret = i % 2 == (0 if i in my_indices else 1)
+ else:
+ ret = j % 2 == (0 if j in my_indices else 1)
+ if ret:
+ self.logger.info("Rank %d: Job (%d,%d) is mine.",rank,i,j)
+ return ret
+
+ if len(cat1) == 1 and cat1[0].npatch == 1:
+ self.process_auto(cat1[0], metric=metric, num_threads=num_threads)
+ else:
+ # When patch processing, keep track of the pair-wise results.
+ if self.npatch1 == 1:
+ self.npatch1 = self.npatch2 = cat1[0].npatch if cat1[0].npatch != 1 else len(cat1)
+ n = self.npatch1
+
+ # Setup for deciding when this is my job.
+ if comm:
+ size = comm.Get_size()
+ rank = comm.Get_rank()
+ my_indices = np.arange(n * rank // size, n * (rank+1) // size)
+ self.logger.info("Rank %d: My indices are %s",rank,my_indices)
+ else:
+ my_indices = None
+
+ self._set_metric(metric, cat1[0].coords)
+ temp = self.copy()
+ temp.results = {} # Don't mess up the original results
+ for ii,c1 in enumerate(cat1):
+ i = c1.patch if c1.patch is not None else ii
+ if is_my_job(my_indices, i, i, n):
+ temp._clear()
+ self.logger.info('Process patch %d auto',i)
+ temp.process_auto(c1, metric=metric, num_threads=num_threads)
+ if (i,i) not in self.results:
+ self.results[(i,i)] = temp.copy()
+ else:
+ self.results[(i,i)] += temp
+ self += temp
+ for jj,c2 in list(enumerate(cat1))[::-1]:
+ j = c2.patch if c2.patch is not None else jj
+ if i < j and is_my_job(my_indices, i, j, n):
+ temp._clear()
+ if not self._trivially_zero(c1,c2,metric):
+ self.logger.info('Process patches %d,%d cross',i,j)
+ temp.process_cross(c1, c2, metric=metric, num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d pair, which are too far apart ' +
+ 'for this set of separations',i,j)
+ if temp.nonzero:
+ if (i,j) not in self.results:
+ self.results[(i,j)] = temp.copy()
+ else:
+ self.results[(i,j)] += temp
+ self += temp
+ else:
+ # NNCorrelation needs to add the tot value
+ self._add_tot(i, j, c1, c2)
+ if low_mem and jj != ii+1:
+ # Don't unload i+1, since that's the next one we'll need.
+ c2.unload()
+ if low_mem:
+ c1.unload()
+ if comm is not None:
+ rank = comm.Get_rank()
+ size = comm.Get_size()
+ self.logger.info("Rank %d: Completed jobs %s",rank,list(self.results.keys()))
+ # Send all the results back to rank 0 process.
+ if rank > 0:
+ comm.send(self, dest=0)
+ else:
+ for p in range(1,size):
+ temp = comm.recv(source=p)
+ self += temp
+ self.results.update(temp.results)
+
+ def _process_all_cross(self, cat1, cat2, metric, num_threads, comm, low_mem):
+
+ def is_my_job(my_indices, i, j, n1, n2):
+ # Helper function to figure out if a given (i,j) job should be done on the
+ # current process.
+
+ # Always my job if not using MPI.
+ if my_indices is None:
+ return True
+
+ # This is much simpler than in the auto case, since the set of catalogs for
+ # cat1 and cat2 are different, we can just split up one of them among the jobs.
+ if n1 > n2:
+ k = i
+ else:
+ k = j
+ if k in my_indices:
+ self.logger.info("Rank %d: Job (%d,%d) is mine.",rank,i,j)
+ return True
+ else:
+ return False
+
+ if get(self.config,'pairwise',bool,False):
+ import warnings
+ warnings.warn("The pairwise option is slated to be removed in a future version. "+
+ "If you are actually using this parameter usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if len(cat1) != len(cat2):
+ raise ValueError("Number of files for 1 and 2 must be equal for pairwise.")
+ for c1,c2 in zip(cat1,cat2):
+ if c1.ntot != c2.ntot:
+ raise ValueError("Number of objects must be equal for pairwise.")
+ self.process_pairwise(c1, c2, metric=metric, num_threads=num_threads)
+ elif len(cat1) == 1 and len(cat2) == 1 and cat1[0].npatch == 1 and cat2[0].npatch == 1:
+ self.process_cross(cat1[0], cat2[0], metric=metric, num_threads=num_threads)
+ else:
+ # When patch processing, keep track of the pair-wise results.
+ if self.npatch1 == 1:
+ self.npatch1 = cat1[0].npatch if cat1[0].npatch != 1 else len(cat1)
+ if self.npatch2 == 1:
+ self.npatch2 = cat2[0].npatch if cat2[0].npatch != 1 else len(cat2)
+ if self.npatch1 != self.npatch2 and self.npatch1 != 1 and self.npatch2 != 1:
+ raise RuntimeError("Cross correlation requires both catalogs use the same patches.")
+
+ # Setup for deciding when this is my job.
+ n1 = self.npatch1
+ n2 = self.npatch2
+ if comm:
+ size = comm.Get_size()
+ rank = comm.Get_rank()
+ n = max(n1,n2)
+ my_indices = np.arange(n * rank // size, n * (rank+1) // size)
+ self.logger.info("Rank %d: My indices are %s",rank,my_indices)
+ else:
+ my_indices = None
+
+ self._set_metric(metric, cat1[0].coords, cat2[0].coords)
+ temp = self.copy()
+ temp.results = {} # Don't mess up the original results
+ for ii,c1 in enumerate(cat1):
+ i = c1.patch if c1.patch is not None else ii
+ for jj,c2 in enumerate(cat2):
+ j = c2.patch if c2.patch is not None else jj
+ if is_my_job(my_indices, i, j, n1, n2):
+ temp._clear()
+ if not self._trivially_zero(c1,c2,metric):
+ self.logger.info('Process patches %d,%d cross',i,j)
+ temp.process_cross(c1, c2, metric=metric, num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d pair, which are too far apart ' +
+ 'for this set of separations',i,j)
+ if temp.nonzero or i==j or n1==1 or n2==1:
+ if (i,j) not in self.results:
+ self.results[(i,j)] = temp.copy()
+ else:
+ self.results[(i,j)] += temp
+ self += temp
+ else:
+ # NNCorrelation needs to add the tot value
+ self._add_tot(i, j, c1, c2)
+ if low_mem:
+ c2.unload()
+ if low_mem:
+ c1.unload()
+ if comm is not None:
+ rank = comm.Get_rank()
+ size = comm.Get_size()
+ self.logger.info("Rank %d: Completed jobs %s",rank,list(self.results.keys()))
+ # Send all the results back to rank 0 process.
+ if rank > 0:
+ comm.send(self, dest=0)
+ else:
+ for p in range(1,size):
+ temp = comm.recv(source=p)
+ self += temp
+ self.results.update(temp.results)
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ Usually, this is just self.xi. But if the metric is TwoD, this becomes self.xi.ravel().
+ And for `GGCorrelation`, it is the concatenation of self.xip and self.xim.
+ """
+ return self.xi.ravel()
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ This is the weight array corresponding to `getStat`. Usually just self.weight, but
+ raveled for TwoD and duplicated for GGCorrelation to match what `getStat` does in
+ those cases.
+ """
+ return self.weight.ravel()
+
+[docs] @depr_pos_kwargs
+ def estimate_cov(self, method, *, func=None, comm=None):
+ """Estimate the covariance matrix based on the data
+
+ This function will calculate an estimate of the covariance matrix according to the
+ given method.
+
+ Options for ``method`` include:
+
+ - 'shot' = The variance based on "shot noise" only. This includes the Poisson
+ counts of points for N statistics, shape noise for G statistics, and the observed
+ scatter in the values for K statistics. In this case, the returned covariance
+ matrix will be diagonal, since there is no way to estimate the off-diagonal terms.
+ - 'jackknife' = A jackknife estimate of the covariance matrix based on the scatter
+ in the measurement when excluding one patch at a time.
+ - 'sample' = An estimate based on the sample covariance of a set of samples,
+ taken as the patches of the input catalog.
+ - 'bootstrap' = A bootstrap covariance estimate. It selects patches at random with
+ replacement and then generates the statistic using all the auto-correlations at
+ their selected repetition plus all the cross terms that aren't actually auto terms.
+ - 'marked_bootstrap' = An estimate based on a marked-point bootstrap resampling of the
+ patches. Similar to bootstrap, but only samples the patches of the first catalog and
+ uses all patches from the second catalog that correspond to each patch selection of
+ the first catalog. Based on the algorithm presented in Loh (2008).
+ cf. https://ui.adsabs.harvard.edu/abs/2008ApJ...681..726L/
+
+ Both 'bootstrap' and 'marked_bootstrap' use the num_bootstrap parameter, which can be set on
+ construction.
+
+ .. note::
+
+ For most classes, there is only a single statistic, so this calculates a covariance
+ matrix for that vector. `GGCorrelation` has two: ``xip`` and ``xim``, so in this
+ case the full data vector is ``xip`` followed by ``xim``, and this calculates the
+ covariance matrix for that full vector including both statistics. The helper
+ function `getStat` returns the relevant statistic in all cases.
+
+ In all cases, the relevant processing needs to already have been completed and finalized.
+ And for all methods other than 'shot', the processing should have involved an appropriate
+ number of patches -- preferably more patches than the length of the vector for your
+ statistic, although this is not checked.
+
+ The default data vector to use for the covariance matrix is given by the method
+ `getStat`. As noted above, this is usually just self.xi. However, there is an option
+ to compute the covariance of some other function of the correlation object by providing
+ an arbitrary function, ``func``, which should act on the current correlation object
+ and return the data vector of interest.
+
+ For instance, for an `NGCorrelation`, you might want to compute the covariance of the
+ imaginary part, ``ng.xi_im``, rather than the real part. In this case you could use
+
+ >>> func = lambda ng: ng.xi_im
+
+ The return value from this func should be a single numpy array. (This is not directly
+ checked, but you'll probably get some kind of exception if it doesn't behave as expected.)
+
+ .. note::
+
+ The optional ``func`` parameter is not valid in conjunction with ``method='shot'``.
+ It only works for the methods that are based on patch combinations.
+
+ This function can be parallelized by passing the comm argument as an mpi4py communicator
+ to parallelize using that. For MPI, all processes should have the same inputs.
+ If method == "shot" then parallelization has no effect.
+
+ Parameters:
+ method (str): Which method to use to estimate the covariance matrix.
+ func (function): A unary function that acts on the current correlation object and
+ returns the desired data vector. [default: None, which is
+ equivalent to ``lambda corr: corr.getStat()``.
+ comm (mpi comm) If not None, run under MPI
+
+ Returns:
+ A numpy array with the estimated covariance matrix.
+ """
+ if func is not None:
+ # Need to convert it to a function of the first item in the list.
+ all_func = lambda corrs: func(corrs[0])
+ else:
+ all_func = None
+ return estimate_multi_cov([self], method=method, func=all_func, comm=comm)
+
+[docs] def build_cov_design_matrix(self, method, *, func=None, comm=None):
+ """Build the design matrix that is used for estimating the covariance matrix.
+
+ The design matrix for patch-based covariance estimates is a matrix where each row
+ corresponds to a different estimate of the data vector, :math:`\\xi_i` (or
+ :math:`f(\\xi_i)` if using the optional ``func`` parameter).
+
+ The different of rows in the matrix for each valid ``method`` are:
+
+ - 'shot': This method is not valid here.
+ - 'jackknife': The data vector when excluding a single patch.
+ - 'sample': The data vector using only a single patch for the first catalog.
+ - 'bootstrap': The data vector for a random resampling of the patches keeping the
+ sample total number, but allowing some to repeat. Cross terms from repeated patches
+ are excluded (since they are really auto terms).
+ - 'marked_bootstrap': The data vector for a random resampling of patches in the first
+ catalog, using all patches for the second catalog. Based on the algorithm in
+ Loh(2008).
+
+ See `estimate_cov` for more details.
+
+ The return value includes both the design matrix and a vector of weights (the total weight
+ array in the computed correlation functions). The weights are used for the sample method
+ when estimating the covariance matrix. The other methods ignore them, but they are provided
+ here in case they are useful.
+
+ Parameters:
+ method (str): Which method to use to estimate the covariance matrix.
+ func (function): A unary function that takes the list ``corrs`` and returns the
+ desired full data vector. [default: None, which is equivalent to
+ ``lambda corrs: np.concatenate([c.getStat() for c in corrs])``]
+ comm (mpi comm) If not None, run under MPI
+
+ Returns:
+ A, w: numpy arrays with the design matrix and weights respectively.
+ """
+ if func is not None:
+ # Need to convert it to a function of the first item in the list.
+ all_func = lambda corrs: func(corrs[0])
+ else:
+ all_func = None
+ return build_multi_cov_design_matrix([self], method=method, func=all_func, comm=comm)
+
+ def _set_num_threads(self, num_threads):
+ if num_threads is None:
+ num_threads = self.config.get('num_threads',None)
+ # Recheck.
+ if num_threads is None:
+ self.logger.debug('Set num_threads automatically')
+ else:
+ self.logger.debug('Set num_threads = %d',num_threads)
+ set_omp_threads(num_threads, self.logger)
+
+ def _set_metric(self, metric, coords1, coords2=None):
+ if metric is None:
+ metric = get(self.config,'metric',str,'Euclidean')
+ coords, metric = parse_metric(metric, coords1, coords2)
+ if coords != '3d':
+ if self.min_rpar != -sys.float_info.max:
+ raise ValueError("min_rpar is only valid for 3d coordinates")
+ if self.max_rpar != sys.float_info.max:
+ raise ValueError("max_rpar is only valid for 3d coordinates")
+ if self.sep_units != '' and coords == '3d' and metric != 'Arc':
+ raise ValueError("sep_units is invalid with 3d coordinates. "
+ "min_sep and max_sep should be in the same units as r (or x,y,z).")
+ if self.coords is not None or self.metric is not None:
+ if coords != self.coords:
+ self.logger.warning("Detected a change in catalog coordinate systems.\n"+
+ "This probably doesn't make sense!")
+ if metric != self.metric:
+ self.logger.warning("Detected a change in metric.\n"+
+ "This probably doesn't make sense!")
+ if metric == 'Periodic':
+ if self.xperiod == 0 or self.yperiod == 0 or (coords=='3d' and self.zperiod == 0):
+ raise ValueError("Periodic metric requires setting the period to use.")
+ else:
+ if self.xperiod != 0 or self.yperiod != 0 or self.zperiod != 0:
+ raise ValueError("period options are not valid for %s metric."%metric)
+ self.coords = coords # These are the regular string values
+ self.metric = metric
+ self._coords = coord_enum(coords) # These are the C++-layer enums
+ self._metric = metric_enum(metric)
+
+ def _apply_units(self, mask):
+ if self.coords == 'spherical' and self.metric == 'Euclidean':
+ # Then our distances are all angles. Convert from the chord distance to a real angle.
+ # L = 2 sin(theta/2)
+ self.meanr[mask] = 2. * np.arcsin(self.meanr[mask]/2.)
+ self.meanlogr[mask] = np.log( 2. * np.arcsin(np.exp(self.meanlogr[mask])/2.) )
+ self.meanr[mask] /= self._sep_units
+ self.meanlogr[mask] -= self._log_sep_units
+
+ def _get_minmax_size(self):
+ if self.metric == 'Euclidean':
+ # The minimum size cell that will be useful is one where two cells that just barely
+ # don't split have (d + s1 + s2) = minsep
+ # The largest s2 we need to worry about is s2 = 2s1.
+ # i.e. d = minsep - 3s1 and s1 = 0.5 * bd
+ # d = minsep - 1.5 bd
+ # d = minsep / (1+1.5 b)
+ # s = 0.5 * b * minsep / (1+1.5 b)
+ # = b * minsep / (2+3b)
+ min_size = self._min_sep * self.b / (2.+3.*self.b)
+
+ # The maximum size cell that will be useful is one where a cell of size s will
+ # be split at the maximum separation even if the other size = 0.
+ # i.e. max_size = max_sep * b
+ max_size = self._max_sep * self.b
+ return min_size, max_size
+ else:
+ # For other metrics, the above calculation doesn't really apply, so just skip
+ # this relatively modest optimization and go all the way to the leaves.
+ # (And for the max_size, always split 10 levels for the top-level cells.)
+ return 0., 0.
+
+[docs] @depr_pos_kwargs
+ def sample_pairs(self, n, cat1, cat2, *, min_sep, max_sep, metric=None):
+ """Return a random sample of n pairs whose separations fall between min_sep and max_sep.
+
+ This would typically be used to get some random subset of the indices of pairs that
+ fell into a particular bin of the correlation. E.g. to get 100 pairs from the third
+ bin of a `BinnedCorr2` instance, corr, you could write::
+
+ >>> min_sep = corr.left_edges[2] # third bin has i=2
+ >>> max_sep = corr.right_edges[2]
+ >>> i1, i2, sep = corr.sample_pairs(100, cat1, cat2, min_sep, max_sep)
+
+ The min_sep and max_sep should use the same units as were defined when constructing
+ the corr instance.
+
+ The selection process will also use the same bin_slop as specified (either explicitly or
+ implicitly) when constructing the corr instance. This means that some of the pairs may
+ have actual separations slightly outside of the specified range. If you want a selection
+ using an exact range without any slop, you should construct a new Correlation instance
+ with bin_slop=0, and call sample_pairs with that.
+
+ The returned separations will likewise correspond to the separation of the cells in the
+ tree that TreeCorr used to place the pairs into the given bin. Therefore, if these cells
+ were not leaf cells, then they will not typically be equal to the real separations for the
+ given metric. If you care about the exact separations for each pair, you should either
+ call sample_pairs from a Correlation instance with brute=True or recalculate the
+ distances yourself from the original data.
+
+ Also, note that min_sep and max_sep may be arbitrary. There is no requirement that they
+ be edges of one of the standard bins for this correlation function. There is also no
+ requirement that this correlation instance has already accumulated pairs via a call
+ to process with these catalogs.
+
+ Parameters:
+ n (int): How many samples to return.
+ cat1 (Catalog): The catalog from which to sample the first object of each pair.
+ cat2 (Catalog): The catalog from which to sample the second object of each pair.
+ (This may be the same as cat1.)
+ min_sep (float): The minimum separation for the returned pairs (modulo some slop
+ allowed by the bin_slop parameter). (Note: keyword name is required
+ for this parameter: min_sep=min_sep)
+ max_sep (float): The maximum separation for the returned pairs (modulo some slop
+ allowed by the bin_slop parameter). (Note: keyword name is required
+ for this parameter: max_sep=max_sep)
+ metric (str): Which metric to use. See `Metrics` for details. (default:
+ self.metric, or 'Euclidean' if not set yet)
+
+ Returns:
+ Tuple containing
+
+ - i1 (array): indices of objects from cat1
+ - i2 (array): indices of objects from cat2
+ - sep (array): separations of the pairs of objects (i1,i2)
+ """
+ from .util import long_ptr as lp
+ from .util import double_ptr as dp
+
+ if metric is None:
+ metric = self.config.get('metric', 'Euclidean')
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+
+ f1 = cat1.field
+ f2 = cat2.field
+
+ if f1 is None or f1._coords != self._coords:
+ # I don't really know if it's possible to get the coords out of sync,
+ # so the 2nd check might be superfluous.
+ # The first one though is definitely possible, so we need to check that.
+ self.logger.debug("In sample_pairs, making default field for cat1")
+ min_size, max_size = self._get_minmax_size()
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ if f2 is None or f2._coords != self._coords:
+ self.logger.debug("In sample_pairs, making default field for cat2")
+ min_size, max_size = self._get_minmax_size()
+ f2 = cat2.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ # Apply units to min_sep, max_sep:
+ min_sep *= self._sep_units
+ max_sep *= self._sep_units
+
+ i1 = np.zeros(n, dtype=int)
+ i2 = np.zeros(n, dtype=int)
+ sep = np.zeros(n, dtype=float)
+ ntot = _lib.SamplePairs(self.corr, f1.data, f2.data, min_sep, max_sep,
+ f1._d, f2._d, self._coords, self._bintype, self._metric,
+ lp(i1), lp(i2), dp(sep), n)
+
+ if ntot < n:
+ n = ntot
+ i1 = i1[:n]
+ i2 = i2[:n]
+ sep = sep[:n]
+ # Convert back to nominal units
+ sep /= self._sep_units
+ self.logger.info("Sampled %d pairs out of a total of %d.", n, ntot)
+
+ return i1, i2, sep
+
+ # Some helper functions that are relevant for doing the covariance stuff below.
+ # Note: the word "pairs" in many of these is appropriate for 2pt, but in the 3pt case
+ # (cf. binnedcorr3.py), these actually refer to triples (i,j,k).
+
+ def _get_npatch(self):
+ return max(self.npatch1, self.npatch2)
+
+ def _calculate_xi_from_pairs(self, pairs):
+ # Compute the xi data vector for the given list of pairs.
+ # pairs is input as a list of (i,j) values.
+
+ # This is the normal calculation. It needs to be overridden when there are randoms.
+ self._sum([self.results[ij] for ij in pairs])
+ self._finalize()
+
+ #########################################################################################
+ # #
+ # Important note for the following two functions. #
+ # These use to have lines like this: ` #
+ # #
+ # return [ [(j,k) for j,k in self.results.keys() if j!=i and k!=i] #
+ # for i in range(self.npatch1) ] #
+ # #
+ # This double list comprehension ends up with a list of lists that takes O(npatch^3) #
+ # memory, which for moderately large npatch values (say 500) can be multip[le GBytes. #
+ # #
+ # The straightforward solution was to change this to using generators: #
+ # #
+ # return [ ((j,k) for j,k in self.results.keys() if j!=i and k!=i) #
+ # for i in range(self.npatch1) ] #
+ # #
+ # But this doesn't work with the MPI covariance calculation, since generators aren't #
+ # picklable. So the iterator classes below hold off making the generator until the #
+ # iteration is actually started, which keeps them picklable. #
+ # #
+ #########################################################################################
+
+ class PairIterator(collections.abc.Iterator):
+ def __init__(self, results, npatch1, npatch2, index, ok=None):
+ self.results = results
+ self.npatch1 = npatch1
+ self.npatch2 = npatch2
+ self.index = index
+ self.ok = ok
+
+ def __iter__(self):
+ self.gen = iter(self.make_gen())
+ return self
+
+ def __next__(self):
+ return next(self.gen)
+
+ class JackknifePairIterator(PairIterator):
+ def make_gen(self):
+ if self.npatch2 == 1:
+ # k=0 here
+ return ((j,k) for j,k in self.results.keys() if j!=self.index)
+ elif self.npatch1 == 1:
+ # j=0 here
+ return ((j,k) for j,k in self.results.keys() if k!=self.index)
+ else:
+ # For each i:
+ # Select all pairs where neither is i.
+ assert self.npatch1 == self.npatch2
+ return ((j,k) for j,k in self.results.keys() if j!=self.index and k!=self.index)
+
+ def _jackknife_pairs(self):
+ np = self.npatch1 if self.npatch1 != 1 else self.npatch2
+ return [self.JackknifePairIterator(self.results, self.npatch1, self.npatch2, i)
+ for i in range(np)]
+
+ class SamplePairIterator(PairIterator):
+ def make_gen(self):
+ if self.npatch2 == 1:
+ # k=0 here.
+ return ((j,k) for j,k in self.results.keys() if j==self.index)
+ elif self.npatch1 == 1:
+ # j=0 here.
+ return ((j,k) for j,k in self.results.keys() if k==self.index)
+ else:
+ assert self.npatch1 == self.npatch2
+ # Note: It's not obvious to me a priori which of these should be the right choice.
+ # Empirically, they both underestimate the variance, but the second one
+ # does so less on the tests I have in test_patch.py. So that's the one I'm
+ # using.
+ # For each i:
+ # Select all pairs where either is i.
+ #return ((j,k) for j,k in self.results.keys() if j==self.index or k==self.index)
+ #
+ # For each i:
+ # Select all pairs where first is i.
+ return ((j,k) for j,k in self.results.keys() if j==self.index)
+
+ def _sample_pairs(self):
+ np = self.npatch1 if self.npatch1 != 1 else self.npatch2
+ return [self.SamplePairIterator(self.results, self.npatch1, self.npatch2, i)
+ for i in range(np)]
+
+ @lazy_property
+ def _ok(self):
+ # It's much faster to make the pair lists for bootstrap iterators if we keep track of
+ # which (i,j) pairs are in the results dict using an "ok" matrix for quick access.
+ ok = np.zeros((self.npatch1, self.npatch2), dtype=bool)
+ for (i,j) in self.results:
+ ok[i,j] = True
+ return ok
+
+ class MarkedPairIterator(PairIterator):
+ def make_gen(self):
+ if self.npatch2 == 1:
+ return ( (i,0) for i in self.index if self.ok[i,0] )
+ elif self.npatch1 == 1:
+ return ( (0,i) for i in self.index if self.ok[0,i] )
+ else:
+ assert self.npatch1 == self.npatch2
+ # Select all pairs where first point is in index (repeating i as appropriate)
+ return ( (i,j) for i in self.index for j in range(self.npatch2) if self.ok[i,j] )
+
+ def _marked_pairs(self, index):
+ return self.MarkedPairIterator(self.results, self.npatch1, self.npatch2, index, self._ok)
+
+ class BootstrapPairIterator(PairIterator):
+ def make_gen(self):
+ if self.npatch2 == 1:
+ return ( (i,0) for i in self.index if self.ok[i,0] )
+ elif self.npatch1 == 1:
+ return ( (0,i) for i in self.index if self.ok[0,i] )
+ else:
+ assert self.npatch1 == self.npatch2
+ # Include all represented auto-correlations once, repeating as appropriate.
+ # This needs to be done separately from the below step to avoid extra pairs (i,i)
+ # that you would get by looping i in index and j in index for cases where i=j at
+ # different places in the index list. E.g. if i=3 shows up 3 times in index, then
+ # the naive way would get 9 instance of (3,3), whereas we only want 3 instances.
+ ret1 = ( (i,i) for i in self.index if self.ok[i,i] )
+
+ # And all other pairs that aren't really auto-correlations.
+ # These can happen at their natural multiplicity from i and j loops.
+ # Note: This is way faster with the precomputed ok matrix.
+ # Like 0.005 seconds per call rather than 1.2 seconds for 128 patches!
+ ret2 = ( (i,j) for i in self.index for j in self.index if self.ok[i,j] and i!=j )
+
+ return itertools.chain(ret1, ret2)
+
+ def _bootstrap_pairs(self, index):
+ return self.BootstrapPairIterator(self.results, self.npatch1, self.npatch2, index, self._ok)
+
+ def _write(self, writer, name, write_patch_results, zero_tot=False):
+ # These helper properties define what to write for each class.
+ col_names = self._write_col_names
+ data = self._write_data
+ params = self._write_params
+
+ if write_patch_results:
+ # Note: Only include npatch1, npatch2 in serialization if we are also serializing
+ # results. Otherwise, the corr that is read in will behave oddly.
+ params['npatch1'] = self.npatch1
+ params['npatch2'] = self.npatch2
+ params['num_rows'] = len(self.rnom.ravel())
+ num_patch_pairs = len(self.results)
+ if zero_tot:
+ i = 0
+ for key, corr in self.results.items():
+ if not corr._nonzero:
+ zp_name = name + '_zp_%d'%i
+ params[zp_name] = repr((key, corr.tot))
+ num_patch_pairs -= 1
+ i += 1
+ params['num_zero_patch'] = i
+ params['num_patch_pairs'] = num_patch_pairs
+
+ writer.write(col_names, data, params=params, ext=name)
+ if write_patch_results:
+ writer.set_precision(16)
+ i = 0
+ for key, corr in self.results.items():
+ if zero_tot and not corr._nonzero: continue
+ col_names = corr._write_col_names
+ data = corr._write_data
+ params = corr._write_params
+ params['key'] = repr(key)
+ pp_name = name + '_pp_%d'%i
+ writer.write(col_names, data, params=params, ext=pp_name)
+ i += 1
+ assert i == num_patch_pairs
+
+ def _read(self, reader, name=None):
+ name = 'main' if 'main' in reader and name is None else name
+ params = reader.read_params(ext=name)
+ num_rows = params.get('num_rows', None)
+ num_patch_pairs = params.get('num_patch_pairs', 0)
+ num_zero_patch = params.get('num_zero_patch', 0)
+ name = 'main' if num_patch_pairs and name is None else name
+ data = reader.read_data(max_rows=num_rows, ext=name)
+
+ # This helper function defines how to set the attributes for each class
+ # based on what was read in.
+ self._read_from_data(data, params)
+
+ self.results = {}
+ for i in range(num_zero_patch):
+ zp_name = name + '_zp_%d'%i
+ key, tot = eval(params[zp_name])
+ self.results[key] = self._zero_copy(tot)
+ for i in range(num_patch_pairs):
+ pp_name = name + '_pp_%d'%i
+ corr = self.copy()
+ params = reader.read_params(ext=pp_name)
+ data = reader.read_data(max_rows=num_rows, ext=pp_name)
+ corr._read_from_data(data, params)
+ key = eval(params['key'])
+ self.results[key] = corr
+
+[docs]@depr_pos_kwargs
+def estimate_multi_cov(corrs, method, *, func=None, comm=None):
+ """Estimate the covariance matrix of multiple statistics.
+
+ This is like the method `BinnedCorr2.estimate_cov`, except that it will acoommodate
+ multiple statistics from a list ``corrs`` of `BinnedCorr2` objects.
+
+ Options for ``method`` include:
+
+ - 'shot' = The variance based on "shot noise" only. This includes the Poisson
+ counts of points for N statistics, shape noise for G statistics, and the observed
+ scatter in the values for K statistics. In this case, the returned covariance
+ matrix will be diagonal, since there is no way to estimate the off-diagonal terms.
+ - 'jackknife' = A jackknife estimate of the covariance matrix based on the scatter
+ in the measurement when excluding one patch at a time.
+ - 'sample' = An estimate based on the sample covariance of a set of samples,
+ taken as the patches of the input catalog.
+ - 'bootstrap' = A bootstrap covariance estimate. It selects patches at random with
+ replacement and then generates the statistic using all the auto-correlations at
+ their selected repetition plus all the cross terms that aren't actually auto terms.
+ - 'marked_bootstrap' = An estimate based on a marked-point bootstrap resampling of the
+ patches. Similar to bootstrap, but only samples the patches of the first catalog and
+ uses all patches from the second catalog that correspond to each patch selection of
+ the first catalog. Based on the algorithm presented in Loh (2008).
+ cf. https://ui.adsabs.harvard.edu/abs/2008ApJ...681..726L/
+
+ Both 'bootstrap' and 'marked_bootstrap' use the num_bootstrap parameter, which can be set on
+ construction.
+
+ For example, to find the combined covariance matrix for an NG tangential shear statistc,
+ along with the GG xi+ and xi- from the same area, using jackknife covariance estimation,
+ you would write::
+
+ >>> cov = treecorr.estimate_multi_cov([ng,gg], method='jackknife')
+
+ In all cases, the relevant processing needs to already have been completed and finalized.
+ And for all methods other than 'shot', the processing should have involved an appropriate
+ number of patches -- preferably more patches than the length of the vector for your
+ statistic, although this is not checked.
+
+ The default order of the covariance matrix is to simply concatenate the data vectors
+ for each corr in the list ``corrs``. However, if you want to do something more complicated,
+ you may provide an arbitrary function, ``func``, which should act on the list of correlations.
+ For instance, if you have several `GGCorrelation` objects and would like to order the
+ covariance such that all xi+ results come first, and then all xi- results, you could use
+
+ >>> func = lambda corrs: np.concatenate([c.xip for c in corrs] + [c.xim for c in corrs])
+
+ Or if you want to compute the covariance matrix of some derived quantity like the ratio
+ of two correlations, you could use
+
+ >>> func = lambda corrs: corrs[0].xi / corrs[1].xi
+
+ This function can be parallelized by passing the comm argument as an mpi4py communicator to
+ parallelize using that. For MPI, all processes should have the same inputs.
+ If method == "shot" then parallelization has no effect.
+
+ The return value from this func should be a single numpy array. (This is not directly
+ checked, but you'll probably get some kind of exception if it doesn't behave as expected.)
+
+ .. note::
+
+ The optional ``func`` parameter is not valid in conjunction with ``method='shot'``.
+ It only works for the methods that are based on patch combinations.
+
+ Parameters:
+ corrs (list): A list of `BinnedCorr2` instances.
+ method (str): Which method to use to estimate the covariance matrix.
+ func (function): A unary function that takes the list ``corrs`` and returns the
+ desired full data vector. [default: None, which is equivalent to
+ ``lambda corrs: np.concatenate([c.getStat() for c in corrs])``]
+ comm (mpi comm) If not None, run under MPI
+
+ Returns:
+ A numpy array with the estimated covariance matrix.
+ """
+ if method == 'shot':
+ if func is not None:
+ raise ValueError("func is invalid with method='shot'")
+ return _cov_shot(corrs)
+ elif method == 'jackknife':
+ return _cov_jackknife(corrs, func, comm=comm)
+ elif method == 'bootstrap':
+ return _cov_bootstrap(corrs, func, comm=comm)
+ elif method == 'marked_bootstrap':
+ return _cov_marked(corrs, func, comm=comm)
+ elif method == 'sample':
+ return _cov_sample(corrs, func, comm=comm)
+ else:
+ raise ValueError("Invalid method: %s"%method)
+
+[docs]def build_multi_cov_design_matrix(corrs, method, *, func=None, comm=None):
+ """Build the design matrix that is used for estimating the covariance matrix.
+
+ The design matrix for patch-based covariance estimates is a matrix where each row
+ corresponds to a different estimate of the data vector, :math:`\\xi_i` (or
+ :math:`f(\\xi_i)` if using the optional ``func`` parameter).
+
+ The different of rows in the matrix for each valid ``method`` are:
+
+ - 'shot': This method is not valid here.
+ - 'jackknife': The data vector when excluding a single patch.
+ - 'sample': The data vector using only a single patch for the first catalog.
+ - 'bootstrap': The data vector for a random resampling of the patches keeping the
+ sample total number, but allowing some to repeat. Cross terms from repeated patches
+ are excluded (since they are really auto terms).
+ - 'marked_bootstrap': The data vector for a random resampling of patches in the first
+ catalog, using all patches for the second catalog. Based on the algorithm in Loh(2008).
+
+ See `estimate_multi_cov` for more details.
+
+ The return value includes both the design matrix and a vector of weights (the total weight
+ array in the computed correlation functions). The weights are used for the sample method
+ when estimating the covariance matrix. The other methods ignore them, but they are provided
+ here in case they are useful.
+
+ Parameters:
+ corrs (list): A list of `BinnedCorr2` instances.
+ method (str): Which method to use to estimate the covariance matrix.
+ func (function): A unary function that takes the list ``corrs`` and returns the
+ desired full data vector. [default: None, which is equivalent to
+ ``lambda corrs: np.concatenate([c.getStat() for c in corrs])``]
+ comm (mpi comm) If not None, run under MPI
+
+ Returns:
+ A, w: numpy arrays with the design matrix and weights respectively.
+ """
+ if method == 'shot':
+ raise ValueError("There is no design matrix for method='shot'")
+ elif method == 'jackknife':
+ return _design_jackknife(corrs, func, comm=comm)
+ elif method == 'bootstrap':
+ return _design_bootstrap(corrs, func, comm=comm)
+ elif method == 'marked_bootstrap':
+ return _design_marked(corrs, func, comm=comm)
+ elif method == 'sample':
+ return _design_sample(corrs, func, comm=comm)
+ else:
+ raise ValueError("Invalid method: %s"%method)
+
+def _make_cov_design_matrix_core(corrs, plist, func, name, rank=0, size=1):
+ # plist has the pairs to use for each row in the design matrix for each correlation fn.
+ # It is a list by row, each element is a list by corr fn of tuples (i,j), being the indices
+ # to use from the results dict.
+ # We aggregate and finalize each correlation function based on those pairs, and then call
+ # the function func on that list of correlation objects. This is the data vector for
+ # each row in the design matrix.
+ # We also make a parallel array of the total weight in each row in case the calling routing
+ # needs it. So far, only sample uses the returned w, but it's very little overhead to compute
+ # it, and only a small memory overhead to build that array and return it.
+
+ # Make a copy of the correlation objects, so we can overwrite things without breaking
+ # the original.
+ corrs = [c.copy() for c in corrs]
+
+ # We can't pickle functions to send via MPI, so have to do this here.
+ if func is None:
+ func = lambda corrs: np.concatenate([c.getStat() for c in corrs])
+
+ # Figure out the shape of the design matrix.
+ v1 = func(corrs)
+ dt = v1.dtype
+ vsize = len(v1)
+ nrows = len(plist)
+
+ # Make the empty return arrays. They are filled with zeros
+ # because we will sum them over processes later.
+ v = np.zeros((nrows,vsize), dtype=dt)
+ w = np.zeros(nrows, dtype=float)
+
+ for row, pairs in enumerate(plist):
+ if row % size != rank:
+ continue
+ for c, cpairs in zip(corrs, pairs):
+ cpairs = list(cpairs)
+ if len(cpairs) == 0:
+ # This will cause problems downstream if we let it go.
+ # It probably indicates user error, using an inappropriate covariance estimator.
+ # So warn about it, and then do something not too crazy.
+ c.logger.error("WARNING: A xi for calculating the %s covariance has no "%name +
+ "patch pairs. This probably means these patch specifications "
+ "are inappropriate for these data.")
+ c._clear()
+ else:
+ c._calculate_xi_from_pairs(cpairs)
+ v[row] = func(corrs)
+ w[row] = np.sum([np.sum(c.getWeight()) for c in corrs])
+ return v,w
+
+def _make_cov_design_matrix(corrs, plist, func, name, comm=None):
+ if comm is not None:
+ from mpi4py import MPI
+ v, w = _make_cov_design_matrix_core(corrs, plist, func, name, comm.rank, comm.size)
+ # These two calls collects the v arrays from w arrays from all the processors,
+ # sums them all together, and then sends them back to each processor where they
+ # are put back in-place, overwriting the original v and w array contents.
+ # Each process has an array which is zeros except for the rows it is responsible
+ # for, so the sum fills in the entire array.
+ # Using "Allreduce" instead of "Reduce" means that all processes get a copy of the
+ # final arrays. This may or may not be needed depending on what users subsequently
+ # do with the matrix, but is fast since this matrix isn't large.
+ comm.Allreduce(MPI.IN_PLACE, v)
+ comm.Allreduce(MPI.IN_PLACE, w)
+ # Otherwise we just use the regular version, which implicitly does the whole matrix
+ else:
+ v, w = _make_cov_design_matrix_core(corrs, plist, func, name)
+ return v, w
+
+def _cov_shot(corrs):
+ # Shot noise "covariance" is just 1/RR or var(g)/weight or var(k)/weight, etc.
+ # Except for NN, the denominator is always corr.weight.
+ # For NN, the denominator is set by calculateXi to be RR.weight.
+ # The numerators are set appropriately for each kind of correlation function as _var_num
+ # when doing finalize, or for NN also in calculateXi.
+ # We return it as a covariance matrix for consistency with the other cov functions,
+ # but the off diagonal terms are all zero.
+ vlist = []
+ for c in corrs:
+ v = c.getWeight().copy()
+ mask1 = v != 0
+ # Note: if w=0 anywhere, leave v=0 there, rather than divide by zero.
+ v[mask1] = c._var_num / v[mask1]
+ vlist.append(v)
+ return np.diag(np.concatenate(vlist)) # Return as a covariance matrix
+
+def _check_patch_nums(corrs, name):
+ # Figure out what pairs (i,j) are possible for these correlation functions.
+ # Check that the patches used are compatible, and return the npatch to use.
+
+ for c in corrs:
+ if len(c.results) == 0:
+ raise ValueError("Using %s covariance requires using patches."%name)
+ npatch = corrs[0]._get_npatch()
+ for c in corrs[1:]:
+ if c._get_npatch() != npatch:
+ raise RuntimeError("All correlations must use the same number of patches")
+ return npatch
+
+def _design_jackknife(corrs, func, comm=None):
+ npatch = _check_patch_nums(corrs, 'jackknife')
+ plist = [c._jackknife_pairs() for c in corrs]
+ # Swap order of plist. Right now it's a list for each corr of a list for each row.
+ # We want a list by row with a list for each corr.
+ plist = list(zip(*plist))
+
+ return _make_cov_design_matrix(corrs, plist, func, 'jackknife', comm=comm)
+
+def _cov_jackknife(corrs, func, comm=None):
+ # Calculate the jackknife covariance for the given statistics
+
+ # The basic jackknife formula is:
+ # C = (1-1/npatch) Sum_i (v_i - v_mean) (v_i - v_mean)^T
+ # where v_i is the vector when excluding patch i, and v_mean is the mean of all {v_i}.
+ # v_i = Sum_jk!=i num_jk / Sum_jk!=i denom_jk
+
+ v,w = _design_jackknife(corrs, func, comm)
+ npatch = v.shape[0]
+ vmean = np.mean(v, axis=0)
+ v -= vmean
+ C = (1.-1./npatch) * v.conj().T.dot(v)
+ return C
+
+def _design_sample(corrs, func, comm=None):
+ npatch = _check_patch_nums(corrs, 'sample')
+ plist = [c._sample_pairs() for c in corrs]
+ # Swap order of plist. Right now it's a list for each corr of a list for each row.
+ # We want a list by row with a list for each corr.
+ plist = list(zip(*plist))
+
+ return _make_cov_design_matrix(corrs, plist, func, 'sample', comm=comm)
+
+def _cov_sample(corrs, func, comm=None):
+ # Calculate the sample covariance.
+
+ # This is kind of the converse of the jackknife. We take each patch and use any
+ # correlations of it with any other patch. The sample variance of these is the estimate
+ # of the overall variance.
+
+ # C = 1/(npatch-1) Sum_i w_i (v_i - v_mean) (v_i - v_mean)^T
+ # where v_i = Sum_j num_ij / Sum_j denom_ij
+ # and w_i is the fraction of the total weight in each patch
+
+ v,w = _design_sample(corrs, func, comm)
+ npatch = v.shape[0]
+
+ if np.any(w == 0):
+ raise RuntimeError("Cannot compute sample variance when some patches have no data.")
+
+ w /= np.sum(w) # Now w is the fractional weight for each patch
+
+ vmean = np.mean(v, axis=0)
+ v -= vmean
+ C = 1./(npatch-1) * (w * v.conj().T).dot(v)
+ return C
+
+def _design_marked(corrs, func, comm=None):
+ npatch = _check_patch_nums(corrs, 'marked_bootstrap')
+ nboot = np.max([c.num_bootstrap for c in corrs]) # use the maximum if they differ.
+
+ plist = []
+ for k in range(nboot):
+ # Select a random set of indices to use. (Will have repeats.)
+ index = corrs[0].rng.randint(npatch, size=npatch)
+ vpairs = [c._marked_pairs(index) for c in corrs]
+ plist.append(vpairs)
+
+ return _make_cov_design_matrix(corrs, plist, func, 'marked_bootstrap', comm=comm)
+
+def _cov_marked(corrs, func, comm=None):
+ # Calculate the marked-point bootstrap covariance
+
+ # This is based on the article A Valid and Fast Spatial Bootstrap for Correlation Functions
+ # by Ji Meng Loh, 2008, cf. https://ui.adsabs.harvard.edu/abs/2008ApJ...681..726L/abstract
+
+ # We do a bootstrap sampling of the patches. For each patch selected, we include
+ # all pairs that have the sampled patch in the first position. In the Loh prescription,
+ # the sums of pairs with a given choice of first patch would be the marks. Here, we
+ # don't quite do that, since the marks would involve a ratio, so the division is biased
+ # when somewhat noisy. Rather, we sum the numerators and denominators of the marks
+ # separately and divide the sums.
+
+ # From the bootstrap totals, v_i, the estimated covariance matrix is
+
+ # C = 1/(nboot) Sum_i (v_i - v_mean) (v_i - v_mean)^T
+
+ v,w = _design_marked(corrs, func, comm)
+ nboot = v.shape[0]
+ vmean = np.mean(v, axis=0)
+ v -= vmean
+ C = 1./(nboot-1) * v.conj().T.dot(v)
+ return C
+
+def _design_bootstrap(corrs, func, comm=None):
+ npatch = _check_patch_nums(corrs, 'bootstrap')
+ nboot = np.max([c.num_bootstrap for c in corrs]) # use the maximum if they differ.
+
+ plist = []
+ for k in range(nboot):
+ index = corrs[0].rng.randint(npatch, size=npatch)
+ vpairs = [c._bootstrap_pairs(index) for c in corrs]
+ plist.append(vpairs)
+
+ return _make_cov_design_matrix(corrs, plist, func, 'bootstrap', comm=comm)
+
+def _cov_bootstrap(corrs, func, comm=None):
+ # Calculate the 2-patch bootstrap covariance estimate.
+
+ # This is a different version of the bootstrap idea. It selects patches at random with
+ # replacement, and then generates the statistic using all the auto-correlations at their
+ # selected repetition plus all the cross terms, which aren't actually auto terms.
+ # It seems to do a slightly better job than the marked-point bootstrap above from the
+ # tests done in the test suite. But the difference is generally pretty small.
+
+ v,w = _design_bootstrap(corrs, func, comm)
+ nboot = v.shape[0]
+ vmean = np.mean(v, axis=0)
+ v -= vmean
+ C = 1./(nboot-1) * v.conj().T.dot(v)
+ return C
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: binnedcorr3
+"""
+
+import math
+import numpy as np
+import sys
+import coord
+
+from . import _lib
+from .config import merge_config, setup_logger, get
+from .util import parse_metric, metric_enum, coord_enum, set_omp_threads, lazy_property
+from .util import make_reader
+from .util import depr_pos_kwargs
+from .binnedcorr2 import estimate_multi_cov, build_multi_cov_design_matrix
+
+class Namespace(object):
+ pass
+
+[docs]class BinnedCorr3(object):
+ """This class stores the results of a 3-point correlation calculation, along with some
+ ancillary data.
+
+ This is a base class that is not intended to be constructed directly. But it has a few
+ helper functions that derived classes can use to help perform their calculations. See
+ the derived classes for more details:
+
+ - `NNNCorrelation` handles count-count-count correlation functions
+ - `KKKCorrelation` handles kappa-kappa-kappa correlation functions
+ - `GGGCorrelation` handles gamma-gamma-gamma correlation functions
+
+ Three-point correlations are a bit more complicated than two-point, since the data need
+ to be binned in triangles, not just the separation between two points. We characterize the
+ triangles according to the following three parameters based on the three side lenghts
+ of the triangle with d1 >= d2 >= d3.
+
+ .. math::
+ r &= d2 \\\\
+ u &= \\frac{d3}{d2} \\\\
+ v &= \\pm \\frac{(d1 - d2)}{d3} \\\\
+
+ The orientation of the triangle is specified by the sign of v.
+ Positive v triangles have the three sides d1,d2,d3 in counter-clockwise orientation.
+ Negative v triangles have the three sides d1,d2,d3 in clockwise orientation.
+
+ .. note::
+ We always bin the same way for positive and negative v values, and the binning
+ specification for v should just be for the positive values. E.g. if you specify
+ min_v=0.2, max_v=0.6, then TreeCorr will also accumulate triangles with
+ -0.6 < v < -0.2 in addition to those with 0.2 < v < 0.6.
+
+ The constructor for all derived classes take a config dict as the first argument,
+ since this is often how we keep track of parameters, but if you don't want to
+ use one or if you want to change some parameters from what are in a config dict,
+ then you can use normal kwargs, which take precedence over anything in the config dict.
+
+ There are three implemented definitions for the ``metric``, which defines how to calculate
+ the distance between two points, for three-point corretions:
+
+ - 'Euclidean' = straight line Euclidean distance between two points. For spherical
+ coordinates (ra,dec without r), this is the chord distance between points on the
+ unit sphere.
+ - 'Arc' = the true great circle distance for spherical coordinates.
+ - 'Periodic' = Like Euclidean, but with periodic boundaries.
+
+ .. note::
+
+ The triangles for three-point correlations can become ambiguous if d1 > period/2,
+ which means the maximum d2 (max_sep) should be less than period/4.
+ This is not enforced.
+
+ So far, there is only one allowed value for the ``bin_type`` for three-point correlations.
+
+ - 'LogRUV' - The bin steps will be uniform in log(r) from log(min_sep) .. log(max_sep).
+ The u and v values are binned linearly from min_u .. max_u and min_v .. max_v.
+
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in the below kwargs if
+ desired. This dict is allowed to have addition entries in addition
+ to those listed below, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+
+ nbins (int): How many bins to use. (Exactly three of nbins, bin_size, min_sep,
+ max_sep are required. If nbins is not given or set to None, it will be
+ calculated from the values of the other three, rounding up to the next
+ highest integer. In this case, bin_size will be readjusted to account
+ for this rounding up.)
+ bin_size (float): The width of the bins in log(separation). (Exactly three of nbins,
+ bin_size, min_sep, max_sep are required. If bin_size is not given or
+ set to None, it will be calculated from the values of the other three.)
+ min_sep (float): The minimum separation in units of sep_units, if relevant. (Exactly
+ three of nbins, bin_size, min_sep, max_sep are required. If min_sep is
+ not given or set to None, it will be calculated from the values of the
+ other three.)
+ max_sep (float): The maximum separation in units of sep_units, if relevant. (Exactly
+ three of nbins, bin_size, min_sep, max_sep are required. If max_sep is
+ not given or set to None, it will be calculated from the values of the
+ other three.)
+
+ sep_units (str): The units to use for the separation values, given as a string. This
+ includes both min_sep and max_sep above, as well as the units of the
+ output distance values. Valid options are arcsec, arcmin, degrees,
+ hours, radians. (default: radians if angular units make sense, but for
+ 3-d or flat 2-d positions, the default will just match the units of
+ x,y[,z] coordinates)
+ bin_slop (float): How much slop to allow in the placement of triangles in the bins.
+ If bin_slop = 1, then the bin into which a particular pair is placed
+ may be incorrect by at most 1.0 bin widths. (default: None, which
+ means to use a bin_slop that gives a maximum error of 10% on any bin,
+ which has been found to yield good results for most application.
+
+ nubins (int): Analogous to nbins for the u values. (The default is to calculate from
+ ubin_size = binsize, min_u = 0, max_u = 1, but this can be overridden
+ by specifying up to 3 of these four parametes.)
+ ubin_size (float): Analogous to bin_size for the u values. (default: bin_size)
+ min_u (float): Analogous to min_sep for the u values. (default: 0)
+ max_u (float): Analogous to max_sep for the u values. (default: 1)
+
+ nvbins (int): Analogous to nbins for the positive v values. (The default is to
+ calculate from vbin_size = binsize, min_v = 0, max_v = 1, but this can
+ be overridden by specifying up to 3 of these four parametes.)
+ vbin_size (float): Analogous to bin_size for the v values. (default: bin_size)
+ min_v (float): Analogous to min_sep for the positive v values. (default: 0)
+ max_v (float): Analogous to max_sep for the positive v values. (default: 1)
+
+ brute (bool): Whether to use the "brute force" algorithm. (default: False) Options
+ are:
+
+ - False (the default): Stop at non-leaf cells whenever the error in
+ the separation is compatible with the given bin_slop.
+ - True: Go to the leaves for both catalogs.
+ - 1: Always go to the leaves for cat1, but stop at non-leaf cells of
+ cat2 when the error is compatible with the given bin_slop.
+ - 2: Always go to the leaves for cat2, but stop at non-leaf cells of
+ cat1 when the error is compatible with the given bin_slop.
+
+ verbose (int): If no logger is provided, this will optionally specify a logging level
+ to use:
+
+ - 0 means no logging output
+ - 1 means to output warnings only (default)
+ - 2 means to output various progress information
+ - 3 means to output extensive debugging information
+
+ log_file (str): If no logger is provided, this will specify a file to write the logging
+ output. (default: None; i.e. output to standard output)
+ output_dots (bool): Whether to output progress dots during the calcualtion of the
+ correlation function. (default: False unless verbose is given and >= 2,
+ in which case True)
+
+ split_method (str): How to split the cells in the tree when building the tree structure.
+ Options are:
+
+ - mean = Use the arithmetic mean of the coordinate being split.
+ (default)
+ - median = Use the median of the coordinate being split.
+ - middle = Use the middle of the range; i.e. the average of the minimum
+ and maximum value.
+ - random: Use a random point somewhere in the middle two quartiles of
+ the range.
+
+ min_top (int): The minimum number of top layers to use when setting up the field.
+ (default: :math:`\\max(3, \\log_2(N_{\\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the field.
+ The top-level cells are where each calculation job starts. There will
+ typically be of order :math:`2^{\\rm max\\_top}` top-level cells.
+ (default: 10)
+ precision (int): The precision to use for the output values. This specifies how many
+ digits to write. (default: 4)
+
+ metric (str): Which metric to use for distance measurements. Options are listed
+ above. (default: 'Euclidean')
+ bin_type (str): What type of binning should be used. Only one option currently.
+ (default: 'LogRUV')
+ period (float): For the 'Periodic' metric, the period to use in all directions.
+ (default: None)
+ xperiod (float): For the 'Periodic' metric, the period to use in the x direction.
+ (default: period)
+ yperiod (float): For the 'Periodic' metric, the period to use in the y direction.
+ (default: period)
+ zperiod (float): For the 'Periodic' metric, the period to use in the z direction.
+ (default: period)
+
+ var_method (str): Which method to use for estimating the variance. Options are:
+ 'shot', 'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'.
+ (default: 'shot')
+ num_bootstrap (int): How many bootstrap samples to use for the 'bootstrap' and
+ 'marked_bootstrap' var_methods. (default: 500)
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for bootstrap
+ random number generation. (default: None)
+
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given in
+ the constructor in the config dict.)
+
+ .. note::
+
+ This won't work if the system's C compiler cannot use OpenMP
+ (e.g. clang prior to version 3.7.)
+ """
+ _valid_params = {
+ 'nbins' : (int, False, None, None,
+ 'The number of output bins to use for sep dimension.'),
+ 'bin_size' : (float, False, None, None,
+ 'The size of the output bins in log(sep).'),
+ 'min_sep' : (float, False, None, None,
+ 'The minimum separation to include in the output.'),
+ 'max_sep' : (float, False, None, None,
+ 'The maximum separation to include in the output.'),
+ 'sep_units' : (str, False, None, coord.AngleUnit.valid_names,
+ 'The units to use for min_sep and max_sep. Also the units of the output '
+ 'distances'),
+ 'bin_slop' : (float, False, None, None,
+ 'The fraction of a bin width by which it is ok to let the triangles miss the '
+ 'correct bin.',
+ 'The default is to use 1 if bin_size <= 0.1, or 0.1/bin_size if bin_size > 0.1.'),
+ 'nubins' : (int, False, None, None,
+ 'The number of output bins to use for u dimension.'),
+ 'ubin_size' : (float, False, None, None,
+ 'The size of the output bins in u.'),
+ 'min_u' : (float, False, None, None,
+ 'The minimum u to include in the output.'),
+ 'max_u' : (float, False, None, None,
+ 'The maximum u to include in the output.'),
+ 'nvbins' : (int, False, None, None,
+ 'The number of output bins to use for positive v values.'),
+ 'vbin_size' : (float, False, None, None,
+ 'The size of the output bins in v.'),
+ 'min_v' : (float, False, None, None,
+ 'The minimum |v| to include in the output.'),
+ 'max_v' : (float, False, None, None,
+ 'The maximum |v| to include in the output.'),
+ 'brute' : (bool, False, False, [False, True],
+ 'Whether to use brute-force algorithm'),
+ 'verbose' : (int, False, 1, [0, 1, 2, 3],
+ 'How verbose the code should be during processing. ',
+ '0 = Errors Only, 1 = Warnings, 2 = Progress, 3 = Debugging'),
+ 'log_file' : (str, False, None, None,
+ 'If desired, an output file for the logging output.',
+ 'The default is to write the output to stdout.'),
+ 'output_dots' : (bool, False, None, None,
+ 'Whether to output dots to the stdout during the C++-level computation.',
+ 'The default is True if verbose >= 2 and there is no log_file. Else False.'),
+ 'split_method' : (str, False, 'mean', ['mean', 'median', 'middle', 'random'],
+ 'Which method to use for splitting cells.'),
+ 'min_top' : (int, False, None, None,
+ 'The minimum number of top layers to use when setting up the field.'),
+ 'max_top' : (int, False, 10, None,
+ 'The maximum number of top layers to use when setting up the field.'),
+ 'precision' : (int, False, 4, None,
+ 'The number of digits after the decimal in the output.'),
+ 'metric': (str, False, 'Euclidean', ['Euclidean', 'Arc', 'Periodic'],
+ 'Which metric to use for the distance measurements'),
+ 'bin_type': (str, False, 'LogRUV', ['LogRUV'],
+ 'Which type of binning should be used'),
+ 'period': (float, False, None, None,
+ 'The period to use for all directions for the Periodic metric'),
+ 'xperiod': (float, False, None, None,
+ 'The period to use for the x direction for the Periodic metric'),
+ 'yperiod': (float, False, None, None,
+ 'The period to use for the y direction for the Periodic metric'),
+ 'zperiod': (float, False, None, None,
+ 'The period to use for the z direction for the Periodic metric'),
+
+ 'var_method': (str, False, 'shot',
+ ['shot', 'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'],
+ 'The method to use for estimating the variance'),
+ 'num_bootstrap': (int, False, 500, None,
+ 'How many bootstrap samples to use for the var_method=bootstrap and '
+ 'marked_bootstrap'),
+ 'num_threads' : (int, False, None, None,
+ 'How many threads should be used. num_threads <= 0 means auto based on num cores.'),
+ }
+
+ @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, rng=None, **kwargs):
+ self._corr = None # Do this first to make sure we always have it for __del__
+ self.config = merge_config(config,kwargs,BinnedCorr3._valid_params)
+ if logger is None:
+ self.logger = setup_logger(get(self.config,'verbose',int,1),
+ self.config.get('log_file',None))
+ else:
+ self.logger = logger
+
+ # We'll make a bunch of attributes here, which we put into a namespace called _ro.
+ # These are the core attributes that won't ever be changed after construction.
+ # This is an efficiency optimization (both memory and flops), since it will allow
+ # copy() to just copy a pointer to the _ro namespace without having to copy each
+ # individual attribute separately.
+ # The access of these attributes are all via read-only properties.
+ self._ro = Namespace()
+
+ if 'output_dots' in self.config:
+ self._ro.output_dots = get(self.config,'output_dots',bool)
+ else:
+ self._ro.output_dots = get(self.config,'verbose',int,1) >= 2
+
+ self._ro.bin_type = self.config.get('bin_type', None)
+ self._ro._bintype = _lib.Log
+
+ self._ro.sep_units = self.config.get('sep_units','')
+ self._ro._sep_units = get(self.config,'sep_units',str,'radians')
+ self._ro._log_sep_units = math.log(self._sep_units)
+ if self.config.get('nbins', None) is None:
+ if self.config.get('max_sep', None) is None:
+ raise TypeError("Missing required parameter max_sep")
+ if self.config.get('min_sep', None) is None:
+ raise TypeError("Missing required parameter min_sep")
+ if self.config.get('bin_size', None) is None:
+ raise TypeError("Missing required parameter bin_size")
+ self._ro.min_sep = float(self.config['min_sep'])
+ self._ro.max_sep = float(self.config['max_sep'])
+ if self.min_sep >= self.max_sep:
+ raise ValueError("max_sep must be larger than min_sep")
+ bin_size = float(self.config['bin_size'])
+ self._ro.nbins = int(math.ceil(math.log(self.max_sep/self.min_sep)/bin_size))
+ # Update self.bin_size given this value of nbins
+ self._ro.bin_size = math.log(self.max_sep/self.min_sep)/self.nbins
+ # Note in this case, bin_size is saved as the nominal bin_size from the config
+ # file, and self.bin_size is the one for the radial bins. We'll use the nominal
+ # bin_size as the default bin_size for u and v below.
+ elif self.config.get('bin_size', None) is None:
+ if self.config.get('max_sep', None) is None:
+ raise TypeError("Missing required parameter max_sep")
+ if self.config.get('min_sep', None) is None:
+ raise TypeError("Missing required parameter min_sep")
+ self._ro.min_sep = float(self.config['min_sep'])
+ self._ro.max_sep = float(self.config['max_sep'])
+ if self.min_sep >= self.max_sep:
+ raise ValueError("max_sep must be larger than min_sep")
+ self._ro.nbins = int(self.config['nbins'])
+ bin_size = self._ro.bin_size = math.log(self.max_sep/self.min_sep)/self.nbins
+ elif self.config.get('max_sep', None) is None:
+ if self.config.get('min_sep', None) is None:
+ raise TypeError("Missing required parameter min_sep")
+ self._ro.min_sep = float(self.config['min_sep'])
+ self._ro.nbins = int(self.config['nbins'])
+ bin_size = self._ro.bin_size = float(self.config['bin_size'])
+ self._ro.max_sep = math.exp(self.nbins*bin_size)*self.min_sep
+ else:
+ if self.config.get('min_sep', None) is not None:
+ raise TypeError("Only 3 of min_sep, max_sep, bin_size, nbins are allowed.")
+ self._ro.max_sep = float(self.config['max_sep'])
+ self._ro.nbins = int(self.config['nbins'])
+ bin_size = self._ro.bin_size = float(self.config['bin_size'])
+ self._ro.min_sep = self.max_sep*math.exp(-self.nbins*bin_size)
+ if self.sep_units == '':
+ self.logger.info("r: nbins = %d, min,max sep = %g..%g, bin_size = %g",
+ self.nbins, self.min_sep, self.max_sep, self.bin_size)
+ else:
+ self.logger.info("r: nbins = %d, min,max sep = %g..%g %s, bin_size = %g",
+ self.nbins, self.min_sep, self.max_sep, self.sep_units,
+ self.bin_size)
+ # The underscore-prefixed names are in natural units (radians for angles)
+ self._ro._min_sep = self.min_sep * self._sep_units
+ self._ro._max_sep = self.max_sep * self._sep_units
+ self._ro._bin_size = self.bin_size # There is not Linear, but if I add it, need to apply
+ # units to _bin_size in that case as well.
+
+ self._ro.min_u = float(self.config.get('min_u', 0.))
+ self._ro.max_u = float(self.config.get('max_u', 1.))
+ if self.min_u >= self.max_u:
+ raise ValueError("max_u must be larger than min_u")
+ if self.min_u < 0. or self.max_u > 1.:
+ raise ValueError("Invalid range for u: %f - %f"%(self.min_u, self.max_u))
+ self._ro.ubin_size = float(self.config.get('ubin_size', bin_size))
+ if 'nubins' not in self.config:
+ self._ro.nubins = int(math.ceil((self.max_u-self.min_u-1.e-10)/self.ubin_size))
+ elif 'max_u' in self.config and 'min_u' in self.config and 'ubin_size' in self.config:
+ raise TypeError("Only 3 of min_u, max_u, ubin_size, nubins are allowed.")
+ else:
+ self._ro.nubins = self.config['nubins']
+ # Allow min or max u to be implicit from nubins and ubin_size
+ if 'ubin_size' in self.config:
+ if 'min_u' not in self.config:
+ self._ro.min_u = max(self.max_u - self.nubins * self.ubin_size, 0.)
+ if 'max_u' not in self.config:
+ self._ro.max_u = min(self.min_u + self.nubins * self.ubin_size, 1.)
+ # Adjust ubin_size given the other values
+ self._ro.ubin_size = (self.max_u-self.min_u)/self.nubins
+ self.logger.info("u: nbins = %d, min,max = %g..%g, bin_size = %g",
+ self.nubins,self.min_u,self.max_u,self.ubin_size)
+
+ self._ro.min_v = float(self.config.get('min_v', 0.))
+ self._ro.max_v = float(self.config.get('max_v', 1.))
+ if self.min_v >= self.max_v:
+ raise ValueError("max_v must be larger than min_v")
+ if self.min_v < 0 or self.max_v > 1.:
+ raise ValueError("Invalid range for |v|: %f - %f"%(self.min_v, self.max_v))
+ self._ro.vbin_size = float(self.config.get('vbin_size', bin_size))
+ if 'nvbins' not in self.config:
+ self._ro.nvbins = int(math.ceil((self.max_v-self.min_v-1.e-10)/self.vbin_size))
+ elif 'max_v' in self.config and 'min_v' in self.config and 'vbin_size' in self.config:
+ raise TypeError("Only 3 of min_v, max_v, vbin_size, nvbins are allowed.")
+ else:
+ self._ro.nvbins = self.config['nvbins']
+ # Allow min or max v to be implicit from nvbins and vbin_size
+ if 'vbin_size' in self.config:
+ if 'max_v' not in self.config:
+ self._ro.max_v = min(self.min_v + self.nvbins * self.vbin_size, 1.)
+ else: # min_v not in config
+ self._ro.min_v = max(self.max_v - self.nvbins * self.vbin_size, -1.)
+ # Adjust vbin_size given the other values
+ self._ro.vbin_size = (self.max_v-self.min_v)/self.nvbins
+ self.logger.info("v: nbins = %d, min,max = %g..%g, bin_size = %g",
+ self.nvbins,self.min_v,self.max_v,self.vbin_size)
+
+ self._ro.split_method = self.config.get('split_method','mean')
+ self.logger.debug("Using split_method = %s",self.split_method)
+
+ self._ro.min_top = get(self.config,'min_top',int,None)
+ self._ro.max_top = get(self.config,'max_top',int,10)
+
+ self._ro.bin_slop = get(self.config,'bin_slop',float,-1.0)
+ if self.bin_slop < 0.0:
+ if self.bin_size <= 0.1:
+ self._ro.bin_slop = 1.0
+ self._ro.b = self.bin_size
+ else:
+ self._ro.bin_slop = 0.1/self.bin_size # The stored bin_slop corresponds to lnr bins.
+ self._ro.b = 0.1
+ if self.ubin_size <= 0.1:
+ self._ro.bu = self.ubin_size
+ else:
+ self._ro.bu = 0.1
+ if self.vbin_size <= 0.1:
+ self._ro.bv = self.vbin_size
+ else:
+ self._ro.bv = 0.1
+ else:
+ self._ro.b = self.bin_size * self.bin_slop
+ self._ro.bu = self.ubin_size * self.bin_slop
+ self._ro.bv = self.vbin_size * self.bin_slop
+
+ if self.b > 0.100001: # Add some numerical slop
+ self.logger.warning(
+ "Using bin_slop = %g, bin_size = %g\n"%(self.bin_slop,self.bin_size)+
+ "The b parameter is bin_slop * bin_size = %g"%(self.b)+
+ " bu = %g, bv = %g\n"%(self.bu,self.bv)+
+ "It is generally recommended to use b <= 0.1 for most applications.\n"+
+ "Larger values of this b parameter may result in significant inaccuracies.")
+ else:
+ self.logger.debug("Using bin_slop = %g, b = %g, bu = %g, bv = %g",
+ self.bin_slop,self.b,self.bu,self.bv)
+
+ # This makes nbins evenly spaced entries in log(r) starting with 0 with step bin_size
+ self._ro.logr1d = np.linspace(start=0, stop=self.nbins*self.bin_size,
+ num=self.nbins, endpoint=False)
+ # Offset by the position of the center of the first bin.
+ self._ro.logr1d += math.log(self.min_sep) + 0.5*self.bin_size
+
+ self._ro.u1d = np.linspace(start=0, stop=self.nubins*self.ubin_size,
+ num=self.nubins, endpoint=False)
+ self._ro.u1d += self.min_u + 0.5*self.ubin_size
+
+ self._ro.v1d = np.linspace(start=0, stop=self.nvbins*self.vbin_size,
+ num=self.nvbins, endpoint=False)
+ self._ro.v1d += self.min_v + 0.5*self.vbin_size
+ self._ro.v1d = np.concatenate([-self.v1d[::-1],self.v1d])
+
+ self._ro.logr = np.tile(self.logr1d[:, np.newaxis, np.newaxis],
+ (1, self.nubins, 2*self.nvbins))
+ self._ro.u = np.tile(self.u1d[np.newaxis, :, np.newaxis],
+ (self.nbins, 1, 2*self.nvbins))
+ self._ro.v = np.tile(self.v1d[np.newaxis, np.newaxis, :],
+ (self.nbins, self.nubins, 1))
+ self._ro.rnom = np.exp(self.logr)
+ self._ro.rnom1d = np.exp(self.logr1d)
+ self._ro.brute = get(self.config,'brute',bool,False)
+ if self.brute:
+ self.logger.info("Doing brute force calculation.",)
+ self.coords = None
+ self.metric = None
+ period = get(self.config,'period',float,0)
+ self._ro.xperiod = get(self.config,'xperiod',float,period)
+ self._ro.yperiod = get(self.config,'yperiod',float,period)
+ self._ro.zperiod = get(self.config,'zperiod',float,period)
+ self._ro._nbins = len(self._ro.logr.ravel())
+
+ self._ro.var_method = get(self.config,'var_method',str,'shot')
+ self._ro.num_bootstrap = get(self.config,'num_bootstrap',int,500)
+ self.results = {} # for jackknife, etc. store the results of each pair of patches.
+ self.npatch1 = self.npatch2 = self.npatch3 = 1
+ self._rng = rng
+
+ @property
+ def rng(self):
+ if self._rng is None:
+ self._rng = np.random.RandomState()
+ return self._rng
+
+ # Properties for all the read-only attributes ("ro" stands for "read-only")
+ @property
+ def output_dots(self): return self._ro.output_dots
+ @property
+ def bin_type(self): return self._ro.bin_type
+ @property
+ def sep_units(self): return self._ro.sep_units
+ @property
+ def _sep_units(self): return self._ro._sep_units
+ @property
+ def _log_sep_units(self): return self._ro._log_sep_units
+ @property
+ def min_sep(self): return self._ro.min_sep
+ @property
+ def max_sep(self): return self._ro.max_sep
+ @property
+ def min_u(self): return self._ro.min_u
+ @property
+ def max_u(self): return self._ro.max_u
+ @property
+ def min_v(self): return self._ro.min_v
+ @property
+ def max_v(self): return self._ro.max_v
+ @property
+ def bin_size(self): return self._ro.bin_size
+ @property
+ def ubin_size(self): return self._ro.ubin_size
+ @property
+ def vbin_size(self): return self._ro.vbin_size
+ @property
+ def nbins(self): return self._ro.nbins
+ @property
+ def nubins(self): return self._ro.nubins
+ @property
+ def nvbins(self): return self._ro.nvbins
+ @property
+ def logr1d(self): return self._ro.logr1d
+ @property
+ def u1d(self): return self._ro.u1d
+ @property
+ def v1d(self): return self._ro.v1d
+ @property
+ def logr(self): return self._ro.logr
+ @property
+ def u(self): return self._ro.u
+ @property
+ def v(self): return self._ro.v
+ @property
+ def rnom(self): return self._ro.rnom
+ @property
+ def rnom1d(self): return self._ro.rnom1d
+ @property
+ def _bintype(self): return self._ro._bintype
+ @property
+ def _nbins(self): return self._ro._nbins
+ @property
+ def _min_sep(self): return self._ro._min_sep
+ @property
+ def _max_sep(self): return self._ro._max_sep
+ @property
+ def _bin_size(self): return self._ro._bin_size
+ @property
+ def split_method(self): return self._ro.split_method
+ @property
+ def min_top(self): return self._ro.min_top
+ @property
+ def max_top(self): return self._ro.max_top
+ @property
+ def bin_slop(self): return self._ro.bin_slop
+ @property
+ def b(self): return self._ro.b
+ @property
+ def bu(self): return self._ro.bu
+ @property
+ def bv(self): return self._ro.bv
+ @property
+ def brute(self): return self._ro.brute
+ @property
+ def xperiod(self): return self._ro.xperiod
+ @property
+ def yperiod(self): return self._ro.yperiod
+ @property
+ def zperiod(self): return self._ro.zperiod
+ @property
+ def var_method(self): return self._ro.var_method
+ @property
+ def num_bootstrap(self): return self._ro.num_bootstrap
+ @property
+ def _d1(self): return self._ro._d1
+ @property
+ def _d2(self): return self._ro._d2
+ @property
+ def _d3(self): return self._ro._d3
+
+ def __getstate__(self):
+ d = self.__dict__.copy()
+ d.pop('_corr',None)
+ d.pop('_ok',None) # Remake this as needed.
+ d.pop('logger',None) # Oh well. This is just lost in the copy. Can't be pickled.
+ return d
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ self._corr = None
+ self.logger = setup_logger(get(self.config,'verbose',int,1),
+ self.config.get('log_file',None))
+
+[docs] def clear(self):
+ """Clear all data vectors, the results dict, and any related values.
+ """
+ self._clear()
+ self.results = {}
+ self.npatch1 = self.npatch2 = self.npatch3 = 1
+ self.__dict__.pop('_ok',None)
+
+ @property
+ def nonzero(self):
+ """Return if there are any values accumulated yet. (i.e. ntri > 0)
+ """
+ return np.any(self.ntri)
+
+ def _add_tot(self, i, j, k, c1, c2, c3):
+ # No op for all but NNCorrelation, which needs to add the tot value
+ pass
+
+ def _trivially_zero(self, c1, c2, c3, metric):
+ # For now, ignore the metric. Just be conservative about how much space we need.
+ x1,y1,z1,s1 = c1._get_center_size()
+ x2,y2,z2,s2 = c2._get_center_size()
+ x3,y3,z3,s3 = c3._get_center_size()
+ d3 = ((x1-x2)**2 + (y1-y2)**2 + (z1-z2)**2)**0.5
+ d1 = ((x2-x3)**2 + (y2-y3)**2 + (z2-z3)**2)**0.5
+ d2 = ((x3-x1)**2 + (y3-y1)**2 + (z3-z1)**2)**0.5
+ d3, d2, d1 = sorted([d1,d2,d3])
+ return (d2 > s1 + s2 + s3 + 2*self._max_sep) # The 2* is where we are being conservative.
+
+ def _process_all_auto(self, cat1, metric, num_threads, comm=None, low_mem=False):
+
+ def is_my_job(my_indices, i, j, k, n):
+ # Helper function to figure out if a given (i,j,k) job should be done on the
+ # current process.
+
+ # Always my job if not using MPI.
+ if my_indices is None:
+ return True
+
+ # Now the tricky part. If using MPI, we need to divide up the jobs smartly.
+ # The first point is to divvy up the auto jobs evenly. This is where most of the
+ # work is done, so we want those to be spreads as evenly as possibly across procs.
+ # Therefore, if all indices are mine, then do the job.
+ # This reduces the number of catalogs this machine needs to load up.
+ n1 = np.sum([i in my_indices, j in my_indices, k in my_indices])
+ if n1 == 3:
+ self.logger.info("Rank %d: Job (%d,%d,%d) is mine.",rank,i,j,k)
+ return True
+
+ # If none of the indices are mine, then it's not my job.
+ if n1 == 0:
+ return False
+
+ # When only one or two of the indices are mine, then we follow the same kind of
+ # procedure as we did in 2pt. There, we decided based on the parity of i.
+ # Here that turns into i mod 3.
+ if ( (i % 3 == 0 and i in my_indices) or
+ (i % 3 == 1 and j in my_indices) or
+ (i % 3 == 2 and k in my_indices) ):
+ self.logger.info("Rank %d: Job (%d,%d,%d) is mine.",rank,i,j,k)
+ return True
+ else:
+ return False
+
+ if len(cat1) == 1 and cat1[0].npatch == 1:
+ self.process_auto(cat1[0], metric=metric, num_threads=num_threads)
+
+ else:
+ # When patch processing, keep track of the pair-wise results.
+ if self.npatch1 == 1:
+ self.npatch1 = cat1[0].npatch if cat1[0].npatch != 1 else len(cat1)
+ self.npatch2 = self.npatch3 = self.npatch1
+ n = self.npatch1
+
+ # Setup for deciding when this is my job.
+ if comm:
+ size = comm.Get_size()
+ rank = comm.Get_rank()
+ my_indices = np.arange(n * rank // size, n * (rank+1) // size)
+ self.logger.info("Rank %d: My indices are %s",rank,my_indices)
+ else:
+ my_indices = None
+
+ temp = self.copy()
+ for ii,c1 in enumerate(cat1):
+ i = c1.patch if c1.patch is not None else ii
+ if is_my_job(my_indices, i, i, i, n):
+ temp.clear()
+ self.logger.info('Process patch %d auto',i)
+ temp.process_auto(c1, metric=metric, num_threads=num_threads)
+ if (i,i,i) in self.results and self.results[(i,i,i)].nonzero:
+ self.results[(i,i,i)] += temp
+ else:
+ self.results[(i,i,i)] = temp.copy()
+ self += temp
+
+ for jj,c2 in list(enumerate(cat1))[::-1]:
+ j = c2.patch if c2.patch is not None else jj
+ if i < j:
+ if is_my_job(my_indices, i, j, j, n):
+ temp.clear()
+ # One point in c1, 2 in c2.
+ if not self._trivially_zero(c1,c2,c2,metric):
+ self.logger.info('Process patches %d,%d cross12',i,j)
+ temp.process_cross12(c1, c2, metric=metric, num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d pair, which are too far apart ' +
+ 'for this set of separations',i,j)
+ if temp.nonzero:
+ if (i,j,j) in self.results and self.results[(i,j,j)].nonzero:
+ self.results[(i,j,j)] += temp
+ else:
+ self.results[(i,j,j)] = temp.copy()
+ self += temp
+ else:
+ # NNNCorrelation needs to add the tot value
+ self._add_tot(i, j, j, c1, c2, c2)
+
+ temp.clear()
+ # One point in c2, 2 in c1.
+ if not self._trivially_zero(c1,c1,c2,metric):
+ self.logger.info('Process patches %d,%d cross12',j,i)
+ temp.process_cross12(c2, c1, metric=metric, num_threads=num_threads)
+ if temp.nonzero:
+ if (i,i,j) in self.results and self.results[(i,i,j)].nonzero:
+ self.results[(i,i,j)] += temp
+ else:
+ self.results[(i,i,j)] = temp.copy()
+ self += temp
+ else:
+ # NNNCorrelation needs to add the tot value
+ self._add_tot(i, i, j, c2, c1, c1)
+
+ # One point in each of c1, c2, c3
+ for kk,c3 in enumerate(cat1):
+ k = c3.patch if c3.patch is not None else kk
+ if j < k and is_my_job(my_indices, i, j, k, n):
+ temp.clear()
+
+ if not self._trivially_zero(c1,c2,c3,metric):
+ self.logger.info('Process patches %d,%d,%d cross',i,j,k)
+ temp.process_cross(c1, c2, c3, metric=metric,
+ num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d,%d, which are too far apart ' +
+ 'for this set of separations',i,j,k)
+ if temp.nonzero:
+ if (i,j,k) in self.results and self.results[(i,j,k)].nonzero:
+ self.results[(i,j,k)] += temp
+ else:
+ self.results[(i,j,k)] = temp.copy()
+ self += temp
+ else:
+ # NNNCorrelation needs to add the tot value
+ self._add_tot(i, j, k, c1, c2, c3)
+ if low_mem:
+ c3.unload()
+
+ if low_mem and jj != ii+1:
+ # Don't unload i+1, since that's the next one we'll need.
+ c2.unload()
+ if low_mem:
+ c1.unload()
+ if comm is not None:
+ rank = comm.Get_rank()
+ size = comm.Get_size()
+ self.logger.info("Rank %d: Completed jobs %s",rank,list(self.results.keys()))
+ # Send all the results back to rank 0 process.
+ if rank > 0:
+ comm.send(self, dest=0)
+ else:
+ for p in range(1,size):
+ temp = comm.recv(source=p)
+ self += temp
+ self.results.update(temp.results)
+
+ def _process_all_cross12(self, cat1, cat2, metric, num_threads, comm=None, low_mem=False):
+
+ def is_my_job(my_indices, i, j, k, n1, n2):
+ # Helper function to figure out if a given (i,j,k) job should be done on the
+ # current process.
+
+ # Always my job if not using MPI.
+ if my_indices is None:
+ return True
+
+ # If n1 is n, then this can be simple. Just split according to i.
+ n = max(n1,n2)
+ if n1 == n:
+ if i in my_indices:
+ self.logger.info("Rank %d: Job (%d,%d,%d) is mine.",rank,i,j,k)
+ return True
+ else:
+ return False
+
+ # If not, then this looks like the decision for 2pt auto using j,k.
+ if j in my_indices and k in my_indices:
+ self.logger.info("Rank %d: Job (%d,%d,%d) is mine.",rank,i,j,k)
+ return True
+
+ if j not in my_indices and k not in my_indices:
+ return False
+
+ if k-j < n//2:
+ ret = j % 2 == (0 if j in my_indices else 1)
+ else:
+ ret = k % 2 == (0 if k in my_indices else 1)
+ if ret:
+ self.logger.info("Rank %d: Job (%d,%d,%d) is mine.",rank,i,j,k)
+ return ret
+
+ if len(cat1) == 1 and len(cat2) == 1 and cat1[0].npatch == 1 and cat2[0].npatch == 1:
+ self.process_cross12(cat1[0], cat2[0], metric=metric, num_threads=num_threads)
+ else:
+ # When patch processing, keep track of the pair-wise results.
+ if self.npatch1 == 1:
+ self.npatch1 = cat1[0].npatch if cat1[0].npatch != 1 else len(cat1)
+ if self.npatch2 == 1:
+ self.npatch2 = cat2[0].npatch if cat2[0].npatch != 1 else len(cat2)
+ self.npatch3 = self.npatch2
+ if self.npatch1 != self.npatch2 and self.npatch1 != 1 and self.npatch2 != 1:
+ raise RuntimeError("Cross correlation requires both catalogs use the same patches.")
+
+ # Setup for deciding when this is my job.
+ n1 = self.npatch1
+ n2 = self.npatch2
+ if comm:
+ size = comm.Get_size()
+ rank = comm.Get_rank()
+ n = max(n1,n2)
+ my_indices = np.arange(n * rank // size, n * (rank+1) // size)
+ self.logger.info("Rank %d: My indices are %s",rank,my_indices)
+ else:
+ my_indices = None
+
+ temp = self.copy()
+ for ii,c1 in enumerate(cat1):
+ i = c1.patch if c1.patch is not None else ii
+ for jj,c2 in enumerate(cat2):
+ j = c2.patch if c2.patch is not None else jj
+ if is_my_job(my_indices, i, i, j, n1, n2):
+ temp.clear()
+ # One point in c1, 2 in c2.
+ if not self._trivially_zero(c1,c2,c2,metric):
+ self.logger.info('Process patches %d,%d cross12',i,j)
+ temp.process_cross12(c1, c2, metric=metric, num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d pair, which are too far apart ' +
+ 'for this set of separations',i,j)
+ if temp.nonzero or i==j or n1==1 or n2==1:
+ if (i,j,j) in self.results and self.results[(i,j,j)].nonzero:
+ self.results[(i,j,j)] += temp
+ else:
+ self.results[(i,j,j)] = temp.copy()
+ self += temp
+ else:
+ # NNNCorrelation needs to add the tot value
+ self._add_tot(i, j, j, c1, c2, c2)
+
+ # One point in each of c1, c2, c3
+ for kk,c3 in list(enumerate(cat2))[::-1]:
+ k = c3.patch if c3.patch is not None else kk
+ if j < k and is_my_job(my_indices, i, j, k, n1, n2):
+ temp.clear()
+
+ if not self._trivially_zero(c1,c2,c3,metric):
+ self.logger.info('Process patches %d,%d,%d cross',i,j,k)
+ temp.process_cross(c1, c2, c3, metric=metric,
+ num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d,%d, which are too far apart ' +
+ 'for this set of separations',i,j,k)
+ if temp.nonzero:
+ if (i,j,k) in self.results and self.results[(i,j,k)].nonzero:
+ self.results[(i,j,k)] += temp
+ else:
+ self.results[(i,j,k)] = temp.copy()
+ self += temp
+ else:
+ # NNNCorrelation needs to add the tot value
+ self._add_tot(i, j, k, c1, c2, c3)
+ if low_mem:
+ c3.unload()
+
+ if low_mem and jj != ii+1:
+ # Don't unload i+1, since that's the next one we'll need.
+ c2.unload()
+ if low_mem:
+ c1.unload()
+ if comm is not None:
+ rank = comm.Get_rank()
+ size = comm.Get_size()
+ self.logger.info("Rank %d: Completed jobs %s",rank,list(self.results.keys()))
+ # Send all the results back to rank 0 process.
+ if rank > 0:
+ comm.send(self, dest=0)
+ else:
+ for p in range(1,size):
+ temp = comm.recv(source=p)
+ self += temp
+ self.results.update(temp.results)
+
+ def _process_all_cross(self, cat1, cat2, cat3, metric, num_threads, comm=None, low_mem=False):
+
+ def is_my_job(my_indices, i, j, k, n1, n2, n3):
+ # Helper function to figure out if a given (i,j,k) job should be done on the
+ # current process.
+
+ # Always my job if not using MPI.
+ if my_indices is None:
+ return True
+
+ # Just split up according to one of the catalogs.
+ n = max(n1,n2,n3)
+ if n1 == n:
+ m = i
+ elif n2 == n:
+ m = j
+ else:
+ m = k
+ if m in my_indices:
+ self.logger.info("Rank %d: Job (%d,%d,%d) is mine.",rank,i,j,k)
+ return True
+ else:
+ return False
+
+ if (len(cat1) == 1 and len(cat2) == 1 and len(cat3) == 1 and
+ cat1[0].npatch == 1 and cat2[0].npatch == 1 and cat3[0].npatch == 1):
+ self.process_cross(cat1[0], cat2[0], cat3[0], metric=metric, num_threads=num_threads)
+ else:
+ # When patch processing, keep track of the pair-wise results.
+ if self.npatch1 == 1:
+ self.npatch1 = cat1[0].npatch if cat1[0].npatch != 1 else len(cat1)
+ if self.npatch2 == 1:
+ self.npatch2 = cat2[0].npatch if cat2[0].npatch != 1 else len(cat2)
+ if self.npatch3 == 1:
+ self.npatch3 = cat3[0].npatch if cat3[0].npatch != 1 else len(cat3)
+ if self.npatch1 != self.npatch2 and self.npatch1 != 1 and self.npatch2 != 1:
+ raise RuntimeError("Cross correlation requires all catalogs use the same patches.")
+ if self.npatch1 != self.npatch3 and self.npatch1 != 1 and self.npatch3 != 1:
+ raise RuntimeError("Cross correlation requires all catalogs use the same patches.")
+
+ # Setup for deciding when this is my job.
+ n1 = self.npatch1
+ n2 = self.npatch2
+ n3 = self.npatch3
+ if comm:
+ size = comm.Get_size()
+ rank = comm.Get_rank()
+ n = max(n1,n2,n3)
+ my_indices = np.arange(n * rank // size, n * (rank+1) // size)
+ self.logger.info("Rank %d: My indices are %s",rank,my_indices)
+ else:
+ my_indices = None
+
+ temp = self.copy()
+ for ii,c1 in enumerate(cat1):
+ i = c1.patch if c1.patch is not None else ii
+ for jj,c2 in enumerate(cat2):
+ j = c2.patch if c2.patch is not None else jj
+ for kk,c3 in enumerate(cat3):
+ k = c3.patch if c3.patch is not None else kk
+ if is_my_job(my_indices, i, j, k, n1, n2, n3):
+ temp.clear()
+ if not self._trivially_zero(c1,c2,c3,metric):
+ self.logger.info('Process patches %d,%d,%d cross',i,j,k)
+ temp.process_cross(c1, c2, c3, metric=metric,
+ num_threads=num_threads)
+ else:
+ self.logger.info('Skipping %d,%d,%d, which are too far apart ' +
+ 'for this set of separations',i,j,k)
+ if (temp.nonzero or (i==j==k)
+ or (i==j and n3==1) or (i==k and n2==1) or (j==k and n1==1)
+ or (n1==n2==1) or (n1==n3==1) or (n2==n3==1)):
+ if (i,j,k) in self.results and self.results[(i,j,k)].nonzero:
+ self.results[(i,j,k)] += temp
+ else:
+ self.results[(i,j,k)] = temp.copy()
+ self += temp
+ else:
+ # NNNCorrelation needs to add the tot value
+ self._add_tot(i, j, k, c1, c2, c3)
+ if low_mem:
+ c3.unload()
+ if low_mem and jj != ii+1:
+ # Don't unload i+1, since that's the next one we'll need.
+ c2.unload()
+ if low_mem:
+ c1.unload()
+ if comm is not None:
+ rank = comm.Get_rank()
+ size = comm.Get_size()
+ self.logger.info("Rank %d: Completed jobs %s",rank,list(self.results.keys()))
+ # Send all the results back to rank 0 process.
+ if rank > 0:
+ comm.send(self, dest=0)
+ else:
+ for p in range(1,size):
+ temp = comm.recv(source=p)
+ self += temp
+ self.results.update(temp.results)
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ Usually, this is just self.zeta. But if the metric is TwoD, this becomes
+ self.zeta.ravel().
+
+ And for `GGGCorrelation`, it is the concatenation of the four different correlations
+ [gam0.ravel(), gam1.ravel(), gam2.ravel(), gam3.ravel()].
+ """
+ return self.zeta.ravel()
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ This is the weight array corresponding to `getStat`. Usually just self.weight, but
+ raveled for TwoD and duplicated for GGGCorrelation to match what `getStat` does in
+ those cases.
+ """
+ return self.weight.ravel()
+
+[docs] @depr_pos_kwargs
+ def estimate_cov(self, method, *, func=None, comm=None):
+ """Estimate the covariance matrix based on the data
+
+ This function will calculate an estimate of the covariance matrix according to the
+ given method.
+
+ Options for ``method`` include:
+
+ - 'shot' = The variance based on "shot noise" only. This includes the Poisson
+ counts of points for N statistics, shape noise for G statistics, and the observed
+ scatter in the values for K statistics. In this case, the returned covariance
+ matrix will be diagonal, since there is no way to estimate the off-diagonal terms.
+ - 'jackknife' = A jackknife estimate of the covariance matrix based on the scatter
+ in the measurement when excluding one patch at a time.
+ - 'sample' = An estimate based on the sample covariance of a set of samples,
+ taken as the patches of the input catalog.
+ - 'bootstrap' = A bootstrap covariance estimate. It selects patches at random with
+ replacement and then generates the statistic using all the auto-correlations at
+ their selected repetition plus all the cross terms that aren't actually auto terms.
+ - 'marked_bootstrap' = An estimate based on a marked-point bootstrap resampling of the
+ patches. Similar to bootstrap, but only samples the patches of the first catalog and
+ uses all patches from the second catalog that correspond to each patch selection of
+ the first catalog. cf. https://ui.adsabs.harvard.edu/abs/2008ApJ...681..726L/
+
+ Both 'bootstrap' and 'marked_bootstrap' use the num_bootstrap parameter, which can be set on
+ construction.
+
+ .. note::
+
+ For most classes, there is only a single statistic, ``zeta``, so this calculates a
+ covariance matrix for that vector. `GGGCorrelation` has four: ``gam0``, ``gam1``,
+ ``gam2``, and ``gam3``, so in this case the full data vector is ``gam0`` followed by
+ ``gam1``, then ``gam2``, then ``gam3``, and this calculates the covariance matrix for
+ that full vector including both statistics. The helper function `getStat` returns the
+ relevant statistic in all cases.
+
+ In all cases, the relevant processing needs to already have been completed and finalized.
+ And for all methods other than 'shot', the processing should have involved an appropriate
+ number of patches -- preferably more patches than the length of the vector for your
+ statistic, although this is not checked.
+
+ The default data vector to use for the covariance matrix is given by the method
+ `getStat`. As noted above, this is usually just self.zeta. However, there is an option
+ to compute the covariance of some other function of the correlation object by providing
+ an arbitrary function, ``func``, which should act on the current correlation object
+ and return the data vector of interest.
+
+ For instance, for an `GGGCorrelation`, you might want to compute the covariance of just
+ gam0 and ignore the others. In this case you could use
+
+ >>> func = lambda ggg: ggg.gam0
+
+ The return value from this func should be a single numpy array. (This is not directly
+ checked, but you'll probably get some kind of exception if it doesn't behave as expected.)
+
+ .. note::
+
+ The optional ``func`` parameter is not valid in conjunction with ``method='shot'``.
+ It only works for the methods that are based on patch combinations.
+
+ This function can be parallelized by passing the comm argument as an mpi4py communicator
+ to parallelize using that. For MPI, all processes should have the same inputs.
+ If method == "shot" then parallelization has no effect.
+
+ Parameters:
+ method (str): Which method to use to estimate the covariance matrix.
+ func (function): A unary function that acts on the current correlation object and
+ returns the desired data vector. [default: None, which is
+ equivalent to ``lambda corr: corr.getStat()``.
+ comm (mpi comm) If not None, run under MPI
+
+ Returns:
+ A numpy array with the estimated covariance matrix.
+ """
+ if func is not None:
+ # Need to convert it to a function of the first item in the list.
+ all_func = lambda corrs: func(corrs[0])
+ else:
+ all_func = None
+ return estimate_multi_cov([self], method, func=all_func, comm=comm)
+
+[docs] def build_cov_design_matrix(self, method, *, func=None, comm=None):
+ """Build the design matrix that is used for estimating the covariance matrix.
+
+ The design matrix for patch-based covariance estimates is a matrix where each row
+ corresponds to a different estimate of the data vector, :math:`\\zeta_i` (or
+ :math:`f(\\zeta_i)` if using the optional ``func`` parameter).
+
+ The different of rows in the matrix for each valid ``method`` are:
+
+ - 'shot': This method is not valid here.
+ - 'jackknife': The data vector when excluding a single patch.
+ - 'sample': The data vector using only a single patch for the first catalog.
+ - 'bootstrap': The data vector for a random resampling of the patches keeping the
+ sample total number, but allowing some to repeat. Cross terms from repeated patches
+ are excluded (since they are really auto terms).
+ - 'marked_bootstrap': The data vector for a random resampling of patches in the first
+ catalog, using all patches for the second catalog. Based on the algorithm in
+ Loh(2008).
+
+ See `estimate_cov` for more details.
+
+ The return value includes both the design matrix and a vector of weights (the total weight
+ array in the computed correlation functions). The weights are used for the sample method
+ when estimating the covariance matrix. The other methods ignore them, but they are provided
+ here in case they are useful.
+
+ Parameters:
+ method (str): Which method to use to estimate the covariance matrix.
+ func (function): A unary function that takes the list ``corrs`` and returns the
+ desired full data vector. [default: None, which is equivalent to
+ ``lambda corrs: np.concatenate([c.getStat() for c in corrs])``]
+ comm (mpi comm) If not None, run under MPI
+
+ Returns:
+ A, w: numpy arrays with the design matrix and weights respectively.
+ """
+ if func is not None:
+ # Need to convert it to a function of the first item in the list.
+ all_func = lambda corrs: func(corrs[0])
+ else:
+ all_func = None
+ return build_multi_cov_design_matrix([self], method=method, func=all_func, comm=comm)
+
+ def _set_num_threads(self, num_threads):
+ if num_threads is None:
+ num_threads = self.config.get('num_threads',None)
+ if num_threads is None:
+ self.logger.debug('Set num_threads automatically from ncpu')
+ else:
+ self.logger.debug('Set num_threads = %d',num_threads)
+ set_omp_threads(num_threads, self.logger)
+
+ def _set_metric(self, metric, coords1, coords2=None, coords3=None):
+ if metric is None:
+ metric = get(self.config,'metric',str,'Euclidean')
+ coords, metric = parse_metric(metric, coords1, coords2, coords3)
+ if self.coords is not None or self.metric is not None:
+ if coords != self.coords:
+ self.logger.warning("Detected a change in catalog coordinate systems. "+
+ "This probably doesn't make sense!")
+ if metric != self.metric:
+ self.logger.warning("Detected a change in metric. "+
+ "This probably doesn't make sense!")
+ if metric == 'Periodic':
+ if self.xperiod == 0 or self.yperiod == 0 or (coords=='3d' and self.zperiod == 0):
+ raise ValueError("Periodic metric requires setting the period to use.")
+ else:
+ if self.xperiod != 0 or self.yperiod != 0 or self.zperiod != 0:
+ raise ValueError("period options are not valid for %s metric."%metric)
+ self.coords = coords
+ self.metric = metric
+ self._coords = coord_enum(coords)
+ self._metric = metric_enum(metric)
+
+ def _apply_units(self, mask):
+ if self.coords == 'spherical' and self.metric == 'Euclidean':
+ # Then our distances are all angles. Convert from the chord distance to a real angle.
+ # L = 2 sin(theta/2)
+ self.meand1[mask] = 2. * np.arcsin(self.meand1[mask]/2.)
+ self.meanlogd1[mask] = np.log(2.*np.arcsin(np.exp(self.meanlogd1[mask])/2.))
+ self.meand2[mask] = 2. * np.arcsin(self.meand2[mask]/2.)
+ self.meanlogd2[mask] = np.log(2.*np.arcsin(np.exp(self.meanlogd2[mask])/2.))
+ self.meand3[mask] = 2. * np.arcsin(self.meand3[mask]/2.)
+ self.meanlogd3[mask] = np.log(2.*np.arcsin(np.exp(self.meanlogd3[mask])/2.))
+
+ self.meand1[mask] /= self._sep_units
+ self.meanlogd1[mask] -= self._log_sep_units
+ self.meand2[mask] /= self._sep_units
+ self.meanlogd2[mask] -= self._log_sep_units
+ self.meand3[mask] /= self._sep_units
+ self.meanlogd3[mask] -= self._log_sep_units
+
+ def _get_minmax_size(self):
+ if self.metric == 'Euclidean':
+ # The minimum separation we care about is that of the smallest size, which is
+ # min_sep * min_u. Do the same calculation as for 2pt to get to min_size.
+ b1 = min(self.b, self.bu, self.bv)
+ min_size = self._min_sep * self.min_u * b1 / (2.+3.*b1)
+
+ # This time, the maximum size is d1 * b. d1 can be as high as 2*max_sep.
+ b2 = max(self.b, self.bu, self.bv)
+ max_size = 2. * self._max_sep * b2
+ return min_size, max_size
+ else:
+ return 0., 0.
+
+ # The three-point versions of the covariance helpers.
+ # Note: the word "pairs" in many of these was appropriate for 2pt, but in the 3pt case
+ # these actually refer to triples (i,j,k).
+
+ def _get_npatch(self):
+ return max(self.npatch1, self.npatch2, self.npatch3)
+
+ def _calculate_xi_from_pairs(self, pairs):
+ # Compute the xi data vector for the given list of pairs.
+ # pairs is input as a list of (i,j) values.
+
+ # This is the normal calculation. It needs to be overridden when there are randoms.
+ self._sum([self.results[ij] for ij in pairs])
+ self._finalize()
+
+ def _jackknife_pairs(self):
+ if self.npatch3 == 1:
+ if self.npatch2 == 1:
+ # k=m=0
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j!=i]
+ for i in range(self.npatch1) ]
+ elif self.npatch1 == 1:
+ # j=m=0
+ return [ [(j,k,m) for j,k,m in self.results.keys() if k!=i]
+ for i in range(self.npatch2) ]
+ else:
+ # m=0
+ assert self.npatch1 == self.npatch2
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j!=i and k!=i]
+ for i in range(self.npatch1) ]
+ elif self.npatch2 == 1:
+ if self.npatch1 == 1:
+ # j=k=0
+ return [ [(j,k,m) for j,k,m in self.results.keys() if m!=i]
+ for i in range(self.npatch3) ]
+ else:
+ # k=0
+ assert self.npatch1 == self.npatch3
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j!=i and m!=i]
+ for i in range(self.npatch1) ]
+ elif self.npatch1 == 1:
+ # j=0
+ assert self.npatch2 == self.npatch3
+ return [ [(j,k,m) for j,k,m in self.results.keys() if k!=i and m!=i]
+ for i in range(self.npatch2) ]
+ else:
+ assert self.npatch1 == self.npatch2 == self.npatch3
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j!=i and k!=i and m!=i]
+ for i in range(self.npatch1) ]
+
+ def _sample_pairs(self):
+ if self.npatch3 == 1:
+ if self.npatch2 == 1:
+ # k=m=0
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j==i]
+ for i in range(self.npatch1) ]
+ elif self.npatch1 == 1:
+ # j=m=0
+ return [ [(j,k,m) for j,k,m in self.results.keys() if k==i]
+ for i in range(self.npatch2) ]
+ else:
+ # m=0
+ assert self.npatch1 == self.npatch2
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j==i]
+ for i in range(self.npatch1) ]
+ elif self.npatch2 == 1:
+ if self.npatch1 == 1:
+ # j=k=0
+ return [ [(j,k,m) for j,k,m in self.results.keys() if m==i]
+ for i in range(self.npatch3) ]
+ else:
+ # k=0
+ assert self.npatch1 == self.npatch3
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j==i]
+ for i in range(self.npatch1) ]
+ elif self.npatch1 == 1:
+ # j=0
+ assert self.npatch2 == self.npatch3
+ return [ [(j,k,m) for j,k,m in self.results.keys() if k==i]
+ for i in range(self.npatch2) ]
+ else:
+ assert self.npatch1 == self.npatch2 == self.npatch3
+ return [ [(j,k,m) for j,k,m in self.results.keys() if j==i]
+ for i in range(self.npatch1) ]
+
+ @lazy_property
+ def _ok(self):
+ ok = np.zeros((self.npatch1, self.npatch2, self.npatch3), dtype=bool)
+ for (i,j,k) in self.results:
+ ok[i,j,k] = True
+ return ok
+
+ def _marked_pairs(self, indx):
+ if self.npatch3 == 1:
+ if self.npatch2 == 1:
+ return [ (i,0,0) for i in indx if self._ok[i,0,0] ]
+ elif self.npatch1 == 1:
+ return [ (0,i,0) for i in indx if self._ok[0,i,0] ]
+ else:
+ assert self.npatch1 == self.npatch2
+ # Select all pairs where first point is in indx (repeating i as appropriate)
+ return [ (i,j,0) for i in indx for j in range(self.npatch2) if self._ok[i,j,0] ]
+ elif self.npatch2 == 1:
+ if self.npatch1 == 1:
+ return [ (0,0,i) for i in indx if self._ok[0,0,i] ]
+ else:
+ assert self.npatch1 == self.npatch3
+ # Select all pairs where first point is in indx (repeating i as appropriate)
+ return [ (i,0,j) for i in indx for j in range(self.npatch3) if self._ok[i,0,j] ]
+ elif self.npatch1 == 1:
+ assert self.npatch2 == self.npatch3
+ # Select all pairs where first point is in indx (repeating i as appropriate)
+ return [ (0,i,j) for i in indx for j in range(self.npatch3) if self._ok[0,i,j] ]
+ else:
+ assert self.npatch1 == self.npatch2 == self.npatch3
+ # Select all pairs where first point is in indx (repeating i as appropriate)
+ return [ (i,j,k) for i in indx for j in range(self.npatch2)
+ for k in range(self.npatch3) if self._ok[i,j,k] ]
+
+ def _bootstrap_pairs(self, indx):
+ if self.npatch3 == 1:
+ if self.npatch2 == 1:
+ return [ (i,0,0) for i in indx if self._ok[i,0,0] ]
+ elif self.npatch1 == 1:
+ return [ (0,i,0) for i in indx if self._ok[0,i,0] ]
+ else:
+ assert self.npatch1 == self.npatch2
+ return ([ (i,i,0) for i in indx if self._ok[i,i,0] ] +
+ [ (i,j,0) for i in indx for j in indx if self._ok[i,j,0] and i!=j ])
+ elif self.npatch2 == 1:
+ if self.npatch1 == 1:
+ return [ (0,0,i) for i in indx if self._ok[0,0,i] ]
+ else:
+ assert self.npatch1 == self.npatch3
+ return ([ (i,0,i) for i in indx if self._ok[i,0,i] ] +
+ [ (i,0,j) for i in indx for j in indx if self._ok[i,0,j] and i!=j ])
+ elif self.npatch1 == 1:
+ assert self.npatch2 == self.npatch3
+ return ([ (0,i,i) for i in indx if self._ok[0,i,i] ] +
+ [ (0,i,j) for i in indx for j in indx if self._ok[0,i,j] and i!=j ])
+ else:
+ # Like for 2pt we want to avoid getting extra copies of what are actually
+ # auto-correlations coming from two indices equalling each other in (i,j,k).
+ # This time, get each (i,i,i) once.
+ # Then get (i,i,j), (i,j,i), and (j,i,i) once per each (i,j) pair with i!=j
+ # repeated as often as they show up in the double for loop.
+ # Finally get all triples (i,j,k) where they are all different repeated as often
+ # as they show up in the triple for loop.
+ assert self.npatch1 == self.npatch2 == self.npatch3
+ return ([ (i,i,i) for i in indx if self._ok[i,i,i] ] +
+ [ (i,i,j) for i in indx for j in indx if self._ok[i,i,j] and i!=j ] +
+ [ (i,j,i) for i in indx for j in indx if self._ok[i,j,i] and i!=j ] +
+ [ (j,i,i) for i in indx for j in indx if self._ok[j,i,i] and i!=j ] +
+ [ (i,j,k) for i in indx for j in indx if i!=j
+ for k in indx if self._ok[i,j,k] and (i!=k and j!=k) ])
+
+ def _write(self, writer, name, write_patch_results, zero_tot=False):
+ # These helper properties define what to write for each class.
+ col_names = self._write_col_names
+ data = self._write_data
+ params = self._write_params
+ params['num_rows'] = len(self.rnom.ravel())
+
+ if write_patch_results:
+ # Note: Only include npatch1, npatch2 in serialization if we are also serializing
+ # results. Otherwise, the corr that is read in will behave oddly.
+ params['npatch1'] = self.npatch1
+ params['npatch2'] = self.npatch2
+ params['npatch3'] = self.npatch3
+ num_patch_tri = len(self.results)
+ if zero_tot:
+ i = 0
+ for key, corr in self.results.items():
+ if not corr._nonzero:
+ zp_name = name + '_zp_%d'%i
+ params[zp_name] = repr((key, corr.tot))
+ num_patch_tri -= 1
+ i += 1
+ params['num_zero_patch'] = i
+ params['num_patch_tri'] = num_patch_tri
+
+ writer.write(col_names, data, params=params, ext=name)
+ if write_patch_results:
+ writer.set_precision(16)
+ i = 0
+ for key, corr in self.results.items():
+ if zero_tot and not corr._nonzero: continue
+ col_names = corr._write_col_names
+ data = corr._write_data
+ params = corr._write_params
+ params['key'] = repr(key)
+ pp_name = name + '_pp_%d'%i
+ writer.write(col_names, data, params=params, ext=pp_name)
+ i += 1
+ assert i == num_patch_tri
+
+ def _read(self, reader, name=None):
+ name = 'main' if 'main' in reader and name is None else name
+ params = reader.read_params(ext=name)
+ num_rows = params.get('num_rows', None)
+ num_patch_tri = params.get('num_patch_tri', 0)
+ num_zero_patch = params.get('num_zero_patch', 0)
+ name = 'main' if num_patch_tri and name is None else name
+ data = reader.read_data(max_rows=num_rows, ext=name)
+
+ # This helper function defines how to set the attributes for each class
+ # based on what was read in.
+ self._read_from_data(data, params)
+
+ self.results = {}
+ for i in range(num_zero_patch):
+ zp_name = name + '_zp_%d'%i
+ key, tot = eval(params[zp_name])
+ self.results[key] = self._zero_copy(tot)
+ for i in range(num_patch_tri):
+ pp_name = name + '_pp_%d'%i
+ corr = self.copy()
+ params = reader.read_params(ext=pp_name)
+ data = reader.read_data(max_rows=num_rows, ext=pp_name)
+ corr._read_from_data(data, params)
+ key = eval(params['key'])
+ self.results[key] = corr
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: catalog
+"""
+
+import numpy as np
+import coord
+import weakref
+import copy
+import os
+
+from . import _lib
+from .reader import FitsReader, HdfReader, AsciiReader, PandasReader, ParquetReader
+from .config import merge_config, setup_logger, get, get_from_list
+from .util import parse_file_type, LRU_Cache, make_writer, make_reader, set_omp_threads
+from .util import double_ptr as dp
+from .util import long_ptr as lp
+from .util import depr_pos_kwargs
+from .field import NField, KField, GField, NSimpleField, KSimpleField, GSimpleField
+
+[docs]class Catalog(object):
+ """A set of input data (positions and other quantities) to be correlated.
+
+ A Catalog object keeps track of the relevant information for a number of objects to
+ be correlated. The objects each have some kind of position (for instance (x,y), (ra,dec),
+ (x,y,z), etc.), and possibly some extra information such as weights (w), shear values (g1,g2),
+ or kappa values (k).
+
+ The simplest way to build a Catalog is to simply pass in numpy arrays for each
+ piece of information you want included. For instance::
+
+ >>> cat = treecorr.Catalog(x=x, y=y, k=k, w=w)
+
+ Each of these input paramters should be a numpy array, where each corresponding element
+ is the value for that object. Of course, all the arrays should be the same size.
+
+ In some cases, there are additional required parameters. For instance, with RA and Dec
+ positions, you need to declare what units the given input values use::
+
+ >>> cat = treecorr.Catalog(ra=ra, dec=dec, g1=g1, g2=g2,
+ ... ra_units='hour', dec_units='deg')
+
+ For (ra,dec) positions, these units fields are required to specify the units of the angular
+ values. For (x,y) positions, the units are optional (and usually unnecessary).
+
+ You can also initialize a Catalog by reading in columns from a file. For instance::
+
+ >>> cat = treecorr.Catalog('data.fits', ra_col='ALPHA2000', dec_col='DELTA2000',
+ ... g1_col='E1', g2_col='E2', ra_units='deg', dec_units='deg')
+
+ This reads the given columns from the input file. The input file may be a FITS file,
+ an HDF5 file, a Parquet file, or an ASCII file. Normally the file type is determined
+ according to the file's extension (e.g. '.fits' here), but it can also be set explicitly
+ with ``file_type``.
+
+ For FITS, HDF5, and Parquet files, the column names should be strings as shown above.
+ For ASCII files, they may be strings if the input file has column names. But you may
+ also use integer values giving the index of which column to use. We use a 1-based convention
+ for these, so x_col=1 would mean to use the first column as the x value. (0 means don't
+ read that column.)
+
+ Finally, you may store all the various parameters in a configuration dict
+ and just pass the dict as an argument after the file name::
+
+ >>> config = { 'ra_col' : 'ALPHA2000',
+ ... 'dec_col' : 'DELTA2000',
+ ... 'g1_col' : 'E1',
+ ... 'g2_col' : 'E2',
+ ... 'ra_units' : 'deg',
+ ... 'dec_units' : 'deg' }
+ >>> cat = treecorr.Catalog(file_name, config)
+
+ This can be useful for encapsulating all the TreeCorr options in a single place in your
+ code, which might be used multiple times. Notably, this syntax ignores any dict keys
+ that are not relevant to the Catalog construction, so you can use the same config dict
+ for the Catalog and your correlation objects, which can be convenient.
+
+ See also `Configuration Parameters` for complete descriptions of all of the relevant
+ configuration parameters, particularly the first section `Parameters about the input file(s)`.
+
+ You may also override any configuration parameters or add additional parameters as kwargs
+ after the config dict. For instance, to flip the sign of the g1 values after reading
+ from the input file, you could write::
+
+ >>> cat1 = treecorr.Catalog(file_name, config, flip_g1=True)
+
+ After construction, a Catalog object will have the following attributes:
+
+ Attributes:
+
+ x: The x positions, if defined, as a numpy array (converted to radians if x_units
+ was given). (None otherwise)
+ y: The y positions, if defined, as a numpy array (converted to radians if y_units
+ was given). (None otherwise)
+ z: The z positions, if defined, as a numpy array. (None otherwise)
+ ra: The right ascension, if defined, as a numpy array (in radians). (None otherwise)
+ dec: The declination, if defined, as a numpy array (in radians). (None otherwise)
+ r: The distance, if defined, as a numpy array. (None otherwise)
+ w: The weights, as a numpy array. (All 1's if no weight column provided.)
+ wpos: The weights for position centroiding, as a numpy array, if given. (None otherwise,
+ which means that implicitly wpos = w.)
+ g1: The g1 component of the shear, if defined, as a numpy array. (None otherwise)
+ g2: The g2 component of the shear, if defined, as a numpy array. (None otherwise)
+ k: The convergence, kappa, if defined, as a numpy array. (None otherwise)
+ patch: The patch number of each object, if patches are being used. (None otherwise)
+ If the entire catalog is a single patch, then ``patch`` may be an int.
+ ntot: The total number of objects (including those with zero weight if
+ ``keep_zero_weight`` is set to True)
+ nobj: The number of objects with non-zero weight
+ sumw: The sum of the weights
+ varg: The shear variance (aka shape noise) (0 if g1,g2 are not defined)
+
+ .. note::
+
+ If there are weights, this is really :math:`\\sum(w^2 |g|^2)/\\sum(w)`,
+ which is more like :math:`\\langle w \\rangle \\mathrm{Var}(g)`.
+ It is only used for ``var_method='shot'``, where the noise estimate is this
+ value divided by the total weight per bin, so this is the right quantity
+ to use for that.
+
+ vark: The kappa variance (0 if k is not defined)
+
+ .. note::
+
+ If there are weights, this is really :math:`\\sum(w^2 \\kappa^2)/\\sum(w)`.
+ As for ``varg``, this is the right quantity to use for the ``'shot'``
+ noise estimate.
+
+ name: When constructed from a file, this will be the file_name. It is only used as
+ a reference name in logging output after construction, so if you construct it
+ from data vectors directly, it will be ``''``. You may assign to it if you want to
+ give this catalog a specific name.
+
+ coords: Which kind of coordinate system is defined for this catalog.
+ The possibilities for this attribute are:
+
+ - 'flat' = 2-dimensional flat coordinates. Set when x,y are given.
+ - 'spherical' = spherical coordinates. Set when ra,dec are given.
+ - '3d' = 3-dimensional coordinates. Set when x,y,z or ra,dec,r are given.
+
+ field: If any of the `get?Field <Catalog.getNField>` methods have been called to construct
+ a field from this catalog (either explicitly or implicitly via a `corr.process()
+ <NNCorrelation.process>` command, then this attribute will hold the most recent
+ field to have been constructed.
+
+ .. note::
+
+ It holds this field as a weakref, so if caching is turned off with
+ ``resize_cache(0)``, and the field has been garbage collected, then this
+ attribute will be None.
+
+ Parameters:
+ file_name (str): The name of the catalog file to be read in. (default: None, in which
+ case the columns need to be entered directly with ``x``, ``y``, etc.)
+
+ config (dict): A configuration dict which defines attributes about how to read the
+ file. Any optional kwargs may be given here in the config dict if
+ desired. Invalid keys in the config dict are ignored. (default: None)
+
+ Keyword Arguments:
+
+ num (int): Which number catalog are we reading. e.g. for NG correlations the
+ catalog for the N has num=0, the one for G has num=1. This is only
+ necessary if you are using a config dict where things like ``x_col``
+ have multiple values. (default: 0)
+ logger: If desired, a Logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+ is_rand (bool): If this is a random file, then setting is_rand to True will let them
+ skip k_col, g1_col, and g2_col if they were set for the main catalog.
+ (default: False)
+
+ x (array): The x values. (default: None; When providing values directly, either
+ x,y are required or ra,dec are required.)
+ y (array): The y values. (default: None; When providing values directly, either
+ x,y are required or ra,dec are required.)
+ z (array): The z values, if doing 3d positions. (default: None; invalid in
+ conjunction with ra, dec.)
+ ra (array): The RA values. (default: None; When providing values directly, either
+ x,y are required or ra,dec are required.)
+ dec (array): The Dec values. (default: None; When providing values directly, either
+ x,y are required or ra,dec are required.)
+ r (array): The r values (the distances of each source from Earth). (default: None;
+ invalid in conjunction with x, y.)
+ w (array): The weights to apply when computing the correlations. (default: None)
+ wpos (array): The weights to use for position centroiding. (default: None, which
+ means to use the value weights, w, to weight the positions as well.)
+ flag (array): An optional array of flags, indicating objects to skip. Rows with
+ flag != 0 (or technically flag & ~ok_flag != 0) will be given a weight
+ of 0. (default: None)
+ g1 (array): The g1 values to use for shear correlations. (g1,g2 may represent any
+ spinor field.) (default: None)
+ g2 (array): The g2 values to use for shear correlations. (g1,g2 may represent any
+ spinor field.) (default: None)
+ k (array): The kappa values to use for scalar correlations. (This may represent
+ any scalar field.) (default: None)
+ patch (array or int): Optionally, patch numbers to use for each object. (default: None)
+
+ .. note::
+
+ This may also be an int if the entire catalog represents a
+ single patch. If ``patch_centers`` is given this will select those
+ items from the full input that correspond to the given patch number.
+
+ patch_centers (array or str): Alternative to setting patch by hand or using kmeans, you
+ may instead give patch_centers either as a file name or an array
+ from which the patches will be determined. (default: None)
+
+ file_type (str): What kind of file is the input file. Valid options are 'ASCII', 'FITS'
+ 'HDF', or 'Parquet' (default: if the file_name extension starts with
+ .fit, then use 'FITS', or with .hdf, then use 'HDF', or with '.par',
+ then use 'Parquet', else 'ASCII')
+ delimiter (str): For ASCII files, what delimiter to use between values. (default: None,
+ which means any whitespace)
+ comment_marker (str): For ASCII files, what token indicates a comment line. (default: '#')
+ first_row (int): Which row to take as the first row to be used. (default: 1)
+ last_row (int): Which row to take as the last row to be used. (default: -1, which means
+ the last row in the file)
+ every_nth (int): Only use every nth row of the input catalog. (default: 1)
+
+ npatch (int): How many patches to split the catalog into (using kmeans if no other
+ patch information is provided) for the purpose of jackknife variance
+ or other options that involve running via patches. (default: 1)
+
+ .. note::
+
+ If the catalog has ra,dec,r positions, the patches will
+ be made using just ra,dec.
+
+ kmeans_init (str): If using kmeans to make patches, which init method to use.
+ cf. `Field.run_kmeans` (default: 'tree')
+ kmeans_alt (bool): If using kmeans to make patches, whether to use the alternate kmeans
+ algorithm. cf. `Field.run_kmeans` (default: False)
+
+ x_col (str or int): The column to use for the x values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.
+ When reading from a file, either x_col and y_col are required or ra_col
+ and dec_col are required.)
+ y_col (str or int): The column to use for the y values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.
+ When reading from a file, either x_col and y_col are required or ra_col
+ and dec_col are required.)
+ z_col (str or int): The column to use for the z values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column;
+ invalid in conjunction with ra_col, dec_col.)
+ ra_col (str or int): The column to use for the ra values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.
+ When reading from a file, either x_col and y_col are required or ra_col
+ and dec_col are required.)
+ dec_col (str or int): The column to use for the dec values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.
+ When reading from a file, either x_col and y_col are required or ra_col
+ and dec_col are required.)
+ r_col (str or int): The column to use for the r values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column;
+ invalid in conjunction with x_col, y_col.)
+
+ x_units (str): The units to use for the x values, given as a string. Valid options are
+ arcsec, arcmin, degrees, hours, radians. (default: radians, although
+ with (x,y) positions, you can often just ignore the units, and the
+ output separations will be in whatever units x and y are in.)
+ y_units (str): The units to use for the y values, given as a string. Valid options are
+ arcsec, arcmin, degrees, hours, radians. (default: radians, although
+ with (x,y) positions, you can often just ignore the units, and the
+ output separations will be in whatever units x and y are in.)
+ ra_units (str): The units to use for the ra values, given as a string. Valid options
+ are arcsec, arcmin, degrees, hours, radians. (required when using
+ ra_col or providing ra directly)
+ dec_units (str): The units to use for the dec values, given as a string. Valid options
+ are arcsec, arcmin, degrees, hours, radians. (required when using
+ dec_col or providing dec directly)
+
+ g1_col (str or int): The column to use for the g1 values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.)
+ g2_col (str or int): The column to use for the g2 values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.)
+ k_col (str or int): The column to use for the kappa values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.)
+ patch_col (str or int): The column to use for the patch numbers. An integer is only allowed
+ for ASCII files. (default: '0', which means not to read in this column.)
+ w_col (str or int): The column to use for the weight values. An integer is only allowed for
+ ASCII files. (default: '0', which means not to read in this column.)
+ wpos_col (str or int): The column to use for the position weight values. An integer is only
+ allowed for ASCII files. (default: '0', which means not to read in this
+ column, in which case wpos=w.)
+ flag_col (str or int): The column to use for the flag values. An integer is only allowed for
+ ASCII files. Any row with flag != 0 (or technically flag & ~ok_flag
+ != 0) will be given a weight of 0. (default: '0', which means not to
+ read in this column.)
+ ignore_flag (int): Which flags should be ignored. (default: all non-zero flags are ignored.
+ Equivalent to ignore_flag = ~0.)
+ ok_flag (int): Which flags should be considered ok. (default: 0. i.e. all non-zero
+ flags are ignored.)
+ allow_xyz (bool): Whether to allow x,y,z values in conjunction with ra,dec. Normally,
+ it is an error to have both kinds of positions, but if you know that
+ the x,y,z, values are consistent with the given ra,dec values, it
+ can save time to input them, rather than calculate them using trig
+ functions. (default: False)
+
+ flip_g1 (bool): Whether to flip the sign of the input g1 values. (default: False)
+ flip_g2 (bool): Whether to flip the sign of the input g2 values. (default: False)
+ keep_zero_weight (bool): Whether to keep objects with wpos=0 in the catalog (including
+ any objects that indirectly get wpos=0 due to NaN or flags), so they
+ would be included in ntot and also in npairs calculations that use
+ this Catalog, although of course not contribute to the accumulated
+ weight of pairs. (default: False)
+ save_patch_dir (str): If desired, when building patches from this Catalog, save them
+ as FITS files in the given directory for more efficient loading when
+ doing cross-patch correlations with the ``low_mem`` option.
+
+ ext (int/str): For FITS/HDF files, Which extension to read. (default: 1 for fits,
+ root for HDF)
+ x_ext (int/str): Which extension to use for the x values. (default: ext)
+ y_ext (int/str): Which extension to use for the y values. (default: ext)
+ z_ext (int/str): Which extension to use for the z values. (default: ext)
+ ra_ext (int/str): Which extension to use for the ra values. (default: ext)
+ dec_ext (int/str): Which extension to use for the dec values. (default: ext)
+ r_ext (int/str): Which extension to use for the r values. (default: ext)
+ g1_ext (int/str): Which extension to use for the g1 values. (default: ext)
+ g2_ext (int/str): Which extension to use for the g2 values. (default: ext)
+ k_ext (int/str): Which extension to use for the k values. (default: ext)
+ patch_ext (int/str): Which extension to use for the patch numbers. (default: ext)
+ w_ext (int/str): Which extension to use for the w values. (default: ext)
+ wpos_ext (int/str): Which extension to use for the wpos values. (default: ext)
+ flag_ext (int/str): Which extension to use for the flag values. (default: ext)
+
+ verbose (int): If no logger is provided, this will optionally specify a logging level
+ to use.
+
+ - 0 means no logging output
+ - 1 means to output warnings only (default)
+ - 2 means to output various progress information
+ - 3 means to output extensive debugging information
+
+ log_file (str): If no logger is provided, this will specify a file to write the logging
+ output. (default: None; i.e. output to standard output)
+
+ split_method (str): How to split the cells in the tree when building the tree structure.
+ Options are:
+
+ - mean: Use the arithmetic mean of the coordinate being split.
+ (default)
+ - median: Use the median of the coordinate being split.
+ - middle: Use the middle of the range; i.e. the average of the
+ minimum and maximum value.
+ - random: Use a random point somewhere in the middle two quartiles
+ of the range.
+
+ cat_precision (int): The precision to use when writing a Catalog to an ASCII file. This
+ should be an integer, which specifies how many digits to write.
+ (default: 16)
+
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for any random
+ number generation (e.g. kmeans patches). (default: None)
+
+ num_threads (int): How many OpenMP threads to use during the catalog load steps.
+ (default: use the number of cpu cores)
+
+ .. note::
+
+ This won't work if the system's C compiler cannot use OpenMP
+ (e.g. clang prior to version 3.7.)
+ """
+ # Dict describing the valid kwarg parameters, what types they are, and a description:
+ # Each value is a tuple with the following elements:
+ # type
+ # may_be_list
+ # default value
+ # list of valid values
+ # description
+ _valid_params = {
+ 'file_type' : (str, True, None, ['ASCII', 'FITS', 'HDF', 'Parquet'],
+ 'The file type of the input files. The default is to use the file name extension.'),
+ 'delimiter' : (str, True, None, None,
+ 'The delimiter between values in an ASCII catalog. The default is any whitespace.'),
+ 'comment_marker' : (str, True, '#', None,
+ 'The first (non-whitespace) character of comment lines in an input ASCII catalog.'),
+ 'first_row' : (int, True, 1, None,
+ 'The first row to use from the input catalog'),
+ 'last_row' : (int, True, -1, None,
+ 'The last row to use from the input catalog. The default is to use all of them.'),
+ 'every_nth' : (int, True, 1, None,
+ 'Only use every nth row of the input catalog. The default is to use all of them.'),
+ 'x_col' : (str, True, '0', None,
+ 'Which column to use for x. Should be an integer for ASCII catalogs.'),
+ 'y_col' : (str, True, '0', None,
+ 'Which column to use for y. Should be an integer for ASCII catalogs.'),
+ 'z_col' : (str, True, '0', None,
+ 'Which column to use for z. Should be an integer for ASCII catalogs.'),
+ 'ra_col' : (str, True, '0', None,
+ 'Which column to use for ra. Should be an integer for ASCII catalogs.'),
+ 'dec_col' : (str, True, '0', None,
+ 'Which column to use for dec. Should be an integer for ASCII catalogs.'),
+ 'r_col' : (str, True, '0', None,
+ 'Which column to use for r. Only valid with ra,dec. ',
+ 'Should be an integer for ASCII catalogs.'),
+ 'x_units' : (str, True, None, coord.AngleUnit.valid_names,
+ 'The units of x values.'),
+ 'y_units' : (str, True, None, coord.AngleUnit.valid_names,
+ 'The units of y values.'),
+ 'ra_units' : (str, True, None, coord.AngleUnit.valid_names,
+ 'The units of ra values. Required when using ra_col.'),
+ 'dec_units' : (str, True, None, coord.AngleUnit.valid_names,
+ 'The units of dec values. Required when using dec_col.'),
+ 'g1_col' : (str, True, '0', None,
+ 'Which column to use for g1. Should be an integer for ASCII catalogs.'),
+ 'g2_col' : (str, True, '0', None,
+ 'Which column to use for g2. Should be an integer for ASCII catalogs.'),
+ 'k_col' : (str, True, '0', None,
+ 'Which column to use for kappa. Should be an integer for ASCII catalogs. '),
+ 'patch_col' : (str, True, '0', None,
+ 'Which column to use for patch numbers. Should be an integer for ASCII catalogs. '),
+ 'w_col' : (str, True, '0', None,
+ 'Which column to use for weight. Should be an integer for ASCII catalogs.'),
+ 'wpos_col' : (str, True, '0', None,
+ 'Which column to use for position weight. Should be an integer for ASCII '
+ 'catalogs.'),
+ 'flag_col' : (str, True, '0', None,
+ 'Which column to use for flag. Should be an integer for ASCII catalogs.'),
+ 'ignore_flag': (int, True, None, None,
+ 'Ignore objects with flag & ignore_flag != 0 (bitwise &)'),
+ 'ok_flag': (int, True, 0, None,
+ 'Ignore objects with flag & ~ok_flag != 0 (bitwise &, ~)'),
+ 'allow_xyz': (bool, True, False, None,
+ 'Whether to allow x,y,z inputs in conjunction with ra,dec'),
+ 'ext': (str, True, None, None,
+ 'Which extension/group in a fits/hdf file to use. Default=1 (fits), root (hdf)'),
+ 'x_ext': (str, True, None, None,
+ 'Which extension to use for the x_col. default is the global ext value.'),
+ 'y_ext': (str, True, None, None,
+ 'Which extension to use for the y_col. default is the global ext value.'),
+ 'z_ext': (str, True, None, None,
+ 'Which extension to use for the z_col. default is the global ext value.'),
+ 'ra_ext': (str, True, None, None,
+ 'Which extension to use for the ra_col. default is the global ext value.'),
+ 'dec_ext': (str, True, None, None,
+ 'Which extension to use for the dec_col. default is the global ext value.'),
+ 'r_ext': (str, True, None, None,
+ 'Which extension to use for the r_col. default is the global ext value.'),
+ 'g1_ext': (str, True, None, None,
+ 'Which extension to use for the g1_col. default is the global ext value.'),
+ 'g2_ext': (str, True, None, None,
+ 'Which extension to use for the g2_col. default is the global ext value.'),
+ 'k_ext': (str, True, None, None,
+ 'Which extension to use for the k_col. default is the global ext value.'),
+ 'patch_ext': (str, True, None, None,
+ 'Which extension to use for the patch_col. default is the global ext value.'),
+ 'w_ext': (str, True, None, None,
+ 'Which extension to use for the w_col. default is the global ext value.'),
+ 'wpos_ext': (str, True, None, None,
+ 'Which extension to use for the wpos_col. default is the global ext value.'),
+ 'flag_ext': (str, True, None, None,
+ 'Which extension to use for the flag_col. default is the global ext value.'),
+ 'flip_g1' : (bool, True, False, None,
+ 'Whether to flip the sign of g1'),
+ 'flip_g2' : (bool, True, False, None,
+ 'Whether to flip the sign of g2'),
+
+ 'keep_zero_weight' : (bool, False, False, None,
+ 'Whether to keep objects with zero weight in the catalog'),
+ 'npatch' : (int, False, 1, None,
+ 'Number of patches to split the catalog into'),
+ 'kmeans_init' : (str, False, 'tree', ['tree','random','kmeans++'],
+ 'Which initialization method to use for kmeans when making patches'),
+ 'kmeans_alt' : (bool, False, False, None,
+ 'Whether to use the alternate kmeans algorithm when making patches'),
+ 'patch_centers' : (str, False, None, None,
+ 'File with patch centers to use to determine patches'),
+ 'save_patch_dir' : (str, False, None, None,
+ 'If desired, save the patches as FITS files in this directory.'),
+ 'verbose' : (int, False, 1, [0, 1, 2, 3],
+ 'How verbose the code should be during processing. ',
+ '0 = Errors Only, 1 = Warnings, 2 = Progress, 3 = Debugging'),
+ 'log_file' : (str, False, None, None,
+ 'If desired, an output file for the logging output.',
+ 'The default is to write the output to stdout.'),
+ 'split_method' : (str, False, 'mean', ['mean', 'median', 'middle', 'random'],
+ 'Which method to use for splitting cells.'),
+ 'cat_precision' : (int, False, 16, None,
+ 'The number of digits after the decimal in the output.'),
+ }
+ _aliases = {
+ 'hdu' : 'ext',
+ 'x_hdu' : 'x_ext',
+ 'y_hdu' : 'y_ext',
+ 'z_hdu' : 'z_ext',
+ 'ra_hdu' : 'ra_ext',
+ 'dec_hdu' : 'dec_ext',
+ 'r_hdu' : 'r_ext',
+ 'g1_hdu' : 'g1_ext',
+ 'g2_hdu' : 'g2_ext',
+ 'k_hdu' : 'k_ext',
+ 'w_hdu' : 'w_ext',
+ 'wpos_hdu' : 'wpos_ext',
+ 'flag_hdu' : 'flag_ext',
+ 'patch_hdu' : 'patch_ext',
+ }
+ _emitted_pandas_warning = False # Only emit the warning once. Set to True once we have.
+
+ @depr_pos_kwargs
+ def __init__(self, file_name=None, config=None, *, num=0, logger=None, is_rand=False,
+ x=None, y=None, z=None, ra=None, dec=None, r=None, w=None, wpos=None, flag=None,
+ g1=None, g2=None, k=None, patch=None, patch_centers=None, rng=None, **kwargs):
+
+ self.config = merge_config(config, kwargs, Catalog._valid_params, Catalog._aliases)
+ self.orig_config = config.copy() if config is not None else {}
+ if config and kwargs:
+ self.orig_config.update(kwargs)
+ self._num = num
+ self._is_rand = is_rand
+
+ if logger is not None:
+ self.logger = logger
+ else:
+ self.logger = setup_logger(get(self.config,'verbose',int,1),
+ self.config.get('log_file',None))
+
+ # Start with everything set to None. Overwrite as appropriate.
+ self._x = None
+ self._y = None
+ self._z = None
+ self._ra = None
+ self._dec = None
+ self._r = None
+ self._w = None
+ self._wpos = None
+ self._flag = None
+ self._g1 = None
+ self._g2 = None
+ self._k = None
+ self._patch = None
+ self._field = lambda : None
+
+ self._nontrivial_w = None
+ self._single_patch = None
+ self._nobj = None
+ self._sumw = None
+ self._sumw2 = None
+ self._varg = None
+ self._vark = None
+ self._patches = None
+ self._centers = None
+ self._rng = rng
+
+ first_row = get_from_list(self.config,'first_row',num,int,1)
+ if first_row < 1:
+ raise ValueError("first_row should be >= 1")
+ last_row = get_from_list(self.config,'last_row',num,int,-1)
+ if last_row > 0 and last_row < first_row:
+ raise ValueError("last_row should be >= first_row")
+ if last_row > 0:
+ self.end = last_row
+ else:
+ self.end = None
+ if first_row > 1:
+ self.start = first_row-1
+ else:
+ self.start = 0
+ self.every_nth = get_from_list(self.config,'every_nth',num,int,1)
+ if self.every_nth < 1:
+ raise ValueError("every_nth should be >= 1")
+
+ if 'npatch' in self.config and self.config['npatch'] != 1:
+ self._npatch = get(self.config,'npatch',int)
+ if self._npatch < 1:
+ raise ValueError("npatch must be >= 1")
+ elif self.config.get('patch_col',0) not in (0,'0'):
+ self._npatch = None # Mark that we need to finish loading to figure out npatch.
+ else:
+ self._npatch = 1 # We might yet change this, but it will be correct at end of init.
+
+ try:
+ self._single_patch = int(patch)
+ except TypeError:
+ pass
+ else:
+ patch = None
+
+ if patch_centers is None and 'patch_centers' in self.config:
+ # file name version may be in a config dict, rather than kwarg.
+ patch_centers = get(self.config,'patch_centers',str)
+
+ if patch_centers is not None:
+ if patch is not None or self.config.get('patch_col',0) not in (0,'0'):
+ raise ValueError("Cannot provide both patch and patch_centers")
+ if isinstance(patch_centers, np.ndarray):
+ self._centers = patch_centers
+ else:
+ self._centers = self.read_patch_centers(patch_centers)
+ if self._npatch not in [None, 1, self._centers.shape[0]]:
+ raise ValueError("npatch is incompatible with provided centers")
+ self._npatch = self._centers.shape[0]
+
+ self.save_patch_dir = self.config.get('save_patch_dir',None)
+ allow_xyz = self.config.get('allow_xyz', False)
+
+ # First style -- read from a file
+ if file_name is not None:
+ if any([v is not None for v in [x,y,z,ra,dec,r,g1,g2,k,patch,w,wpos,flag]]):
+ raise TypeError("Vectors may not be provided when file_name is provided.")
+ self.file_name = file_name
+ self.name = file_name
+ if self._single_patch is not None:
+ self.name += " patch " + str(self._single_patch)
+
+ # Figure out which file type the catalog is
+ file_type = get_from_list(self.config,'file_type',num)
+ file_type = parse_file_type(file_type, file_name, output=False, logger=self.logger)
+ if file_type == 'FITS':
+ self.reader = FitsReader(file_name)
+ self._check_file(file_name, self.reader, num, is_rand)
+ elif file_type == 'HDF':
+ self.reader = HdfReader(file_name)
+ self._check_file(file_name, self.reader, num, is_rand)
+ elif file_type == 'PARQUET':
+ self.reader = ParquetReader(file_name)
+ self._check_file(file_name, self.reader, num, is_rand)
+ else:
+ delimiter = self.config.get('delimiter',None)
+ comment_marker = self.config.get('comment_marker','#')
+ try:
+ self.reader = PandasReader(file_name, delimiter, comment_marker)
+ except ImportError:
+ self._pandas_warning()
+ self.reader = AsciiReader(file_name, delimiter, comment_marker)
+ self._check_file(file_name, self.reader, num, is_rand)
+
+ self.file_type = file_type
+
+ # Second style -- pass in the vectors directly
+ else:
+ self.file_type = None
+ if x is not None or y is not None:
+ if x is None or y is None:
+ raise TypeError("x and y must both be provided")
+ if (ra is not None or dec is not None) and not allow_xyz:
+ raise TypeError("ra and dec may not be provided with x,y")
+ if r is not None and not allow_xyz:
+ raise TypeError("r may not be provided with x,y")
+ if ra is not None or dec is not None:
+ if ra is None or dec is None:
+ raise TypeError("ra and dec must both be provided")
+ if g1 is not None or g2 is not None:
+ if g1 is None or g2 is None:
+ raise TypeError("g1 and g2 must both be provided")
+ self.file_name = None
+ self.name = ''
+ if self._single_patch is not None:
+ self.name = "patch " + str(self._single_patch)
+ self._x = self.makeArray(x,'x')
+ self._y = self.makeArray(y,'y')
+ self._z = self.makeArray(z,'z')
+ self._ra = self.makeArray(ra,'ra')
+ self._dec = self.makeArray(dec,'dec')
+ self._r = self.makeArray(r,'r')
+ self._w = self.makeArray(w,'w')
+ self._wpos = self.makeArray(wpos,'wpos')
+ self._flag = self.makeArray(flag,'flag',int)
+ self._g1 = self.makeArray(g1,'g1')
+ self._g2 = self.makeArray(g2,'g2')
+ self._k = self.makeArray(k,'k')
+ self._patch = self.makeArray(patch,'patch',int)
+ if self._patch is not None:
+ self._set_npatch()
+ if self._x is not None:
+ self._apply_xyz_units()
+ if self._ra is not None:
+ self._apply_radec_units()
+
+ # Check that all columns have the same length. (This is impossible in file input)
+ if self._x is not None:
+ ntot = len(self._x)
+ if len(self._y) != ntot:
+ raise ValueError("x and y have different numbers of elements")
+ else:
+ ntot = len(self._ra)
+ if len(self._dec) != ntot:
+ raise ValueError("ra and dec have different numbers of elements")
+ if self._z is not None and len(self._z) != ntot:
+ raise ValueError("z has the wrong numbers of elements")
+ if self._r is not None and len(self._r) != ntot:
+ raise ValueError("r has the wrong numbers of elements")
+ if self._w is not None and len(self._w) != ntot:
+ raise ValueError("w has the wrong numbers of elements")
+ if self._wpos is not None and len(self._wpos) != ntot:
+ raise ValueError("wpos has the wrong numbers of elements")
+ if self._g1 is not None and len(self._g1) != ntot:
+ raise ValueError("g1 has the wrong numbers of elements")
+ if self._g2 is not None and len(self._g2) != ntot:
+ raise ValueError("g2 has the wrong numbers of elements")
+ if self._k is not None and len(self._k) != ntot:
+ raise ValueError("k has the wrong numbers of elements")
+ if self._patch is not None and len(self._patch) != ntot:
+ raise ValueError("patch has the wrong numbers of elements")
+ if ntot == 0:
+ raise ValueError("Input arrays have zero length")
+
+ if x is not None or self.config.get('x_col','0') not in [0,'0']:
+ if 'x_units' in self.config and 'y_units' not in self.config:
+ raise TypeError("x_units specified without specifying y_units")
+ if 'y_units' in self.config and 'x_units' not in self.config:
+ raise TypeError("y_units specified without specifying x_units")
+ else:
+ if 'x_units' in self.config:
+ raise TypeError("x_units is invalid without x")
+ if 'y_units' in self.config:
+ raise TypeError("y_units is invalid without y")
+ if ra is not None or self.config.get('ra_col','0') not in [0,'0']:
+ if not self.config.get('ra_units',None):
+ raise TypeError("ra_units is required when using ra, dec")
+ if not self.config.get('dec_units',None):
+ raise TypeError("dec_units is required when using ra, dec")
+ else:
+ if 'ra_units' in self.config:
+ raise TypeError("ra_units is invalid without ra")
+ if 'dec_units' in self.config:
+ raise TypeError("dec_units is invalid without dec")
+
+ if file_name is None:
+ # For vector input option, can finish up now.
+ if self._single_patch is not None:
+ self._select_patch(self._single_patch)
+ self._finish_input()
+
+ @property
+ def loaded(self):
+ # _x gets set regardless of whether input used x,y or ra,dec, so the state of this
+ # attribute is a good sentinal for whether the file has been loaded yet.
+ return self._x is not None
+
+ @property
+ def x(self):
+ self.load()
+ return self._x
+
+ @property
+ def y(self):
+ self.load()
+ return self._y
+
+ @property
+ def z(self):
+ self.load()
+ return self._z
+
+ @property
+ def ra(self):
+ self.load()
+ return self._ra
+
+ @property
+ def dec(self):
+ self.load()
+ return self._dec
+
+ @property
+ def r(self):
+ self.load()
+ return self._r
+
+ @property
+ def w(self):
+ self.load()
+ return self._w
+
+ @property
+ def wpos(self):
+ self.load()
+ return self._wpos
+
+ @property
+ def g1(self):
+ self.load()
+ return self._g1
+
+ @property
+ def g2(self):
+ self.load()
+ return self._g2
+
+ @property
+ def k(self):
+ self.load()
+ return self._k
+
+ @property
+ def npatch(self):
+ if self._npatch is None:
+ self.load()
+ return self._npatch
+
+ @property
+ def patch(self):
+ if self._single_patch is not None:
+ return self._single_patch
+ else:
+ self.load()
+ return self._patch
+
+ @property
+ def patches(self):
+ return self.get_patches()
+
+ @property
+ def patch_centers(self):
+ return self.get_patch_centers()
+
+ @property
+ def varg(self):
+ if self._varg is None:
+ if self.nontrivial_w:
+ if self.g1 is not None:
+ use = self.w != 0
+ self._varg = np.sum(self.w[use]**2 * (self.g1[use]**2 + self.g2[use]**2))
+ # The 2 is because we need the variance _per componenet_.
+ self._varg /= 2.*self.sumw
+ else:
+ self._varg = 0.
+ else:
+ if self.g1 is not None:
+ self._varg = np.sum(self.g1**2 + self.g2**2) / (2.*self.nobj)
+ else:
+ self._varg = 0.
+ return self._varg
+
+ @property
+ def vark(self):
+ if self._vark is None:
+ if self.nontrivial_w:
+ if self.k is not None:
+ use = self.w != 0
+ self._meank = np.sum(self.w[use] * self.k[use]) / self.sumw
+ self._meank2 = np.sum(self.w[use]**2 * self.k[use]) / self.sumw2
+ self._vark = np.sum(self.w[use]**2 * (self.k[use]-self._meank)**2) / self.sumw
+ else:
+ self._meank = self._meank2 = 0.
+ self._vark = 0.
+ else:
+ if self.k is not None:
+ self._meank = self._meank2 = np.mean(self.k)
+ self._vark = np.sum((self.k-self._meank)**2) / self.nobj
+ else:
+ self._meank = self._meank2 = 0.
+ self._vark = 0.
+ return self._vark
+
+ @property
+ def nontrivial_w(self):
+ if self._nontrivial_w is None: self.load()
+ return self._nontrivial_w
+
+ @property
+ def ntot(self):
+ return len(self.x)
+
+ @property
+ def nobj(self):
+ if self._nobj is None:
+ if self.nontrivial_w:
+ use = self._w != 0
+ self._nobj = np.sum(use)
+ else:
+ self._nobj = self.ntot
+ return self._nobj
+
+ @property
+ def sumw(self):
+ if self._sumw is None: self.load()
+ return self._sumw
+
+ @property
+ def sumw2(self):
+ if self._sumw2 is None:
+ if self.nontrivial_w:
+ self._sumw2 = np.sum(self.w**2)
+ else:
+ self._sumw2 = self.ntot
+ return self._sumw2
+
+ @property
+ def coords(self):
+ if self.ra is not None:
+ if self.r is None:
+ return 'spherical'
+ else:
+ return '3d'
+ else:
+ if self.z is None:
+ return 'flat'
+ else:
+ return '3d'
+
+ def _get_center_size(self):
+ if not hasattr(self, '_cen_s'):
+ mx = np.mean(self.x)
+ my = np.mean(self.y)
+ mz = 0
+ dsq = (self.x - mx)**2 + (self.y - my)**2
+ if self.z is not None:
+ mz = np.mean(self.z)
+ dsq += (self.z - mz)**2
+ s = np.max(dsq)**0.5
+ self._cen_s = (mx, my, mz, s)
+ return self._cen_s
+
+ def _finish_input(self):
+ # Finish processing the data based on given inputs.
+
+ # Apply flips if requested
+ flip_g1 = get_from_list(self.config,'flip_g1',self._num,bool,False)
+ flip_g2 = get_from_list(self.config,'flip_g2',self._num,bool,False)
+ if flip_g1:
+ self.logger.info(" Flipping sign of g1.")
+ self._g1 = -self._g1
+ if flip_g2:
+ self.logger.info(" Flipping sign of g2.")
+ self._g2 = -self._g2
+
+ # Convert the flag to a weight
+ if self._flag is not None:
+ if 'ignore_flag' in self.config:
+ ignore_flag = get_from_list(self.config,'ignore_flag',self._num,int)
+ else:
+ ok_flag = get_from_list(self.config,'ok_flag',self._num,int,0)
+ ignore_flag = ~ok_flag
+ # If we don't already have a weight column, make one with all values = 1.
+ if self._w is None:
+ self._w = np.ones_like(self._flag, dtype=float)
+ self._w[(self._flag & ignore_flag)!=0] = 0
+ if self._wpos is not None:
+ self._wpos[(self._flag & ignore_flag)!=0] = 0
+ self.logger.debug('Applied flag')
+
+ # Check for NaN's:
+ self.checkForNaN(self._x,'x')
+ self.checkForNaN(self._y,'y')
+ self.checkForNaN(self._z,'z')
+ self.checkForNaN(self._ra,'ra')
+ self.checkForNaN(self._dec,'dec')
+ self.checkForNaN(self._r,'r')
+ self.checkForNaN(self._g1,'g1')
+ self.checkForNaN(self._g2,'g2')
+ self.checkForNaN(self._k,'k')
+ self.checkForNaN(self._w,'w')
+ self.checkForNaN(self._wpos,'wpos')
+
+ # If using ra/dec, generate x,y,z
+ # Note: This also makes self.ntot work properly.
+ self._generate_xyz()
+
+ # Copy w to wpos if necessary (Do this after checkForNaN's, since this may set some
+ # entries to have w=0.)
+ if self._wpos is None:
+ self.logger.debug('Using w for wpos')
+ else:
+ # Check that any wpos == 0 points also have w == 0
+ if np.any(self._wpos == 0.):
+ if self._w is None:
+ self.logger.warning('Some wpos values are zero, setting w=0 for these points.')
+ self._w = np.ones((self.ntot), dtype=float)
+ else:
+ if np.any(self._w[self._wpos == 0.] != 0.):
+ self.logger.error('Some wpos values = 0 but have w!=0. This is invalid.\n'
+ 'Setting w=0 for these points.')
+ self._w[self._wpos == 0.] = 0.
+
+ if self._w is not None:
+ self._nontrivial_w = True
+ self._sumw = np.sum(self._w)
+ if self._sumw == 0:
+ raise ValueError("Catalog has invalid sumw == 0")
+ else:
+ self._nontrivial_w = False
+ self._sumw = self.ntot
+ # Make w all 1s to simplify the use of w later in code.
+ self._w = np.ones((self.ntot), dtype=float)
+
+ keep_zero_weight = get(self.config,'keep_zero_weight',bool,False)
+ if self._nontrivial_w and not keep_zero_weight:
+ wpos = self._wpos if self._wpos is not None else self._w
+ if np.any(wpos == 0):
+ self.select(np.where(wpos != 0)[0])
+
+ if self._single_patch is not None or self._patch is not None:
+ # Easier to get these options out of the way first.
+ pass
+ elif self._centers is not None:
+ if ((self.coords == 'flat' and self._centers.shape[1] != 2) or
+ (self.coords != 'flat' and self._centers.shape[1] != 3)):
+ raise ValueError("Centers array has wrong shape.")
+ self._assign_patches()
+ self.logger.info("Assigned patch numbers according %d centers",self._npatch)
+ elif self._npatch is not None and self._npatch != 1:
+ init = get(self.config,'kmeans_init',str,'tree')
+ alt = get(self.config,'kmeans_alt',bool,False)
+ max_top = int.bit_length(self._npatch)-1
+ c = 'spherical' if self._ra is not None else self.coords
+ field = self.getNField(max_top=max_top, coords=c)
+ self.logger.info("Finding %d patches using kmeans.",self._npatch)
+ self._patch, self._centers = field.run_kmeans(self._npatch, init=init, alt=alt)
+ # Clear the cached NField, since we will almost certainly not want this
+ # particular one again, even if doing N-based correlations (since max_top, etc.
+ # is almost certainly going to be different).
+ self.nfields.clear()
+
+ self.logger.info(" nobj = %d",self.nobj)
+
+ def _assign_patches(self):
+ # This is equivalent to the following:
+ # field = self.getNField()
+ # self._patch = field.kmeans_assign_patches(self._centers)
+ # However, when the field is not already created, it's faster to just run through
+ # all the points directly and assign which one is closest.
+ self._patch = np.empty(self.ntot, dtype=int)
+ centers = np.ascontiguousarray(self._centers)
+ set_omp_threads(self.config.get('num_threads',None))
+ _lib.QuickAssign(dp(centers), self._npatch,
+ dp(self.x), dp(self.y), dp(self.z), lp(self._patch), self.ntot)
+
+ def _set_npatch(self):
+ npatch = max(self._patch) + 1
+ if self._npatch not in [None, 1] and npatch > self._npatch:
+ # Note: it's permissible for self._npatch to be larger, but not smaller.
+ raise ValueError("npatch is incompatible with provided patch numbers")
+ self._npatch = npatch
+ self.logger.info("Assigned patch numbers 0..%d",self._npatch-1)
+
+ def _get_patch_index(self, single_patch):
+ if self._patch is not None:
+ # This is straightforward. Just select the rows with patch == single_patch
+ use = np.where(self._patch == single_patch)[0]
+ elif self._centers is not None:
+ self._generate_xyz()
+ use = np.empty(self.ntot, dtype=int)
+ from .util import double_ptr as dp
+ from .util import long_ptr as lp
+ npatch = self._centers.shape[0]
+ centers = np.ascontiguousarray(self._centers)
+ if self._z is None:
+ assert centers.shape[1] == 2
+ else:
+ assert centers.shape[1] == 3
+ set_omp_threads(self.config.get('num_threads',None))
+ _lib.SelectPatch(single_patch, dp(centers), npatch,
+ dp(self._x), dp(self._y), dp(self._z),
+ lp(use), self.ntot)
+ use = np.where(use)[0]
+ else:
+ use = slice(None) # Which ironically means use all. :)
+ return use
+
+ def _apply_radec_units(self):
+ self.ra_units = get_from_list(self.config,'ra_units',self._num)
+ self.dec_units = get_from_list(self.config,'dec_units',self._num)
+ self._ra *= self.ra_units
+ self._dec *= self.dec_units
+
+ def _apply_xyz_units(self):
+ self.x_units = get_from_list(self.config,'x_units',self._num,str, 'radians')
+ self.y_units = get_from_list(self.config,'y_units',self._num,str, 'radians')
+ self._x *= self.x_units
+ self._y *= self.y_units
+
+ def _generate_xyz(self):
+ if self._x is None:
+ assert self._y is None
+ assert self._z is None
+ assert self._ra is not None
+ assert self._dec is not None
+ ntot = len(self._ra)
+ self._x = np.empty(ntot, dtype=float)
+ self._y = np.empty(ntot, dtype=float)
+ self._z = np.empty(ntot, dtype=float)
+ from .util import double_ptr as dp
+ set_omp_threads(self.config.get('num_threads',None))
+ _lib.GenerateXYZ(dp(self._x), dp(self._y), dp(self._z),
+ dp(self._ra), dp(self._dec), dp(self._r), ntot)
+ self.x_units = self.y_units = 1.
+
+ def _select_patch(self, single_patch):
+ # Trim the catalog to only include a single patch
+ # Note: This is slightly inefficient in that it reads the whole catalog first
+ # and then removes all but one patch. But that's easier for now that figuring out
+ # which items to remove along the way based on the patch_centers.
+ indx = self._get_patch_index(single_patch)
+ self._patch = None
+ self.select(indx)
+
+[docs] def select(self, indx):
+ """Trim the catalog to only include those objects with the give indices.
+
+ Parameters:
+ indx: A numpy array of index values to keep.
+ """
+ if type(indx) == slice and indx == slice(None):
+ return
+ self._x = self._x[indx] if self._x is not None else None
+ self._y = self._y[indx] if self._y is not None else None
+ self._z = self._z[indx] if self._z is not None else None
+ self._ra = self._ra[indx] if self._ra is not None else None
+ self._dec = self._dec[indx] if self._dec is not None else None
+ self._r = self._r[indx] if self._r is not None else None
+ self._w = self._w[indx] if self._w is not None else None
+ self._wpos = self._wpos[indx] if self._wpos is not None else None
+ self._g1 = self._g1[indx] if self._g1 is not None else None
+ self._g2 = self._g2[indx] if self._g2 is not None else None
+ self._k = self._k[indx] if self._k is not None else None
+ self._patch = self._patch[indx] if self._patch is not None else None
+
+[docs] def makeArray(self, col, col_str, dtype=float):
+ """Turn the input column into a numpy array if it wasn't already.
+ Also make sure the input is 1-d.
+
+ Parameters:
+ col (array-like): The input column to be converted into a numpy array.
+ col_str (str): The name of the column. Used only as information in logging output.
+ dtype (type): The dtype for the returned array. (default: float)
+
+ Returns:
+ The column converted to a 1-d numpy array.
+ """
+ if col is not None:
+ col = np.array(col,dtype=dtype)
+ if len(col.shape) != 1:
+ s = col.shape
+ col = col.reshape(-1)
+ self.logger.warning("Warning: Input %s column was not 1-d.\n"%col_str +
+ " Reshaping from %s to %s"%(s,col.shape))
+ col = np.ascontiguousarray(col[self.start:self.end:self.every_nth])
+ return col
+
+[docs] def checkForNaN(self, col, col_str):
+ """Check if the column has any NaNs. If so, set those rows to have w[k]=0.
+
+ Parameters:
+ col (array): The input column to check.
+ col_str (str): The name of the column. Used only as information in logging output.
+ """
+ if col is not None and np.any(np.isnan(col)):
+ index = np.where(np.isnan(col))[0]
+ s = 's' if len(index) > 1 else ''
+ self.logger.warning("Warning: %d NaN%s found in %s column.",len(index),s,col_str)
+ if len(index) < 20:
+ self.logger.info("Skipping row%s %s.",s,index.tolist())
+ else:
+ self.logger.info("Skipping rows starting %s",
+ str(index[:10].tolist()).replace(']',' ...]'))
+ if self._w is None:
+ self._w = np.ones_like(col, dtype=float)
+ self._w[index] = 0
+ col[index] = 0 # Don't leave the nans there.
+
+ def _check_file(self, file_name, reader, num=0, is_rand=False):
+ # Just check the consistency of the various column numbers so we can fail fast.
+
+ # Get the column names
+ x_col = get_from_list(self.config,'x_col',num,str,'0')
+ y_col = get_from_list(self.config,'y_col',num,str,'0')
+ z_col = get_from_list(self.config,'z_col',num,str,'0')
+ ra_col = get_from_list(self.config,'ra_col',num,str,'0')
+ dec_col = get_from_list(self.config,'dec_col',num,str,'0')
+ r_col = get_from_list(self.config,'r_col',num,str,'0')
+ w_col = get_from_list(self.config,'w_col',num,str,'0')
+ wpos_col = get_from_list(self.config,'wpos_col',num,str,'0')
+ flag_col = get_from_list(self.config,'flag_col',num,str,'0')
+ g1_col = get_from_list(self.config,'g1_col',num,str,'0')
+ g2_col = get_from_list(self.config,'g2_col',num,str,'0')
+ k_col = get_from_list(self.config,'k_col',num,str,'0')
+ patch_col = get_from_list(self.config,'patch_col',num,str,'0')
+ allow_xyz = self.config.get('allow_xyz', False)
+
+ if x_col != '0' or y_col != '0':
+ if x_col == '0':
+ raise ValueError("x_col missing for file %s"%file_name)
+ if y_col == '0':
+ raise ValueError("y_col missing for file %s"%file_name)
+ if ra_col != '0' and not allow_xyz:
+ raise ValueError("ra_col not allowed in conjunction with x/y cols")
+ if dec_col != '0' and not allow_xyz:
+ raise ValueError("dec_col not allowed in conjunction with x/y cols")
+ if r_col != '0' and not allow_xyz:
+ raise ValueError("r_col not allowed in conjunction with x/y cols")
+ elif ra_col != '0' or dec_col != '0':
+ if ra_col == '0':
+ raise ValueError("ra_col missing for file %s"%file_name)
+ if dec_col == '0':
+ raise ValueError("dec_col missing for file %s"%file_name)
+ if z_col != '0' and not allow_xyz:
+ raise ValueError("z_col not allowed in conjunction with ra/dec cols")
+ else:
+ raise ValueError("No valid position columns specified for file %s"%file_name)
+
+ if g1_col == '0' and isGColRequired(self.orig_config,num):
+ raise ValueError("g1_col is missing for file %s"%file_name)
+ if g2_col == '0' and isGColRequired(self.orig_config,num):
+ raise ValueError("g2_col is missing for file %s"%file_name)
+ if k_col == '0' and isKColRequired(self.orig_config,num):
+ raise ValueError("k_col is missing for file %s"%file_name)
+
+ # Either both shoudl be 0 or both != 0.
+ if (g1_col == '0') != (g2_col == '0'):
+ raise ValueError("g1_col, g2_col=(%s, %s) are invalid for file %s"%(
+ g1_col,g2_col,file_name))
+
+ # This opens the file enough to read things inside. The full read doesn't happen here.
+ with reader:
+
+ # get the vanilla "ext" parameter
+ ext = get_from_list(self.config, 'ext', num, str, reader.default_ext)
+
+ # Technically, this doesn't catch all possible errors. If someone specifies
+ # an invalid flag_ext or something, then they'll get the fitsio error message.
+ # But this should probably catch the majorit of error cases.
+ reader.check_valid_ext(ext)
+
+ if x_col != '0':
+ x_ext = get_from_list(self.config, 'x_ext', num, str, ext)
+ y_ext = get_from_list(self.config, 'y_ext', num, str, ext)
+ if x_col not in reader.names(x_ext):
+ raise ValueError("x_col=%s is invalid for file %s"%(x_col,file_name))
+ if y_col not in reader.names(y_ext):
+ raise ValueError("y_col=%s is invalid for file %s"%(y_col, file_name))
+ if z_col != '0':
+ z_ext = get_from_list(self.config, 'z_ext', num, str, ext)
+ if z_col not in reader.names(z_ext):
+ raise ValueError("z_col=%s is invalid for file %s"%(z_col, file_name))
+ else:
+ ra_ext = get_from_list(self.config, 'ra_ext', num, str, ext)
+ dec_ext = get_from_list(self.config, 'dec_ext', num, str, ext)
+ if ra_col not in reader.names(ra_ext):
+ raise ValueError("ra_col=%s is invalid for file %s"%(ra_col, file_name))
+ if dec_col not in reader.names(dec_ext):
+ raise ValueError("dec_col=%s is invalid for file %s"%(dec_col, file_name))
+ if r_col != '0':
+ r_ext = get_from_list(self.config, 'r_ext', num, str, ext)
+ if r_col not in reader.names(r_ext):
+ raise ValueError("r_col=%s is invalid for file %s"%(r_col, file_name))
+
+ if w_col != '0':
+ w_ext = get_from_list(self.config, 'w_ext', num, str, ext)
+ if w_col not in reader.names(w_ext):
+ raise ValueError("w_col=%s is invalid for file %s"%(w_col, file_name))
+
+ if wpos_col != '0':
+ wpos_ext = get_from_list(self.config, 'wpos_ext', num, str, ext)
+ if wpos_col not in reader.names(wpos_ext):
+ raise ValueError("wpos_col=%s is invalid for file %s"%(wpos_col, file_name))
+
+ if flag_col != '0':
+ flag_ext = get_from_list(self.config, 'flag_ext', num, str, ext)
+ if flag_col not in reader.names(flag_ext):
+ raise ValueError("flag_col=%s is invalid for file %s"%(flag_col, file_name))
+
+ if patch_col != '0':
+ patch_ext = get_from_list(self.config, 'patch_ext', num, str, ext)
+ if patch_col not in reader.names(patch_ext):
+ raise ValueError("patch_col=%s is invalid for file %s"%(patch_col, file_name))
+
+ if is_rand: return
+
+ if g1_col != '0':
+ g1_ext = get_from_list(self.config, 'g1_ext', num, str, ext)
+ g2_ext = get_from_list(self.config, 'g2_ext', num, str, ext)
+ if (g1_col not in reader.names(g1_ext) or
+ g2_col not in reader.names(g2_ext)):
+ if isGColRequired(self.orig_config,num):
+ raise ValueError(
+ "g1_col, g2_col=(%s, %s) are invalid for file %s"%(
+ g1_col, g2_col, file_name))
+ else:
+ self.logger.warning(
+ "Warning: skipping g1_col, g2_col=(%s, %s) for %s, num=%d "%(
+ g1_col, g2_col, file_name, num) +
+ "because they are invalid, but unneeded.")
+
+ if k_col != '0':
+ k_ext = get_from_list(self.config, 'k_ext', num, str, ext)
+ if k_col not in reader.names(k_ext):
+ if isKColRequired(self.orig_config,num):
+ raise ValueError("k_col=%s is invalid for file %s"%(k_col, file_name))
+ else:
+ self.logger.warning(
+ "Warning: skipping k_col=%s for %s, num=%d "%(
+ k_col, file_name, num) +
+ "because it is invalid, but unneeded.")
+
+ def _pandas_warning(self):
+ if Catalog._emitted_pandas_warning:
+ return
+ self.logger.warning(
+ "Unable to import pandas.. Using np.genfromtxt instead.\n"+
+ "Installing pandas is recommended for increased speed when "+
+ "reading ASCII catalogs.")
+ Catalog._emitted_pandas_warning = True
+
+ def _read_file(self, file_name, reader, num, is_rand):
+ # Helper functions for things we might do in one of two places.
+ def set_pos(data, x_col, y_col, z_col, ra_col, dec_col, r_col):
+ if x_col != '0' and x_col in data:
+ self._x = data[x_col].astype(float)
+ self.logger.debug('read x')
+ self._y = data[y_col].astype(float)
+ self.logger.debug('read y')
+ if z_col != '0':
+ self._z = data[z_col].astype(float)
+ self.logger.debug('read z')
+ self._apply_xyz_units()
+ if ra_col != '0' and ra_col in data:
+ self._ra = data[ra_col].astype(float)
+ self.logger.debug('read ra')
+ self._dec = data[dec_col].astype(float)
+ self.logger.debug('read dec')
+ if r_col != '0':
+ self._r = data[r_col].astype(float)
+ self.logger.debug('read r')
+ self._apply_radec_units()
+
+ def set_patch(data, patch_col):
+ if patch_col != '0' and patch_col in data:
+ self._patch = data[patch_col].astype(int)
+ self.logger.debug('read patch')
+ self._set_npatch()
+
+ # Get the column names
+ x_col = get_from_list(self.config,'x_col',num,str,'0')
+ y_col = get_from_list(self.config,'y_col',num,str,'0')
+ z_col = get_from_list(self.config,'z_col',num,str,'0')
+ ra_col = get_from_list(self.config,'ra_col',num,str,'0')
+ dec_col = get_from_list(self.config,'dec_col',num,str,'0')
+ r_col = get_from_list(self.config,'r_col',num,str,'0')
+ w_col = get_from_list(self.config,'w_col',num,str,'0')
+ wpos_col = get_from_list(self.config,'wpos_col',num,str,'0')
+ flag_col = get_from_list(self.config,'flag_col',num,str,'0')
+ g1_col = get_from_list(self.config,'g1_col',num,str,'0')
+ g2_col = get_from_list(self.config,'g2_col',num,str,'0')
+ k_col = get_from_list(self.config,'k_col',num,str,'0')
+ patch_col = get_from_list(self.config,'patch_col',num,str,'0')
+
+ with reader:
+
+ ext = get_from_list(self.config, 'ext', num, str, reader.default_ext)
+
+ # Figure out what slice to use. If all rows, then None is faster,
+ # otherwise give the range explicitly.
+ if self.start == 0 and self.end is None and self.every_nth == 1:
+ s = slice(None)
+ # fancy indexing in h5py is incredibly slow, so we explicitly
+ # check if we can slice or not. This checks for the fitsio version in
+ # the fits case
+ elif reader.can_slice:
+ s = slice(self.start, self.end, self.every_nth)
+ else:
+ # Note: this is a workaround for a bug in fitsio <= 1.0.6.
+ # cf. https://github.com/esheldon/fitsio/pull/286
+ # We should be able to always use s = slice(self.start, self.end, self.every_nth)
+ if x_col != '0':
+ x_ext = get_from_list(self.config, 'x_ext', num, str, ext)
+ col = x_col
+ else:
+ x_ext = get_from_list(self.config, 'ra_ext', num, str, ext)
+ col = ra_col
+ end = self.end if self.end is not None else reader.row_count(col, x_ext)
+ s = np.arange(self.start, end, self.every_nth)
+
+ all_cols = [x_col, y_col, z_col,
+ ra_col, dec_col, r_col,
+ patch_col,
+ w_col, wpos_col, flag_col,
+ g1_col, g2_col, k_col]
+
+ # It's faster in FITS to read in all the columns in one read, rather than individually.
+ # Typically (very close to always!), all the columns are in the same extension.
+ # Thus, the following would normally work fine.
+ # use_cols = [c for c in all_cols if c != '0']
+ # data = fits[ext][use_cols][:]
+ # However, we allow the option to have different columns read from different extensions.
+ # So this is slightly more complicated.
+ x_ext = get_from_list(self.config, 'x_ext', num, str, ext)
+ y_ext = get_from_list(self.config, 'y_ext', num, str, ext)
+ z_ext = get_from_list(self.config, 'z_ext', num, str, ext)
+ ra_ext = get_from_list(self.config, 'ra_ext', num, str, ext)
+ dec_ext = get_from_list(self.config, 'dec_ext', num, str, ext)
+ r_ext = get_from_list(self.config, 'r_ext', num, str, ext)
+ patch_ext = get_from_list(self.config, 'patch_ext', num, str, ext)
+ w_ext = get_from_list(self.config, 'w_ext', num, str, ext)
+ wpos_ext = get_from_list(self.config, 'wpos_ext', num, str, ext)
+ flag_ext = get_from_list(self.config, 'flag_ext', num, str, ext)
+ g1_ext = get_from_list(self.config, 'g1_ext', num, str, ext)
+ g2_ext = get_from_list(self.config, 'g2_ext', num, str, ext)
+ k_ext = get_from_list(self.config, 'k_ext', num, str, ext)
+ all_exts = [x_ext, y_ext, z_ext,
+ ra_ext, dec_ext, r_ext,
+ patch_ext,
+ w_ext, wpos_ext, flag_ext,
+ g1_ext, g2_ext, k_ext]
+ col_by_ext = dict(zip(all_cols,all_exts))
+ all_exts = set(all_exts + [ext])
+ all_cols = [c for c in all_cols if c != '0']
+
+ data = {}
+ # Also, if we are only reading in one patch, we should adjust s before doing this.
+ if self._single_patch is not None:
+ if patch_col != '0':
+ data[patch_col] = reader.read(patch_col, s, patch_ext)
+ all_cols.remove(patch_col)
+ set_patch(data, patch_col)
+ elif self._centers is not None:
+ pos_cols = [x_col, y_col, z_col, ra_col, dec_col, r_col]
+ pos_cols = [c for c in pos_cols if c != '0']
+ for c in pos_cols:
+ all_cols.remove(c)
+ for ext in all_exts:
+ use_cols1 = [c for c in pos_cols if col_by_ext[c] == ext]
+ data1 = reader.read(use_cols1, s, ext)
+ for c in use_cols1:
+ data[c] = data1[c]
+ set_pos(data, x_col, y_col, z_col, ra_col, dec_col, r_col)
+ x_col = y_col = z_col = ra_col = dec_col = r_col = '0'
+ use = self._get_patch_index(self._single_patch)
+ self.select(use)
+ if isinstance(s,np.ndarray):
+ s = s[use]
+ elif s == slice(None):
+ s = use
+ else:
+ end1 = np.max(use)+s.start+1
+ s = np.arange(s.start, end1, s.step)[use]
+ self._patch = None
+ data = {} # Start fresh, since the ones we used so far are done.
+
+ # We might actually be done now, in which case, just return.
+ # (Else the fits read below won't actually work.)
+ if len(all_cols) == 0 or (isinstance(s,np.ndarray) and len(s) == 0):
+ return
+
+ # Now read the rest using the updated s
+ for ext in all_exts:
+ use_cols1 = [c for c in all_cols
+ if col_by_ext[c] == ext and c in reader.names(ext)]
+ if len(use_cols1) == 0:
+ continue
+ data1 = reader.read(use_cols1, s, ext)
+ for c in use_cols1:
+ data[c] = data1[c]
+
+ # Set position values
+ set_pos(data, x_col, y_col, z_col, ra_col, dec_col, r_col)
+
+ # Set patch
+ set_patch(data, patch_col)
+
+ # Set w
+ if w_col != '0':
+ self._w = data[w_col].astype(float)
+ self.logger.debug('read w')
+
+ # Set wpos
+ if wpos_col != '0':
+ self._wpos = data[wpos_col].astype(float)
+ self.logger.debug('read wpos')
+
+ # Set flag
+ if flag_col != '0':
+ self._flag = data[flag_col].astype(int)
+ self.logger.debug('read flag')
+
+ # Skip g1,g2,k if this file is a random catalog
+ if not is_rand:
+ # Set g1,g2
+ if g1_col in reader.names(g1_ext):
+ self._g1 = data[g1_col].astype(float)
+ self.logger.debug('read g1')
+ self._g2 = data[g2_col].astype(float)
+ self.logger.debug('read g2')
+
+ # Set k
+ if k_col in reader.names(k_ext):
+ self._k = data[k_col].astype(float)
+ self.logger.debug('read k')
+
+ @property
+ def nfields(self):
+ if not hasattr(self, '_nfields'):
+ # Make simple functions that call NField, etc. with self as the first argument.
+ # Note: LRU_Cache keys on the args, not kwargs, so everything but logger should
+ # be in args for this function. We convert them to kwargs for the NFields init call.
+ def get_nfield(min_size, max_size, split_method, brute, min_top, max_top, coords,
+ rng, logger=None):
+ return NField(self, min_size=min_size, max_size=max_size,
+ split_method=split_method, brute=brute,
+ min_top=min_top, max_top=max_top, coords=coords,
+ rng=rng, logger=logger)
+ # Now wrap these in LRU_Caches with (initially) just 1 element being cached.
+ self._nfields = LRU_Cache(get_nfield, 1)
+ return self._nfields
+
+ @property
+ def kfields(self):
+ if not hasattr(self, '_kfields'):
+ def get_kfield(min_size, max_size, split_method, brute, min_top, max_top, coords,
+ rng, logger=None):
+ return KField(self, min_size=min_size, max_size=max_size,
+ split_method=split_method, brute=brute,
+ min_top=min_top, max_top=max_top, coords=coords,
+ rng=rng, logger=logger)
+ self._kfields = LRU_Cache(get_kfield, 1)
+ return self._kfields
+
+ @property
+ def gfields(self):
+ if not hasattr(self, '_gfields'):
+ def get_gfield(min_size, max_size, split_method, brute, min_top, max_top, coords,
+ rng, logger=None):
+ return GField(self, min_size=min_size, max_size=max_size,
+ split_method=split_method, brute=brute,
+ min_top=min_top, max_top=max_top, coords=coords,
+ rng=rng, logger=logger)
+ self._gfields = LRU_Cache(get_gfield, 1)
+ return self._gfields
+
+ @property
+ def nsimplefields(self):
+ if not hasattr(self, '_nsimplefields'):
+ def get_nsimplefield(logger=None):
+ return NSimpleField(self, logger=logger)
+ self._nsimplefields = LRU_Cache(get_nsimplefield, 1)
+ return self._nsimplefields
+
+ @property
+ def ksimplefields(self):
+ if not hasattr(self, '_ksimplefields'):
+ def get_ksimplefield(logger=None):
+ return KSimpleField(self, logger=logger)
+ self._ksimplefields = LRU_Cache(get_ksimplefield, 1)
+ return self._ksimplefields
+
+ @property
+ def gsimplefields(self):
+ if not hasattr(self, '_gsimplefields'):
+ def get_gsimplefield(logger=None):
+ return GSimpleField(self, logger=logger)
+ self._gsimplefields = LRU_Cache(get_gsimplefield, 1)
+ return self._gsimplefields
+
+[docs] def resize_cache(self, maxsize):
+ """Resize all field caches.
+
+ The various kinds of fields built from this catalog are cached. This may or may not
+ be an optimization for your use case. Normally only a single field is built for a
+ given catalog, and it is usually efficient to cache it, so it can be reused multiple
+ times. E.g. for the usual Landy-Szalay NN calculation:
+
+ >>> dd.process(data_cat)
+ >>> rr.process(rand_cat)
+ >>> dr.process(data_cat, rand_cat)
+
+ the third line will be able to reuse the same fields built for the data and randoms
+ in the first two lines.
+
+ However, if you are making many different fields from the same catalog -- for instance
+ because you keep changing the min_sep and max_sep for different calls -- then saving
+ them all will tend to blow up the memory.
+
+ Therefore, the default number of fields (of each type) to cache is 1. This lets the
+ first use case be efficient, but not use too much memory for the latter case.
+
+ If you prefer a different behavior, this method lets you change the number of fields to
+ cache. The cache is an LRU (Least Recently Used) cache, which means only the n most
+ recently used fields are saved. I.e. when it is full, the least recently used field
+ is removed from the cache.
+
+ If you call this with maxsize=0, then caching will be turned off. A new field will be
+ built each time you call a process function with this catalog.
+
+ If you call this with maxsize>1, then mutiple fields will be saved according to whatever
+ number you set. This will use more memory, but may be an optimization for you depending
+ on what your are doing.
+
+ Finally, if you want to set different sizes for the different kinds of fields, then
+ you can call resize separately for the different caches:
+
+ >>> cat.nfields.resize(maxsize)
+ >>> cat.kfields.resize(maxsize)
+ >>> cat.gfields.resize(maxsize)
+ >>> cat.nsimplefields.resize(maxsize)
+ >>> cat.ksimplefields.resize(maxsize)
+ >>> cat.gsimplefields.resize(maxsize)
+
+ Parameters:
+ maxsize (float): The new maximum number of fields of each type to cache.
+ """
+ if hasattr(self, '_nfields'): self.nfields.resize(maxsize)
+ if hasattr(self, '_kfields'): self.kfields.resize(maxsize)
+ if hasattr(self, '_gfields'): self.gfields.resize(maxsize)
+ if hasattr(self, '_nsimplefields'): self.nsimplefields.resize(maxsize)
+ if hasattr(self, '_ksimplefields'): self.ksimplefields.resize(maxsize)
+ if hasattr(self, '_gsimplefields'): self.gsimplefields.resize(maxsize)
+
+[docs] def clear_cache(self):
+ """Clear all field caches.
+
+ The various kinds of fields built from this catalog are cached. This may or may not
+ be an optimization for your use case. Normally only a single field is built for a
+ given catalog, and it is usually efficient to cache it, so it can be reused multiple
+ times. E.g. for the usual Landy-Szalay NN calculation:
+
+ >>> dd.process(data_cat)
+ >>> rr.process(rand_cat)
+ >>> dr.process(data_cat, rand_cat)
+
+ the third line will be able to reuse the same fields built for the data and randoms
+ in the first two lines.
+
+ However, this also means that the memory used for the field will persist as long as
+ the catalog object does. If you need to recover this memory and don't want to delete
+ the catalog yet, this method lets you clear the cache.
+
+ There are separate caches for each kind of field. If you want to clear just one or
+ some of them, you can call clear separately for the different caches:
+
+ >>> cat.nfields.clear()
+ >>> cat.kfields.clear()
+ >>> cat.gfields.clear()
+ >>> cat.nsimplefields.clear()
+ >>> cat.ksimplefields.clear()
+ >>> cat.gsimplefields.clear()
+ """
+ if hasattr(self, '_nfields'): self.nfields.clear()
+ if hasattr(self, '_kfields'): self.kfields.clear()
+ if hasattr(self, '_gfields'): self.gfields.clear()
+ if hasattr(self, '_nsimplefields'): self.nsimplefields.clear()
+ if hasattr(self, '_ksimplefields'): self.ksimplefields.clear()
+ if hasattr(self, '_gsimplefields'): self.gsimplefields.clear()
+ self._field = lambda : None # Acts like a dead weakref
+
+ @property
+ def field(self):
+ # The default is to return None here.
+ # This might also return None if weakref has expired.
+ # But if the weakref is alive, this returns the field we want.
+ return self._field()
+
+[docs] @depr_pos_kwargs
+ def getNField(self, *, min_size=0, max_size=None, split_method=None, brute=False,
+ min_top=None, max_top=10, coords=None, logger=None):
+ """Return an `NField` based on the positions in this catalog.
+
+ The `NField` object is cached, so this is efficient to call multiple times.
+ cf. `resize_cache` and `clear_cache`
+
+ Parameters:
+ min_size (float): The minimum radius cell required (usually min_sep). (default: 0)
+ max_size (float): The maximum radius cell required (usually max_sep). (default: None)
+ split_method (str): Which split method to use ('mean', 'median', 'middle', or 'random')
+ (default: 'mean'; this value can also be given in the Catalog
+ constructor in the config dict.)
+ brute (bool): Whether to force traversal to the leaves. (default: False)
+ min_top (int): The minimum number of top layers to use when setting up the
+ field. (default: :math:`\\max(3, \\log_2(N_{\\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the
+ field. (default: 10)
+ coords (str): The kind of coordinate system to use. (default: self.coords)
+ logger: A Logger object if desired (default: self.logger)
+
+ Returns:
+ An `NField` object
+ """
+ if split_method is None:
+ split_method = get(self.config,'split_method',str,'mean')
+ if logger is None:
+ logger = self.logger
+ field = self.nfields(min_size, max_size, split_method, brute, min_top, max_top, coords,
+ rng=self._rng, logger=logger)
+ self._field = weakref.ref(field)
+ return field
+
+
+[docs] @depr_pos_kwargs
+ def getKField(self, *, min_size=0, max_size=None, split_method=None, brute=False,
+ min_top=None, max_top=10, coords=None, logger=None):
+ """Return a `KField` based on the k values in this catalog.
+
+ The `KField` object is cached, so this is efficient to call multiple times.
+ cf. `resize_cache` and `clear_cache`
+
+ Parameters:
+ min_size (float): The minimum radius cell required (usually min_sep). (default: 0)
+ max_size (float): The maximum radius cell required (usually max_sep). (default: None)
+ split_method (str): Which split method to use ('mean', 'median', 'middle', or 'random')
+ (default: 'mean'; this value can also be given in the Catalog
+ constructor in the config dict.)
+ brute (bool): Whether to force traversal to the leaves. (default: False)
+ min_top (int): The minimum number of top layers to use when setting up the
+ field. (default: :math:`\\max(3, \\log_2(N_{\\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the
+ field. (default: 10)
+ coords (str): The kind of coordinate system to use. (default self.coords)
+ logger: A Logger object if desired (default: self.logger)
+
+ Returns:
+ A `KField` object
+ """
+ if split_method is None:
+ split_method = get(self.config,'split_method',str,'mean')
+ if self.k is None:
+ raise TypeError("k is not defined.")
+ if logger is None:
+ logger = self.logger
+ field = self.kfields(min_size, max_size, split_method, brute, min_top, max_top, coords,
+ rng=self._rng, logger=logger)
+ self._field = weakref.ref(field)
+ return field
+
+
+[docs] @depr_pos_kwargs
+ def getGField(self, *, min_size=0, max_size=None, split_method=None, brute=False,
+ min_top=None, max_top=10, coords=None, logger=None):
+ """Return a `GField` based on the g1,g2 values in this catalog.
+
+ The `GField` object is cached, so this is efficient to call multiple times.
+ cf. `resize_cache` and `clear_cache`.
+
+ Parameters:
+ min_size (float): The minimum radius cell required (usually min_sep). (default: 0)
+ max_size (float): The maximum radius cell required (usually max_sep). (default: None)
+ split_method (str): Which split method to use ('mean', 'median', 'middle', or 'random')
+ (default: 'mean'; this value can also be given in the Catalog
+ constructor in the config dict.)
+ brute (bool): Whether to force traversal to the leaves. (default: False)
+ min_top (int): The minimum number of top layers to use when setting up the
+ field. (default: :math:`\\max(3, \\log_2(N_{\\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the
+ field. (default: 10)
+ coords (str): The kind of coordinate system to use. (default self.coords)
+ logger: A Logger object if desired (default: self.logger)
+
+ Returns:
+ A `GField` object
+ """
+ if split_method is None:
+ split_method = get(self.config,'split_method',str,'mean')
+ if self.g1 is None or self.g2 is None:
+ raise TypeError("g1,g2 are not defined.")
+ if logger is None:
+ logger = self.logger
+ field = self.gfields(min_size, max_size, split_method, brute, min_top, max_top, coords,
+ rng=self._rng, logger=logger)
+ self._field = weakref.ref(field)
+ return field
+
+
+[docs] @depr_pos_kwargs
+ def getNSimpleField(self, *, logger=None):
+ """Return an `NSimpleField` based on the positions in this catalog.
+
+ The `NSimpleField` object is cached, so this is efficient to call multiple times.
+ cf. `resize_cache` and `clear_cache`
+
+ Parameters:
+ logger: A Logger object if desired (default: self.logger)
+
+ Returns:
+ An `NSimpleField` object
+ """
+ if logger is None:
+ logger = self.logger
+ return self.nsimplefields(logger=logger)
+
+
+[docs] @depr_pos_kwargs
+ def getKSimpleField(self, *, logger=None):
+ """Return a `KSimpleField` based on the k values in this catalog.
+
+ The `KSimpleField` object is cached, so this is efficient to call multiple times.
+ cf. `resize_cache` and `clear_cache`
+
+ Parameters:
+ logger: A Logger object if desired (default: self.logger)
+
+ Returns:
+ A `KSimpleField` object
+ """
+ if self.k is None:
+ raise TypeError("k is not defined.")
+ if logger is None:
+ logger = self.logger
+ return self.ksimplefields(logger=logger)
+
+
+[docs] @depr_pos_kwargs
+ def getGSimpleField(self, *, logger=None):
+ """Return a `GSimpleField` based on the g1,g2 values in this catalog.
+
+ The `GSimpleField` object is cached, so this is efficient to call multiple times.
+ cf. `resize_cache` and `clear_cache`
+
+ Parameters:
+ logger: A Logger object if desired (default: self.logger)
+
+ Returns:
+ A `GSimpleField` object
+ """
+ if self.g1 is None or self.g2 is None:
+ raise TypeError("g1,g2 are not defined.")
+ if logger is None:
+ logger = self.logger
+ return self.gsimplefields(logger=logger)
+
+ def _weighted_mean(self, x, idx=None):
+ # Find the weighted mean of some column.
+ # If weights are set, then return sum(w * x) / sum(w)
+ # Else, just sum(x) / N
+ if self.nontrivial_w:
+ if idx is None:
+ return np.sum(x * self.w) / self.sumw
+ else:
+ return np.sum(x[idx] * self.w[idx]) / np.sum(self.w[idx])
+ else:
+ return np.mean(x[idx])
+
+[docs] def get_patch_centers(self):
+ """Return an array of patch centers corresponding to the patches in this catalog.
+
+ If the patches were set either using K-Means or by giving the centers, then this
+ will just return that same center array. Otherwise, it will be calculated from the
+ positions of the objects with each patch number.
+
+ This function is automatically called when accessing the property
+ ``patch_centers``. So you should not normally need to call it directly.
+
+ Returns:
+ An array of center coordinates used to make the patches.
+ Shape is (npatch, 2) for flat geometries or (npatch, 3) for 3d or
+ spherical geometries. In the latter case, the centers represent
+ (x,y,z) coordinates on the unit sphere.
+ """
+ # Early exit
+ if self._centers is not None:
+ return self._centers
+
+ self.load()
+ if self._patch is None:
+ if self.coords == 'flat':
+ self._centers = np.array([[self._weighted_mean(self.x),
+ self._weighted_mean(self.y)]])
+ else:
+ self._centers = np.array([[self._weighted_mean(self.x),
+ self._weighted_mean(self.y),
+ self._weighted_mean(self.z)]])
+ else:
+ self._centers = np.empty((self.npatch,2 if self.z is None else 3))
+ for p in range(self.npatch):
+ indx = np.where(self.patch == p)[0]
+ if len(indx) == 0:
+ raise RuntimeError("Cannot find center for patch %s."%p +
+ " No items with this patch number")
+ if self.coords == 'flat':
+ self._centers[p] = [self._weighted_mean(self.x,indx),
+ self._weighted_mean(self.y,indx)]
+ else:
+ self._centers[p] = [self._weighted_mean(self.x,indx),
+ self._weighted_mean(self.y,indx),
+ self._weighted_mean(self.z,indx)]
+ if self.coords == 'spherical':
+ self._centers /= np.sqrt(np.sum(self._centers**2,axis=1))[:,np.newaxis]
+ return self._centers
+
+[docs] def write_patch_centers(self, file_name):
+ """Write the patch centers to a file.
+
+ The output file will include the following columns:
+
+ ======== =======================================================
+ Column Description
+ ======== =======================================================
+ patch patch number (0..npatch-1)
+ x mean x values
+ y mean y values
+ z mean z values (only for spherical or 3d coordinates)
+ ======== =======================================================
+
+ It will write a FITS file if the file name ends with '.fits', otherwise an ASCII file.
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ """
+ self.logger.info('Writing centers to %s',file_name)
+
+ centers = self.patch_centers
+ col_names = ['patch', 'x', 'y']
+ if self.coords != 'flat':
+ col_names.append('z')
+ columns = [np.arange(centers.shape[0])]
+ for i in range(centers.shape[1]):
+ columns.append(centers[:,i])
+
+ with make_writer(file_name, precision=16, logger=self.logger) as writer:
+ writer.write(col_names, columns)
+
+[docs] def read_patch_centers(self, file_name):
+ """Read patch centers from a file.
+
+ This function typically gets called automatically when setting patch_centers as a
+ string, being the file name. The patch centers are read from the file and returned.
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+
+ Returns:
+ The centers, as an array, which can be used to determine the patches.
+ """
+ self.logger.info('Reading centers from %s',file_name)
+
+ with make_reader(file_name, logger=self.logger) as reader:
+ data = reader.read_data()
+ if 'z' in data.dtype.names:
+ return np.column_stack((data['x'],data['y'],data['z']))
+ else:
+ return np.column_stack((data['x'],data['y']))
+
+[docs] def load(self):
+ """Load the data from a file, if it isn't yet loaded.
+
+ When a Catalog is read in from a file, it tries to delay the loading of the data from
+ disk until it is actually needed. This is especially important when running over a
+ set of patches, since you may not be able to fit all the patches in memory at once.
+
+ One does not normally need to call this method explicitly. It will run automatically
+ whenever the data is needed. However, if you want to directly control when the disk
+ access happens, you can use this function.
+ """
+ if not self.loaded:
+ self.logger.info("Reading input file %s",self.name)
+ self._read_file(self.file_name, self.reader, self._num, self._is_rand)
+ self._finish_input()
+
+[docs] def unload(self):
+ """Bring the Catalog back to an "unloaded" state, if possible.
+
+ When a Catalog is read in from a file, it tries to delay the loading of the data from
+ disk until it is actually needed. After loading, this method will return the Catalog
+ back to the unloaded state to recover the memory in the data arrays. If the Catalog is
+ needed again during further processing, it will re-load the data from disk at that time.
+
+ This will also call `clear_cache` to recover any memory from fields that have been
+ constructed as well.
+
+ If the Catalog was not read in from a file, then this function will only do the
+ `clear_cache` step.
+ """
+ if self.file_type is not None:
+ self._x = None
+ self._y = None
+ self._z = None
+ self._ra = None
+ self._dec = None
+ self._r = None
+ self._w = None
+ self._wpos = None
+ self._g1 = None
+ self._g2 = None
+ self._k = None
+ self._patch = None
+ if self._patches is not None:
+ for p in self._patches:
+ p.unload()
+ self.clear_cache()
+
+[docs] def get_patch_file_names(self, save_patch_dir):
+ """Get the names of the files to use for reading/writing patches in save_patch_dir
+ """
+ if self.file_name is not None:
+ base, ext = os.path.splitext(os.path.basename(self.file_name))
+ # Default to FITS file if we do not otherwise recognize the file type.
+ # It's not critical to match this, just a convenience for if the user
+ # wants to pre-create their own patch files.
+ if ext.lower() not in [".fits", ".fit", ".hdf5", ".hdf"]:
+ ext = ".fits"
+ names = [base + '_%03d%s'%(i, ext) for i in range(self.npatch)]
+ else:
+ names = ['patch%03d.fits'%i for i in range(self.npatch)]
+ return [os.path.join(save_patch_dir, n) for n in names]
+
+[docs] def write_patches(self, save_patch_dir=None):
+ """Write the patches to disk as separate files.
+
+ This can be used in conjunction with ``low_mem=True`` option of `get_patches` (and
+ implicitly by the various `process <GGCorrelation.process>` methods) to only keep
+ at most two patches in memory at a time.
+
+ Parameters:
+ save_patch_dir (str): The directory to write the patches to. [default: None, in which
+ case self.save_patch_dir will be used. If that is None, a
+ ValueError will be raised.]
+ """
+ if save_patch_dir is None:
+ save_patch_dir = self.save_patch_dir
+ if save_patch_dir is None:
+ raise ValueError("save_patch_dir is required here, since not given in constructor.")
+
+ file_names = self.get_patch_file_names(save_patch_dir)
+ for i, p, file_name in zip(range(self.npatch), self.patches, file_names):
+ self.logger.info('Writing patch %d to %s',i,file_name)
+ if p.ra is not None:
+ # Don't multiply and divide by the units on round trip.
+ p.ra_units = p.dec_units = 1
+ p.write(file_name)
+
+[docs] def read_patches(self, save_patch_dir=None):
+ """Read the patches from files on disk.
+
+ This function assumes that the patches were written using `write_patches`.
+ In particular, the file names are not arbitrary, but must match what TreeCorr uses
+ in that method.
+
+ .. note::
+
+ The patches that are read in will be in an "unloaded" state. They will load
+ as needed when some functionality requires it. So this is compatible with using
+ the ``low_mem`` option in various places.
+
+ Parameters:
+ save_patch_dir (str): The directory to read from. [default: None, in which
+ case self.save_patch_dir will be used. If that is None, a
+ ValueError will be raised.]
+ """
+ if save_patch_dir is None:
+ save_patch_dir = self.save_patch_dir
+ if save_patch_dir is None:
+ raise ValueError("save_patch_dir is required here, since not given in constructor.")
+
+ # Need to be careful here to not trigger a load by accident.
+ # This would be easier if we just checked e.g. if self.ra is not None, etc.
+ # But that would trigger an unnecessary load if we aren't loaded yet.
+ # So do all this with the underscore attributes.
+ kwargs = {}
+ if self._ra is not None or self.config.get('ra_col','0') != '0':
+ kwargs['ra_col'] = 'ra'
+ kwargs['dec_col'] = 'dec'
+ kwargs['ra_units'] = 'rad'
+ kwargs['dec_units'] = 'rad'
+ if self._r is not None or self.config.get('r_col','0') != '0':
+ kwargs['r_col'] = 'r'
+ else:
+ kwargs['x_col'] = 'x'
+ kwargs['y_col'] = 'y'
+ if self._z is not None or self.config.get('z_col','0') != '0':
+ kwargs['z_col'] = 'z'
+ if (self._w is not None and self._nontrivial_w) or self.config.get('w_col','0') != '0':
+ kwargs['w_col'] = 'w'
+ if self._wpos is not None or self.config.get('wpos_col','0') != '0':
+ kwargs['wpos_col'] = 'wpos'
+ if self._g1 is not None or self.config.get('g1_col','0') != '0':
+ kwargs['g1_col'] = 'g1'
+ if self._g2 is not None or self.config.get('g2_col','0') != '0':
+ kwargs['g2_col'] = 'g2'
+ if self._k is not None or self.config.get('k_col','0') != '0':
+ kwargs['k_col'] = 'k'
+
+ file_names = self.get_patch_file_names(save_patch_dir)
+ self._patches = []
+ # Check that the files exist, although we won't actually load them yet.
+ for i, file_name in zip(range(self.npatch), file_names):
+ if not os.path.isfile(file_name):
+ raise OSError("Patch file %s not found"%file_name)
+ self._patches = [Catalog(file_name=name, patch=i, **kwargs)
+ for i, name in enumerate(file_names)]
+ self.logger.info('Patches created from files %s .. %s',file_names[0],file_names[-1])
+
+[docs] @depr_pos_kwargs
+ def get_patches(self, *, low_mem=False):
+ """Return a list of Catalog instances each representing a single patch from this Catalog
+
+ After calling this function once, the patches may be repeatedly accessed by the
+ ``patches`` attribute, without triggering a rebuild of the patches. Furthermore,
+ if ``patches`` is accessed before calling this function, it will be called automatically
+ (with the default low_mem parameter).
+
+ Parameters:
+ low_mem (bool): Whether to try to leave the returned patch catalogs in an
+ "unloaded" state, wherein they will not load the data from a
+ file until they are used. This only works if the current catalog
+ was loaded from a file or the patches were saved (using
+ ``save_patch_dir``). (default: False)
+ """
+ # Early exit
+ if self._patches is not None:
+ return self._patches
+ if self.npatch == 1 or self._single_patch is not None:
+ self._patches = [self]
+ return self._patches
+
+ # See if we have patches already written to disk. If so, use them.
+ if self.save_patch_dir is not None:
+ try:
+ self.read_patches()
+ except OSError:
+ # No problem. We'll make them and write them out below.
+ pass
+ else:
+ return self._patches
+
+ if low_mem and self.file_name is not None:
+ # This is a litle tricky, since we don't want to trigger a load if the catalog
+ # isn't loaded yet. So try to get the patches from centers or single_patch first.
+ if self._centers is not None:
+ patch_set = range(len(self._centers))
+ else:
+ # This triggers a load of the current catalog, but no choice here.
+ patch_set = sorted(set(self.patch))
+ centers = self._centers if self._patch is None else None
+ self._patches = [Catalog(config=self.config, file_name=self.file_name,
+ patch=i, npatch=self.npatch, patch_centers=centers)
+ for i in patch_set]
+ else:
+ patch_set = sorted(set(self.patch))
+ if len(patch_set) != self.npatch:
+ self.logger.error("WARNING: Some patch numbers do not contain any objects!")
+ missing = set(range(self.npatch)) - set(patch_set)
+ self.logger.warning("The following patch numbers have no objects: %s",missing)
+ self.logger.warning("This may be a problem depending on your use case.")
+ self._patches = []
+ for i in patch_set:
+ indx = np.where(self.patch == i)[0]
+ x=self.x[indx] if self.x is not None else None
+ y=self.y[indx] if self.y is not None else None
+ z=self.z[indx] if self.z is not None else None
+ ra=self.ra[indx] if self.ra is not None else None
+ dec=self.dec[indx] if self.dec is not None else None
+ r=self.r[indx] if self.r is not None else None
+ w=self.w[indx] if self.nontrivial_w else None
+ wpos=self.wpos[indx] if self.wpos is not None else None
+ g1=self.g1[indx] if self.g1 is not None else None
+ g2=self.g2[indx] if self.g2 is not None else None
+ k=self.k[indx] if self.k is not None else None
+ check_wpos = self._wpos if self._wpos is not None else self._w
+ kwargs = dict(keep_zero_weight=np.any(check_wpos==0))
+ if self.ra is not None:
+ kwargs['ra_units'] = 'rad'
+ kwargs['dec_units'] = 'rad'
+ kwargs['allow_xyz'] = True
+ p = Catalog(x=x, y=y, z=z, ra=ra, dec=dec, r=r, w=w, wpos=wpos,
+ g1=g1, g2=g2, k=k, patch=i, npatch=self.npatch, **kwargs)
+ self._patches.append(p)
+
+ # Write the patches to files if requested.
+ if self.save_patch_dir is not None:
+ self.write_patches()
+ if low_mem:
+ # If low_mem, replace _patches with a version the reads from these files.
+ # This will typically be a lot faster for when the load does happen.
+ self.read_patches()
+
+ return self._patches
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, cat_precision=None):
+ """Write the catalog to a file.
+
+ The position columns are output using the same units as were used when building the
+ Catalog. If you want to use a different unit, you can set the catalog's units directly
+ before writing. e.g.:
+
+ >>> cat = treecorr.Catalog('cat.dat', ra=ra, dec=dec,
+ ra_units='hours', dec_units='degrees')
+ >>> cat.ra_units = coord.degrees
+ >>> cat.write('new_cat.dat')
+
+ The output file will include some of the following columns (those for which the
+ corresponding attribute is not None):
+
+ ======== =======================================================
+ Column Description
+ ======== =======================================================
+ ra self.ra if not None
+ dec self.dec if not None
+ r self.r if not None
+ x self.x if not None
+ y self.y if not None
+ z self.z if not None
+ w self.w if not None and self.nontrivial_w
+ wpos self.wpos if not None
+ g1 self.g1 if not None
+ g2 self.g2 if not None
+ k self.k if not None
+ patch self.patch if not None
+ meanR The mean value <R> of pairs that fell into each bin.
+ meanlogR The mean value <logR> of pairs that fell into each bin.
+ ======== =======================================================
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default:
+ determine the type automatically from the extension of file_name.)
+ cat_precision (int): For ASCII output catalogs, the desired precision. (default: 16;
+ this value can also be given in the Catalog constructor in the
+ config dict.)
+ Returns:
+ The column names that were written to the file as a list.
+ """
+ self.logger.info('Writing catalog to %s',file_name)
+
+ col_names = []
+ columns = []
+ if self.ra is not None:
+ col_names.append('ra')
+ columns.append(self.ra / self.ra_units)
+ col_names.append('dec')
+ columns.append(self.dec / self.dec_units)
+ if self.r is not None:
+ col_names.append('r')
+ columns.append(self.r)
+ else:
+ col_names.append('x')
+ columns.append(self.x / self.x_units)
+ col_names.append('y')
+ columns.append(self.y / self.y_units)
+ if self.z is not None:
+ col_names.append('z')
+ columns.append(self.z)
+ if self.nontrivial_w:
+ col_names.append('w')
+ columns.append(self.w)
+ if self.wpos is not None:
+ col_names.append('wpos')
+ columns.append(self.wpos)
+ if self.g1 is not None:
+ col_names.append('g1')
+ columns.append(self.g1)
+ if self.g2 is not None:
+ col_names.append('g2')
+ columns.append(self.g2)
+ if self.k is not None:
+ col_names.append('k')
+ columns.append(self.k)
+ if self._patch is not None:
+ col_names.append('patch')
+ columns.append(self.patch)
+
+ if cat_precision is None:
+ cat_precision = get(self.config,'cat_precision',int,16)
+
+ writer = make_writer(file_name, precision=cat_precision, file_type=file_type,
+ logger=self.logger)
+ with writer:
+ writer.write(col_names, columns)
+ return col_names
+
+
+
+ def __getstate__(self):
+ d = self.__dict__.copy()
+ d.pop('logger',None) # Oh well. This is just lost in the copy. Can't be pickled.
+ d.pop('_field',None)
+ d.pop('_nfields',None)
+ d.pop('_kfields',None)
+ d.pop('_gfields',None)
+ d.pop('_nsimplefields',None)
+ d.pop('_ksimplefields',None)
+ d.pop('_gsimplefields',None)
+ return d
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ self.logger = setup_logger(get(self.config,'verbose',int,1),
+ self.config.get('log_file',None))
+ self._field = lambda : None
+
+ def __repr__(self):
+ s = 'treecorr.Catalog('
+ if self.loaded:
+ if self.x is not None and self.ra is None: s += 'x='+repr(self.x)+','
+ if self.y is not None and self.ra is None: s += 'y='+repr(self.y)+','
+ if self.z is not None and self.ra is None: s += 'z='+repr(self.z)+','
+ if self.ra is not None: s += 'ra='+repr(self.ra)+",ra_units='rad',"
+ if self.dec is not None: s += 'dec='+repr(self.dec)+",dec_units='rad',"
+ if self.r is not None: s += 'r='+repr(self.r)+','
+ if self.nontrivial_w: s += 'w='+repr(self.w)+','
+ if self.wpos is not None: s += 'wpos='+repr(self.wpos)+','
+ if self.g1 is not None: s += 'g1='+repr(self.g1)+','
+ if self.g2 is not None: s += 'g2='+repr(self.g2)+','
+ if self.k is not None: s += 'k='+repr(self.k)+','
+ if self.patch is not None: s += 'patch='+repr(self.patch)+','
+ wpos = self._wpos if self._wpos is not None else self._w
+ if np.any(wpos == 0): s += 'keep_zero_weight=True,'
+ # remove the last ','
+ s = s[:-1] + ')'
+ else:
+ # Catalog isn't loaded yet. Use file_name info here instead.
+ s += 'file_name='+repr(self.file_name)+','
+ s += 'config ='+repr(self.config)
+ s += ')'
+ return s
+
+ def __eq__(self, other):
+ return (isinstance(other, Catalog) and
+ np.array_equal(self.x, other.x) and
+ np.array_equal(self.y, other.y) and
+ np.array_equal(self.z, other.z) and
+ np.array_equal(self.ra, other.ra) and
+ np.array_equal(self.dec, other.dec) and
+ np.array_equal(self.r, other.r) and
+ np.array_equal(self.w, other.w) and
+ np.array_equal(self.wpos, other.wpos) and
+ np.array_equal(self.g1, other.g1) and
+ np.array_equal(self.g2, other.g2) and
+ np.array_equal(self.k, other.k) and
+ np.array_equal(self.patch, other.patch))
+
+
+[docs]@depr_pos_kwargs
+def read_catalogs(config, key=None, list_key=None, *, num=0, logger=None, is_rand=None):
+ """Read in a list of catalogs for the given key.
+
+ key should be the file_name parameter or similar key word.
+ list_key should be be corresponging file_list parameter, if appropriate.
+ At least one of key or list_key must be provided. If both are provided, then only
+ one of these should be in the config dict.
+
+ num indicates which key to use if any of the fields like x_col, flip_g1, etc. are lists.
+ The default is 0, which means to use the first item in the list if they are lists.
+
+ If the config dict specifies that patches be used, the returned list of Catalogs will be
+ a concatenation of the patches for each of the specified names.
+
+ Parameters:
+ config (dict): The configuration dict to use for the appropriate parameters
+ key (str): Which key name to use for the file names. e.g. 'file_name' (default: None)
+ list_key (str): Which key name to use for the name of a list file. e.g. 'file_list'.
+ Either key or list_key is required. (default: None)
+ num (int): Which number catalog does this correspond to. e.g. file_name should use
+ num=0, file_name2 should use num=1. (default: 0)
+ logger: If desired, a Logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+ is_rand (bool): If this is a random file, then setting is_rand to True will let them
+ skip k_col, g1_col, and g2_col if they were set for the main catalog.
+ (default: False)
+
+ Returns:
+ A list of Catalogs or None if no catalogs are specified.
+ """
+ if logger is None:
+ logger = setup_logger(get(config,'verbose',int,1), config.get('log_file',None))
+
+ if key is None and list_key is None:
+ raise TypeError("Must provide either key or list_key")
+ if key is not None and key in config:
+ if list_key is not None and list_key in config:
+ raise TypeError("Cannot provide both key and list_key")
+ file_names = config[key]
+ elif list_key is not None and list_key in config:
+ list_file = config[list_key]
+ with open(list_file,'r') as fin:
+ file_names = [ f.strip() for f in fin ]
+ else:
+ # If this key was required (i.e. file_name) then let the caller check this.
+ return []
+ if is_rand is None:
+ if key is not None:
+ is_rand = 'rand' in key
+ else:
+ is_rand = 'rand' in list_key
+ if not isinstance(file_names,list):
+ file_names = file_names.split()
+ ret = []
+ for file_name in file_names:
+ ret += Catalog(file_name, config, num=num, logger=logger, is_rand=is_rand).get_patches()
+ return ret
+
+
+[docs]@depr_pos_kwargs
+def calculateVarG(cat_list, *, low_mem=False):
+ """Calculate the overall shear variance from a list of catalogs.
+
+ The catalogs are assumed to be equivalent, so this is just the average shear
+ variance (per component) weighted by the number of objects in each catalog.
+
+ Parameters:
+ cat_list: A Catalog or a list of Catalogs for which to calculate the shear variance.
+ low_mem: Whether to try to conserve memory when cat is a list by unloading each
+ catalog after getting its individual varg. [default: False]
+
+ Returns:
+ The shear variance per component, aka shape noise.
+ """
+ if isinstance(cat_list, Catalog):
+ return cat_list.varg
+ elif len(cat_list) == 1:
+ return cat_list[0].varg
+ else:
+ varg = 0
+ sumw = 0
+ for cat in cat_list:
+ varg += cat.varg * cat.sumw
+ sumw += cat.sumw
+ if low_mem:
+ cat.unload()
+ return varg / sumw
+
+[docs]@depr_pos_kwargs
+def calculateVarK(cat_list, *, low_mem=False):
+ """Calculate the overall kappa variance from a list of catalogs.
+
+ The catalogs are assumed to be equivalent, so this is just the average kappa
+ variance weighted by the number of objects in each catalog.
+
+ Parameters:
+ cat_list: A Catalog or a list of Catalogs for which to calculate the kappa variance.
+ low_mem: Whether to try to conserve memory when cat is a list by unloading each
+ catalog after getting its individual vark. [default: False]
+
+ Returns:
+ The kappa variance
+ """
+ if isinstance(cat_list, Catalog):
+ return cat_list.vark
+ elif len(cat_list) == 1:
+ return cat_list[0].vark
+ else:
+ # Unlike for g, we allow k to have a non-zero mean around which we take the
+ # variance. When building up from multiple catalogs, we need to calculate the
+ # overall mean and get the variance around that. So this is a little more complicated.
+ # In practice, it probably doesn't matter at all for real data sets, but some of the
+ # unit tests have small enough N that this matters.
+ vark = 0
+ meank = 0
+ meank2 = 0
+ sumw = 0
+ sumw2 = 0
+ for cat in cat_list:
+ vark += cat.vark * cat.sumw + cat._meank * cat.sumw2 * (2*cat._meank2 - cat._meank)
+ meank += cat._meank * cat.sumw
+ meank2 += cat._meank2 * cat.sumw2
+ sumw += cat.sumw
+ sumw2 += cat.sumw2
+ if low_mem:
+ cat.unload()
+ meank /= sumw
+ meank2 /= sumw2
+ vark = (vark - meank * sumw2 * (2*meank2 - meank)) / sumw
+ return vark
+
+[docs]def isGColRequired(config, num):
+ """A quick helper function that checks whether we need to bother reading the g1,g2 columns.
+
+ It checks the config dict for the output file names gg_file_name, ng_file_name (only if
+ num == 1), etc. If the output files indicate that we don't need the g1/g2 columns, then
+ we don't need to raise an error if the g1_col or g2_col is invalid.
+
+ This makes it easier to specify columns. e.g. for an NG correlation function, the
+ first catalog does not need to have the g1,g2 columns, and typically wouldn't. So
+ if you specify g1_col=5, g2_col=6, say, and the first catalog does not have these columns,
+ you would normally get an error.
+
+ But instead, we check that the calculation is going to be NG from the presence of an
+ ng_file_name parameter, and we let the would-be error pass.
+
+ Parameters:
+ config (dict): The configuration file to check.
+ num (int): Which number catalog are we working on.
+
+ Returns:
+ True if some output file requires this catalog to have valid g1/g2 columns,
+ False if not.
+
+ """
+ return config and ( 'gg_file_name' in config
+ or 'm2_file_name' in config
+ or (num==1 and 'norm_file_name' in config)
+ or (num==1 and 'ng_file_name' in config)
+ or (num==1 and 'nm_file_name' in config)
+ or (num==1 and 'kg_file_name' in config) )
+
+
+[docs]def isKColRequired(config, num):
+ """A quick helper function that checks whether we need to bother reading the k column.
+
+ The logic here is the same as for `isGColRequired`, but we check for output files that require
+ the k column rather than g1,g2.
+
+ Parameters:
+ config (dict): The configuration file to check.
+ num (int): Which number catalog are we working on.
+
+ Returns:
+ True if some output file requires this catalog to have valid g1/g2 columns,
+ False if not.
+
+ """
+ return config and ( 'kk_file_name' in config
+ or (num==0 and 'kg_file_name' in config)
+ or (num==1 and 'nk_file_name' in config) )
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: config
+"""
+
+import sys
+import coord
+import numpy as np
+import warnings
+import logging
+import os
+
+
+[docs]def parse_variable(config, v):
+ """Parse a configuration variable from a string that should look like 'key = value'
+ and write that value to config[key].
+
+ :param config: The configuration dict to wich to write the key,value pair
+ :param v: A string of the form 'key = value'
+ """
+ if '=' not in v:
+ raise ValueError('Improper variable specificationi: %s. Use syntax: key = value.'%v)
+ key, value = v.split('=',1)
+ key = key.strip()
+ # Cut off any trailing comment
+ if '#' in value:
+ value = value.split('#')[0]
+ value = value.strip()
+ if value[0] in ['{','[','(']:
+ if value[-1] not in ['}',']',')']:
+ raise ValueError('List symbol %s not properly matched'%value[0])
+ values = value[1:-1].split(',')
+ values = [ vv.strip() for vv in values ]
+ else:
+ values = value.split() # on whitespace
+ if len(values) == 1:
+ config[key] = values[0]
+ else:
+ config[key] = values
+
+
+[docs]def parse_bool(value):
+ """Parse a value as a boolean.
+
+ Valid string values for True are: 'true', 'yes', 't', 'y'
+ Valid string values for False are: 'false', 'no', 'f', 'n', 'none'
+ Capitalization is ignored.
+
+ If value is a number, it is converted to a bool in the usual way.
+
+ :param value: The value to parse.
+
+ :returns: The value converted to a bool.
+ """
+ if isinstance(value,str):
+ if value.strip().upper() in [ 'TRUE', 'YES', 'T', 'Y' ]:
+ return True
+ elif value.strip().upper() in [ 'FALSE', 'NO', 'F', 'N', 'NONE' ]:
+ return False
+ else:
+ try:
+ bool(int(value))
+ except Exception:
+ raise ValueError("Unable to parse %s as a bool."%value)
+ else:
+ return int(value)
+ elif isinstance(value,(bool, np.bool_)):
+ return value
+ elif isinstance(value,int):
+ # Note: integers aren't converted to bool, since brute distinguishes 1 vs 2 vs True.
+ return value
+ else:
+ raise ValueError("Unable to parse %s as a bool."%value)
+
+[docs]def parse_unit(value):
+ """Parse the input value as a string that should be one of the valid angle units in
+ coord.AngleUnit.valid_names.
+
+ The value is allowed to merely start with one of the unit names. So 'deg', 'degree',
+ 'degrees' all convert to 'deg' which is the name in coord.AngleUnit.valid_names.
+ The return value in this case would be coord.AngleUnit.from_name('deg').value,
+ which has the value pi/180.
+
+ :param value: The unit as a string value to parse.
+
+ :returns: The given unit in radians.
+ """
+ for unit in coord.AngleUnit.valid_names:
+ if value.startswith(unit):
+ return coord.AngleUnit.from_name(value).value
+ raise ValueError("Unable to parse %s as an angle unit"%value)
+
+
+[docs]def read_config(file_name, file_type='auto'):
+ """Read a configuration dict from a file.
+
+ :param file_name: The file name from which the configuration dict should be read.
+ :param file_type: The type of config file. Options are 'auto', 'yaml', 'json', 'params'.
+ (default: 'auto', which tries to determine the type from the extension)
+
+ :returns: A config dict built from the configuration file.
+ """
+ if file_type == 'auto':
+ if file_name.endswith('.yaml'):
+ file_type = 'yaml'
+ elif file_name.endswith('.json'):
+ file_type = 'json'
+ elif file_name.endswith('.params'):
+ file_type = 'params'
+ else:
+ raise ValueError("Unable to determine the type of config file from the extension")
+ if file_type == 'yaml':
+ return _read_yaml_file(file_name)
+ elif file_type == 'json':
+ return _read_json_file(file_name)
+ elif file_type == 'params':
+ return _read_params_file(file_name)
+ else:
+ raise ValueError("Invalid file_type %s"%file_type)
+
+def _read_yaml_file(file_name):
+ import yaml
+ with open(file_name) as fin:
+ config = yaml.safe_load(fin.read())
+ return config
+
+def _read_json_file(file_name):
+ import json
+ with open(file_name) as fin:
+ config = json.load(fin)
+ return config
+
+def _read_params_file(file_name):
+ config = dict()
+ with open(file_name) as fin:
+ for v in fin:
+ v = v.strip()
+ if len(v) == 0 or v[0] == '#':
+ pass
+ elif v[0] == '+':
+ include_file_name = v[1:]
+ config1 = read_config(include_file_name)
+ config.update(config1)
+ else:
+ parse_variable(config,v)
+ return config
+
+
+[docs]def setup_logger(verbose, log_file=None):
+ """Parse the integer verbosity level from the command line args into a logging_level string
+
+ :param verbose: An integer indicating what verbosity level to use.
+ :param log_file: If given, a file name to which to write the logging output.
+ If omitted or None, then output to stdout.
+
+ :returns: The logging.Logger object to use.
+ """
+ logging_levels = { 0: logging.CRITICAL,
+ 1: logging.WARNING,
+ 2: logging.INFO,
+ 3: logging.DEBUG }
+ logging_level = logging_levels[int(verbose)]
+
+ # Setup logging to go to sys.stdout or (if requested) to an output file
+ if log_file is None:
+ name = 'treecorr'
+ else:
+ name = 'treecorr_' + log_file
+ logger = logging.getLogger(name)
+
+ if len(logger.handlers) == 0: # only add handler once!
+ if log_file is None:
+ handle = logging.StreamHandler(stream=sys.stdout)
+ else:
+ handle = logging.FileHandler(log_file)
+ formatter = logging.Formatter('%(message)s') # Simple text output
+ handle.setFormatter(formatter)
+ logger.addHandler(handle)
+ logger.setLevel(logging_level)
+ return logger
+
+
+[docs]def parse(value, value_type, name):
+ """Parse the input value as the given type.
+
+ :param value: The value to parse.
+ :param value_type: The type expected for this.
+ :param name: The name of this value. Only used for error reporting.
+
+ :returns: value
+ """
+ try:
+ if value_type is bool:
+ return parse_bool(value)
+ elif value is None:
+ return None
+ else:
+ return value_type(value)
+ except ValueError:
+ raise ValueError("Could not parse {}={} as type {}".format(name, value, value_type))
+
+
+[docs]def check_config(config, params, aliases=None, logger=None):
+ """Check (and update) a config dict to conform to the given parameter rules.
+ The params dict has an entry for each valid config parameter whose value is a tuple
+ with the following items:
+
+ - type
+ - can be a list?
+ - default value
+ - valid values
+ - description (Multiple entries here are allowed for longer strings)
+
+ The file corr2.py has a list of parameters for the corr2 program.
+
+ :param config: The config dict to check.
+ :param params: A dict of valid parameters with information about each one.
+ :param aliases: A dict of deprecated parameters that are still aliases for new names.
+ (default: None)
+ :param logger: If desired, a logger object for logging any warnings here. (default: None)
+
+ :returns: The updated config dict.
+ """
+ config = config.copy()
+ for key in list(config.keys()):
+ # Check if this is a deprecated alias
+ if aliases and key in aliases:
+ if logger:
+ logger.warning("The parameter %s is deprecated. You should use %s instead."%(
+ key, aliases[key]))
+ else:
+ warnings.warn("The parameter %s is deprecated. You should use %s instead."%(
+ key, aliases[key]), FutureWarning)
+ new_key = aliases[key]
+ config[new_key] = config[key]
+ del config[key]
+ key = new_key
+
+ # Check that this is a valid key
+ if key not in params:
+ raise TypeError("Invalid parameter %s."%key)
+
+ value_type, may_be_list, default_value, valid_values = params[key][:4]
+
+ # Get the value
+ if may_be_list and isinstance(config[key], list):
+ value = [parse(v, value_type, key) for v in config[key] ]
+ else:
+ value = parse(config[key], value_type, key)
+ if value is None:
+ continue
+
+ # If limited allowed value, check that this is one of them.
+ if valid_values is not None and value is not None:
+ if value_type is str:
+ matches = [ v for v in valid_values if value == v ]
+ if len(matches) == 0:
+ # Allow the string to be longer.
+ # e.g. degrees is valid if 'deg' is in valid_values.
+ matches = [v for v in valid_values if isinstance(v,str) and value.startswith(v)]
+ if len(matches) != 1:
+ raise ValueError("Parameter %s has invalid value %s. Valid values are %s."%(
+ key, config[key], str(valid_values)))
+ value = matches[0]
+ else:
+ if value not in valid_values:
+ raise ValueError("Parameter %s has invalid value %s. Valid values are %s."%(
+ key, config[key], str(valid_values)))
+
+ # Write it back to the dict with the right type
+ config[key] = value
+
+ # Write the defaults for other parameters to simplify the syntax of getting the values
+ for key in params:
+ if key in config:
+ continue
+ value_type, may_be_list, default_value, valid_values = params[key][:4]
+ if default_value is not None:
+ config[key] = default_value
+
+ return config
+
+
+[docs]def print_params(params):
+ """Print the information about the valid parameters, given by the given params dict.
+ See check_config for the structure of the params dict.
+
+ :param params: A dict of valid parameters with information about each one.
+ """
+ max_len = max(len(key) for key in params)
+ for key in params:
+ value_type, may_be_list, default_value, valid_values = params[key][:4]
+ description = params[key][4:]
+ print(("{0:<"+str(max_len)+"} {1}").format(key,description[0]))
+ for d in description[1:]:
+ print(" {0}".format(d))
+
+ # str(value_type) looks like "<type 'float'>"
+ # value_type.__name__ looks like 'float'
+ value_type_str = value_type.__name__
+ if may_be_list:
+ print(" Type must be {0} or a list of {0}.".format(value_type_str))
+ else:
+ print(" Type must be {0}.".format(value_type_str))
+
+ if valid_values is not None:
+ print(" Valid values are {0!s}".format(valid_values))
+ if default_value is not None:
+ print(" Default value is {0!s}".format(default_value))
+ print()
+
+
+[docs]def convert(value, value_type, key):
+ """Convert the given value to the given type.
+
+ The ``key`` helps determine what kind of conversion should be performed.
+ Specifically if 'unit' is in the ``key`` value, then a unit conversion is done.
+ Otherwise, it just parses the ``value`` according to the ``value_type``.
+
+ :param value: The input value to be converted. Usually a string.
+ :param value_type: The type to convert to.
+ :param key: The key for this value. Only used to see if it includes 'unit'.
+
+ :returns: The converted value.
+ """
+ if value is None:
+ return None
+ elif 'unit' in key:
+ return parse_unit(value)
+ elif value_type == bool:
+ return parse_bool(value)
+ else:
+ return value_type(value)
+
+[docs]def get_from_list(config, key, num, value_type=str, default=None):
+ """A helper function to get a key from config that is allowed to be a list
+
+ Some of the config values are allowed to be lists of values, in which case we take the
+ ``num`` item from the list. If they are not a list, then the given value is used for
+ all values of ``num``.
+
+ :param config: The configuration dict from which to get the key value.
+ :param key: What key to get from config.
+ :param num: Which number element to use if the item is a list.
+ :param value_type: What type should the value be converted to. (default: str)
+ :param default: What value should be used if the key is not in the config dict,
+ or the value corresponding to the key is None.
+ (default: None)
+
+ :returns: The specified value, converted as needed.
+ """
+ values = config.get(key, None)
+ if isinstance(values, list):
+ try:
+ value = values[num]
+ except IndexError:
+ raise IndexError("num=%d is out of range of list for %s"%(num,key))
+
+ if value is not None:
+ return convert(value, value_type, key)
+ elif default is not None:
+ return convert(default, value_type, key)
+ elif values is not None:
+ return convert(values, value_type, key)
+ elif default is not None:
+ return convert(default, value_type, key)
+
+
+[docs]def get(config, key, value_type=str, default=None):
+ """A helper function to get a key from config converting to a particular type
+
+ :param config: The configuration dict from which to get the key value.
+ :param key: Which key to get from config.
+ :param value_type: Which type should the value be converted to. (default: str)
+ :param default: What value should be used if the key is not in the config dict,
+ or the value corresponding to the key is None.
+ (default: None)
+
+ :returns: The specified value, converted as needed.
+ """
+ value = config.get(key, default)
+ if value is not None:
+ return convert(value, value_type, key)
+ elif default is not None:
+ return convert(default, value_type, key)
+
+[docs]def merge_config(config, kwargs, valid_params, aliases=None):
+ """Merge in the values from kwargs into config.
+
+ If either of these is None, then the other one is returned.
+ If they are both dicts, then the values in kwargs take precedence over ones in config
+ if there are any keys that are in both. Also, the kwargs dict will be modified in this case.
+
+ :param config: The root config (will not be modified)
+ :param kwargs: A second dict with more or updated values
+ :param valid_params: A dict of valid parameters that are allowed for this usage.
+ The config dict is allowed to have extra items, but kwargs is not.
+ :param aliases: An optional dict of aliases. (default: None)
+
+ :returns: The merged dict, including only items that are in valid_params.
+ """
+ if kwargs is None:
+ kwargs = {}
+ if config:
+ for key, value in config.items():
+ if key in valid_params and key not in kwargs:
+ kwargs[key] = value
+ return check_config(kwargs, valid_params, aliases)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: corr2
+"""
+
+from .catalog import Catalog, read_catalogs
+from .binnedcorr2 import BinnedCorr2
+from .config import setup_logger, check_config, print_params, get
+from .util import set_omp_threads
+from .nncorrelation import NNCorrelation
+from .ngcorrelation import NGCorrelation
+from .nkcorrelation import NKCorrelation
+from .kkcorrelation import KKCorrelation
+from .kgcorrelation import KGCorrelation
+from .ggcorrelation import GGCorrelation
+
+# Dict describing the valid parameters, what types they are, and a description:
+# Each value is a tuple with the following elements:
+# type
+# may_be_list
+# default value
+# list of valid values
+# description
+corr2_valid_params = {
+
+ # Parameters about the input catalogs
+
+ 'file_name' : (str, True, None, None,
+ 'The file(s) with the galaxy data.'),
+ 'file_name2' : (str, True, None, None,
+ 'The file(s) to use for the second field for a cross-correlation.'),
+ 'rand_file_name' : (str, True, None, None,
+ 'For NN correlations, a list of random files.'),
+ 'rand_file_name2' : (str, True, None, None,
+ 'The randoms for the second field for a cross-correlation.'),
+ 'file_list' : (str, False, None, None,
+ 'A text file with file names in lieu of file_name.'),
+ 'file_list2' : (str, False, None, None,
+ 'A text file with file names in lieu of file_name2.'),
+ 'rand_file_list' : (str, False, None, None,
+ 'A text file with file names in lieu of rand_file_name.'),
+ 'rand_file_list2' : (str, False, None, None,
+ 'A text file with file names in lieu of rand_file_name2.'),
+
+ # Parameters about the output file(s)
+
+ 'nn_file_name' : (str, False, None, None,
+ 'The output filename for point-point correlation function.'),
+ 'nn_statistic' : (str, False, 'compensated', ['compensated','simple'],
+ 'Which statistic to use for omega as the estimator fo the NN correlation function. '),
+ 'ng_file_name' : (str, False, None, None,
+ 'The output filename for point-shear correlation function.'),
+ 'ng_statistic' : (str, False, None, ['compensated', 'simple'],
+ 'Which statistic to use for the mean shear estimator of the NG correlation function. ',
+ 'The default is compensated if rand_files is given, otherwise simple'),
+ 'gg_file_name' : (str, False, None, None,
+ 'The output filename for shear-shear correlation function.'),
+ 'nk_file_name' : (str, False, None, None,
+ 'The output filename for point-kappa correlation function.'),
+ 'nk_statistic' : (str, False, None, ['compensated', 'simple'],
+ 'Which statistic to use for the mean kappa estimator of the NK correlation function. ',
+ 'The default is compensated if rand_files is given, otherwise simple'),
+ 'kk_file_name' : (str, False, None, None,
+ 'The output filename for kappa-kappa correlation function.'),
+ 'kg_file_name' : (str, False, None, None,
+ 'The output filename for kappa-shear correlation function.'),
+
+ # Derived output quantities
+
+ 'm2_file_name' : (str, False, None, None,
+ 'The output filename for the aperture mass statistics.'),
+ 'nm_file_name' : (str, False, None, None,
+ 'The output filename for <N Map> and related values.'),
+ 'norm_file_name' : (str, False, None, None,
+ 'The output filename for <N Map>^2/<N^2><Map^2> and related values.'),
+}
+
+# Add in the valid parameters for the relevant classes
+for c in [ Catalog, BinnedCorr2 ]:
+ corr2_valid_params.update(c._valid_params)
+
+corr2_aliases = {
+}
+
+[docs]def corr2(config, logger=None):
+ """Run the full two-point correlation function code based on the parameters in the
+ given config dict.
+
+ The function `print_corr2_params` will output information about the valid parameters
+ that are expected to be in the config dict.
+
+ Optionally a logger parameter maybe given, in which case it is used for logging.
+ If not given, the logging will be based on the verbose and log_file parameters.
+
+ :param config: The configuration dict which defines what to do.
+ :param logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+ """
+ # Setup logger based on config verbose value
+ if logger is None:
+ logger = setup_logger(config.get('verbose',1), config.get('log_file',None))
+
+ # Check that config doesn't have any extra parameters.
+ # (Such values are probably typos.)
+ # Also convert the given parameters to the correct type, etc.
+ config = check_config(config, corr2_valid_params, corr2_aliases, logger)
+
+ import pprint
+ logger.debug('Using configuration dict:\n%s',pprint.pformat(config))
+
+ if ('output_dots' not in config
+ and config.get('log_file',None) is None
+ and config['verbose'] >= 2):
+ config['output_dots'] = True
+
+ # Set the number of threads
+ num_threads = config.get('num_threads',None)
+ logger.debug('From config dict, num_threads = %s',num_threads)
+ set_omp_threads(num_threads, logger)
+
+ # Read in the input files. Each of these is a list.
+ cat1 = read_catalogs(config, 'file_name', 'file_list', num=0, logger=logger)
+ cat2 = read_catalogs(config, 'file_name2', 'file_list2', num=1, logger=logger)
+ rand1 = read_catalogs(config, 'rand_file_name', 'rand_file_list', num=0, logger=logger)
+ rand2 = read_catalogs(config, 'rand_file_name2', 'rand_file_list2', num=1, logger=logger)
+ if len(cat1) == 0:
+ raise TypeError("Either file_name or file_list is required")
+ if len(cat2) == 0: cat2 = None
+ if len(rand1) == 0: rand1 = None
+ if len(rand2) == 0: rand2 = None
+ if cat2 is None and rand2 is not None:
+ raise TypeError("rand_file_name2 is invalid without file_name2")
+ logger.info("Done creating input catalogs")
+
+ # Do GG correlation function if necessary
+ if 'gg_file_name' in config or 'm2_file_name' in config:
+ logger.warning("Performing GG calculations...")
+ gg = GGCorrelation(config, logger=logger)
+ gg.process(cat1,cat2)
+ logger.info("Done GG calculations.")
+ if 'gg_file_name' in config:
+ gg.write(config['gg_file_name'])
+ logger.warning("Wrote GG correlation to %s",config['gg_file_name'])
+ if 'm2_file_name' in config:
+ gg.writeMapSq(config['m2_file_name'], m2_uform=config['m2_uform'])
+ logger.warning("Wrote Mapsq values to %s",config['m2_file_name'])
+
+ # Do NG correlation function if necessary
+ if 'ng_file_name' in config or 'nm_file_name' in config or 'norm_file_name' in config:
+ if cat2 is None:
+ raise TypeError("file_name2 is required for ng correlation")
+ logger.warning("Performing NG calculations...")
+ ng = NGCorrelation(config, logger=logger)
+ ng.process(cat1,cat2)
+ logger.info("Done NG calculation.")
+
+ # The default ng_statistic is compensated _iff_ rand files are given.
+ rg = None
+ if rand1 is None:
+ if config.get('ng_statistic',None) == 'compensated':
+ raise TypeError("rand_files is required for ng_statistic = compensated")
+ elif config.get('ng_statistic','compensated') == 'compensated':
+ rg = NGCorrelation(config, logger=logger)
+ rg.process(rand1,cat2)
+ logger.info("Done RG calculation.")
+
+ if 'ng_file_name' in config:
+ ng.write(config['ng_file_name'], rg=rg)
+ logger.warning("Wrote NG correlation to %s",config['ng_file_name'])
+ if 'nm_file_name' in config:
+ ng.writeNMap(config['nm_file_name'], rg=rg, m2_uform=config['m2_uform'],
+ precision=config.get('precision',None))
+ logger.warning("Wrote NMap values to %s",config['nm_file_name'])
+
+ if 'norm_file_name' in config:
+ gg = GGCorrelation(config, logger=logger)
+ gg.process(cat2)
+ logger.info("Done GG calculation for norm")
+ dd = NNCorrelation(config, logger=logger)
+ dd.process(cat1)
+ logger.info("Done DD calculation for norm")
+ rr = NNCorrelation(config, logger=logger)
+ rr.process(rand1)
+ logger.info("Done RR calculation for norm")
+ if config['nn_statistic'] == 'compensated':
+ dr = NNCorrelation(config, logger=logger)
+ dr.process(cat1,rand1)
+ logger.info("Done DR calculation for norm")
+ else:
+ dr = None
+ ng.writeNorm(config['norm_file_name'],gg=gg,dd=dd,rr=rr,dr=dr,rg=rg,
+ m2_uform=config['m2_uform'], precision=config.get('precision',None))
+ logger.warning("Wrote Norm values to %s",config['norm_file_name'])
+
+ # Do NN correlation function if necessary
+ if 'nn_file_name' in config:
+ logger.warning("Performing DD calculations...")
+ dd = NNCorrelation(config, logger=logger)
+ dd.process(cat1,cat2)
+ logger.info("Done DD calculations.")
+
+ dr = None
+ rd = None
+ if rand1 is None:
+ logger.warning("No random catalogs given. Only doing npairs calculation.")
+ rr = None
+ elif cat2 is None:
+ logger.warning("Performing RR calculations...")
+ rr = NNCorrelation(config, logger=logger)
+ rr.process(rand1)
+ logger.info("Done RR calculations.")
+
+ if config['nn_statistic'] == 'compensated':
+ logger.warning("Performing DR calculations...")
+ dr = NNCorrelation(config, logger=logger)
+ dr.process(cat1,rand1)
+ logger.info("Done DR calculations.")
+ else:
+ if rand2 is None:
+ raise TypeError("rand_file_name2 is required when file_name2 is given")
+ logger.warning("Performing RR calculations...")
+ rr = NNCorrelation(config, logger=logger)
+ rr.process(rand1,rand2)
+ logger.info("Done RR calculations.")
+
+ if config['nn_statistic'] == 'compensated':
+ logger.warning("Performing DR calculations...")
+ dr = NNCorrelation(config, logger=logger)
+ dr.process(cat1,rand2)
+ logger.info("Done DR calculations.")
+ rd = NNCorrelation(config, logger=logger)
+ rd.process(rand1,cat2)
+ logger.info("Done RD calculations.")
+ dd.write(config['nn_file_name'], rr=rr, dr=dr, rd=rd)
+ logger.warning("Wrote NN correlation to %s",config['nn_file_name'])
+
+ # Do KK correlation function if necessary
+ if 'kk_file_name' in config:
+ logger.warning("Performing KK calculations...")
+ kk = KKCorrelation(config, logger=logger)
+ kk.process(cat1,cat2)
+ logger.info("Done KK calculations.")
+ kk.write(config['kk_file_name'])
+ logger.warning("Wrote KK correlation to %s",config['kk_file_name'])
+
+ # Do NG correlation function if necessary
+ if 'nk_file_name' in config:
+ if cat2 is None:
+ raise TypeError("file_name2 is required for nk correlation")
+ logger.warning("Performing NK calculations...")
+ nk = NKCorrelation(config, logger=logger)
+ nk.process(cat1,cat2)
+ logger.info("Done NK calculation.")
+
+ rk = None
+ if rand1 is None:
+ if config.get('nk_statistic',None) == 'compensated':
+ raise TypeError("rand_files is required for nk_statistic = compensated")
+ elif config.get('nk_statistic','compensated') == 'compensated':
+ rk = NKCorrelation(config, logger=logger)
+ rk.process(rand1,cat2)
+ logger.info("Done RK calculation.")
+
+ nk.write(config['nk_file_name'], rk=rk)
+ logger.warning("Wrote NK correlation to %s",config['nk_file_name'])
+
+ # Do KG correlation function if necessary
+ if 'kg_file_name' in config:
+ if cat2 is None:
+ raise TypeError("file_name2 is required for kg correlation")
+ logger.warning("Performing KG calculations...")
+ kg = KGCorrelation(config, logger=logger)
+ kg.process(cat1,cat2)
+ logger.info("Done KG calculation.")
+ kg.write(config['kg_file_name'])
+ logger.warning("Wrote KG correlation to %s",config['kg_file_name'])
+
+
+[docs]def print_corr2_params():
+ """Print information about the valid parameters that may be given to the `corr2` function.
+ """
+ print_params(corr2_valid_params)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: corr3
+"""
+
+from .catalog import Catalog, read_catalogs
+from .binnedcorr3 import BinnedCorr3
+from .config import setup_logger, check_config, print_params
+from .util import set_omp_threads
+from .nnncorrelation import NNNCorrelation
+from .kkkcorrelation import KKKCorrelation
+from .gggcorrelation import GGGCorrelation
+
+
+# Dict describing the valid parameters, what types they are, and a description:
+# Each value is a tuple with the following elements:
+# type
+# may_be_list
+# default value
+# list of valid values
+# description
+corr3_valid_params = {
+
+ # Parameters about the input catlogs
+
+ 'file_name' : (str, True, None, None,
+ 'The file(s) with the galaxy data.'),
+ 'rand_file_name' : (str, True, None, None,
+ 'For NNN correlations, a list of random files.'),
+ 'file_list' : (str, False, None, None,
+ 'A text file with file names in lieu of file_name.'),
+ 'rand_file_list' : (str, False, None, None,
+ 'A text file with file names in lieu of rand_file_name.'),
+
+ # Parameters about the output file(s)
+
+ 'nnn_file_name' : (str, False, None, None,
+ 'The output filename for point-point correlation function.'),
+ 'nnn_statistic' : (str, False, 'compensated', ['compensated','simple'],
+ 'Which statistic to use for omega as the estimator fo the NN correlation function. '),
+ 'kkk_file_name' : (str, False, None, None,
+ 'The output filename for kappa-kappa-kappa correlation function.'),
+ 'ggg_file_name' : (str, False, None, None,
+ 'The output filename for gamma-gamma-gamma correlation function.'),
+
+ # Derived output quantities
+
+ 'm3_file_name' : (str, False, None, None,
+ 'The output filename for the aperture mass skewness.'),
+}
+
+# Add in the valid parameters for the relevant classes
+for c in [ Catalog, BinnedCorr3 ]:
+ corr3_valid_params.update(c._valid_params)
+
+
+corr3_aliases = {
+}
+
+[docs]def corr3(config, logger=None):
+ """Run the full three-point correlation function code based on the parameters in the
+ given config dict.
+
+ The function `print_corr3_params` will output information about the valid parameters
+ that are expected to be in the config dict.
+
+ Optionally a logger parameter maybe given, in which case it is used for logging.
+ If not given, the logging will be based on the verbose and log_file parameters.
+
+ :param config: The configuration dict which defines what to do.
+ :param logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+ """
+ # Setup logger based on config verbose value
+ if logger is None:
+ logger = setup_logger(config.get('verbose',1), config.get('log_file',None))
+
+ # Check that config doesn't have any extra parameters.
+ # (Such values are probably typos.)
+ # Also convert the given parameters to the correct type, etc.
+ config = check_config(config, corr3_valid_params, corr3_aliases, logger)
+
+ import pprint
+ logger.debug('Using configuration dict:\n%s',pprint.pformat(config))
+
+ if ('output_dots' not in config
+ and config.get('log_file',None) is None
+ and config['verbose'] >= 2):
+ config['output_dots'] = True
+
+ # Set the number of threads
+ num_threads = config.get('num_threads',None)
+ logger.debug('From config dict, num_threads = %s',num_threads)
+ set_omp_threads(num_threads, logger)
+
+ # Read in the input files. Each of these is a list.
+ cat1 = read_catalogs(config, 'file_name', 'file_list', num=0, logger=logger)
+ # TODO: when giving file_name2, file_name3, should now do the real CrossCorrelation process.
+ rand1 = read_catalogs(config, 'rand_file_name', 'rand_file_list', num=0, logger=logger)
+ if len(cat1) == 0:
+ raise TypeError("Either file_name or file_list is required")
+ if len(rand1) == 0: rand1 = None
+ logger.info("Done creating input catalogs")
+
+ # Do GGG correlation function if necessary
+ if 'ggg_file_name' in config or 'm3_file_name' in config:
+ logger.warning("Performing GGG calculations...")
+ ggg = GGGCorrelation(config, logger=logger)
+ ggg.process(cat1)
+ logger.info("Done GGG calculations.")
+ if 'ggg_file_name' in config:
+ ggg.write(config['ggg_file_name'])
+ logger.warning("Wrote GGG correlation to %s",config['ggg_file_name'])
+ if 'm3_file_name' in config:
+ ggg.writeMap3(config['m3_file_name'])
+ logger.warning("Wrote Map3 values to %s",config['m3_file_name'])
+
+ # Do NNN correlation function if necessary
+ if 'nnn_file_name' in config:
+ logger.warning("Performing DDD calculations...")
+ ddd = NNNCorrelation(config, logger=logger)
+ ddd.process(cat1)
+ logger.info("Done DDD calculations.")
+
+ drr = None
+ rdd = None
+ if rand1 is None:
+ logger.warning("No random catalogs given. Only doing ntri calculation.")
+ rrr = None
+ else:
+ logger.warning("Performing RRR calculations...")
+ rrr = NNNCorrelation(config, logger=logger)
+ rrr.process(rand1)
+ logger.info("Done RRR calculations.")
+
+ if rrr is not None and config['nnn_statistic'] == 'compensated':
+ logger.warning("Performing DRR calculations...")
+ drr = NNNCorrelation(config, logger=logger)
+ drr.process(cat1,rand1)
+ logger.info("Done DRR calculations.")
+ logger.warning("Performing DDR calculations...")
+ rdd = NNNCorrelation(config, logger=logger)
+ rdd.process(rand1,cat1)
+ logger.info("Done DDR calculations.")
+ ddd.write(config['nnn_file_name'], rrr=rrr, drr=drr, rdd=rdd)
+ logger.warning("Wrote NNN correlation to %s",config['nnn_file_name'])
+
+ # Do KKK correlation function if necessary
+ if 'kkk_file_name' in config:
+ logger.warning("Performing KKK calculations...")
+ kkk = KKKCorrelation(config, logger=logger)
+ kkk.process(cat1)
+ logger.info("Done KKK calculations.")
+ kkk.write(config['kkk_file_name'])
+ logger.warning("Wrote KKK correlation to %s",config['kkk_file_name'])
+
+
+[docs]def print_corr3_params():
+ """Print information about the valid parameters that may be given to the `corr3` function.
+ """
+ print_params(corr3_valid_params)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: field
+"""
+
+import numpy as np
+import weakref
+
+from . import _lib, _ffi
+from .util import get_omp_threads, parse_xyzsep, coord_enum
+from .util import long_ptr as lp
+from .util import double_ptr as dp
+from .util import depr_pos_kwargs
+
+def _parse_split_method(split_method):
+ if split_method == 'middle': return 0
+ elif split_method == 'median': return 1
+ elif split_method == 'mean': return 2
+ else: return 3 # random
+
+
+[docs]class Field(object):
+ r"""A Field in TreeCorr is the object that stores the tree structure we use for efficient
+ calculation of the correlation functions.
+
+ The root "cell" in the tree has information about the whole field, including the total
+ number of points, the total weight, the mean position, the size (by which we mean the
+ maximum distance of any point from the mean position), and possibly more information depending
+ on which kind of field we have.
+
+ It also points to two sub-cells which each describe about half the points. These are commonly
+ referred to as "daughter cells". They in turn point to two more cells each, and so on until
+ we get to cells that are considered "small enough" according to the ``min_size`` parameter given
+ in the constructor. These lowest level cells are referred to as "leaves".
+
+ Technically, a Field doesn't have just one of these trees. To make parallel computation
+ more efficient, we actually skip the first few layers of the tree as described above and
+ store a list of root cells. The three parameters that determine how many of these there
+ will be are ``max_size``, ``min_top``, and ``max_top``:
+
+ - ``max_size`` sets the maximum size cell that we want to make sure we have in the trees,
+ so the root cells will be at least this large. The default is None, which means
+ we care about all sizes, so there may be only one root cell (but typically more
+ because of ``min_top``).
+ - ``min_top`` sets the minimum number of initial levels to skip. The default is either 3
+ or :math:`\log_2(N_{cpu})`, whichever is larger. This means there will be at least 8
+ (or :math:`N_{cpu}`) root cells (assuming ntot is at least this large of course).
+ - ``max_top`` sets the maximum number of initial levels to skip. The default is 10,
+ which means there could be up to 1024 root cells.
+
+ Finally, the ``split_method`` parameter sets how the points in a cell should be divided
+ when forming the two daughter cells. The split is always done according to whichever
+ dimension has the largest extent. E.g. if max(\|x - meanx\|) is larger than max(\|y - meany\|)
+ and (for 3d) max(\|z - meanz\|), then it will split according to the x values. But then
+ it may split in different ways according to ``split_method``. The allowed values are:
+
+ - 'mean' means to divide the points at the average (mean) value of x, y or z.
+ - 'median' means to divide the points at the median value of x, y, or z.
+ - 'middle' means to divide the points at midpoint between the minimum and maximum values.
+ - 'random' means to divide the points randomly somewhere between the 40th and 60th
+ percentile locations in the sorted list.
+
+ Field itself is an abstract base class for the specific types of field classes.
+ As such, it cannot be constructed directly. You should make one of the concrete subclasses:
+
+ - `NField` describes a field of objects to be counted only.
+ - `KField` describes a field of points sampling a scalar field (e.g. kappa in the
+ weak lensing context). In addition to the above values, cells keep track of
+ the mean kappa value in the given region.
+ - `GField` describes a field of points sampling a spinor field (e.g. gamma in the
+ weak lensing context). In addition to the above values, cells keep track of
+ the mean (complex) gamma value in the given region.
+ """
+ def __init__(self):
+ raise NotImplementedError("Field is an abstract base class. It cannot be instantiated.")
+
+ def _determine_top(self, min_top, max_top):
+ if min_top is None:
+ n_cpu = get_omp_threads()
+ # int.bit_length is a trick to avoid going through float.
+ # bit_length(n-1) == ceil(log2(n)), which is what we want.
+ min_top = max(3, int.bit_length(n_cpu-1))
+ else:
+ min_top = int(min_top)
+ max_top = int(max_top)
+ min_top = min(min_top, max_top) # If min_top > max_top favor max_top.
+ return min_top, max_top
+
+ @property
+ def nTopLevelNodes(self):
+ """The number of top-level nodes.
+ """
+ return _lib.FieldGetNTopLevel(self.data, self._d, self._coords)
+
+ @property
+ def cat(self):
+ """The catalog from which this field was constructed.
+
+ It is stored as a weakref, so if the Catalog has already been garbage collected, this
+ might be None.
+ """
+ # _cat is a weakref. This gets back to a Catalog object.
+ return self._cat()
+
+[docs] def count_near(self, *args, **kwargs):
+ """Count how many points are near a given coordinate.
+
+ Use the existing tree structure to count how many points are within some given separation
+ of a target coordinate.
+
+ There are several options for how to specify the reference coordinate, which depends
+ on the type of coordinate system this field implements.
+
+ 1. For flat 2-dimensional coordinates:
+
+ Parameters:
+ x (float): The x coordinate of the target location
+ y (float): The y coordinate of the target location
+ sep (float): The separation distance
+
+ 2. For 3-dimensional Cartesian coordinates:
+
+ Parameters:
+ x (float): The x coordinate of the target location
+ y (float): The y coordinate of the target location
+ z (float): The z coordinate of the target location
+ sep (float): The separation distance
+
+ 3. For spherical coordinates:
+
+ Parameters:
+ ra (float or Angle): The right ascension of the target location
+ dec (float or Angle): The declination of the target location
+ c (CelestialCorod): A ``coord.CelestialCoord`` object in lieu of (ra, dec)
+ sep (float or Angle): The separation distance
+ ra_units (str): The units of ra if given as a float
+ dec_units (str): The units of dec if given as a float
+ sep_units (str): The units of sep if given as a float
+
+ 4. For spherical coordinates with distances:
+
+ Parameters:
+ ra (float or Angle): The right ascension of the target location
+ dec (float or Angle): The declination of the target location
+ c (CelestialCorod): A ``coord.CelestialCoord`` object in lieu of (ra, dec)
+ r (float): The distance to the target location
+ sep (float): The separation distance
+ ra_units (str): The units of ra if given as a float
+ dec_units (str): The units of dec if given as a float
+
+ In all cases, for parameters that are angles (ra, dec, sep for 'spherical'), you may either
+ provide this quantity as a ``coord.Angle`` instance, or you may provide ra_units, dec_units
+ or sep_units respectively to specify which angular units are providing.
+
+ Finally, in cases where ra, dec are allowed, you may instead provide a
+ ``coord.CelestialCoord`` instance as the first argument to specify both RA and Dec.
+ """
+ if self.min_size == 0:
+ # If min_size = 0, then regular method is already exact.
+ x,y,z,sep = parse_xyzsep(args, kwargs, self._coords)
+ return self._count_near(x, y, z, sep)
+ else:
+ # Otherwise, we need to expand the radius a bit and then check the actual radii
+ # using the catalog values. This is already done in get_near, so just do that
+ # and take the len of the result.
+ return len(self.get_near(*args, **kwargs))
+
+ def _count_near(self, x, y, z, sep):
+ # If self.min_size > 0, these results may be approximate, since the tree will have
+ # grouped points within this separation together.
+ return _lib.FieldCountNear(self.data, x, y, z, sep, self._d, self._coords)
+
+[docs] def get_near(self, *args, **kwargs):
+ """Get the indices of points near a given coordinate.
+
+ Use the existing tree structure to find the points that are within some given separation
+ of a target coordinate.
+
+ There are several options for how to specify the reference coordinate, which depends
+ on the type of coordinate system this field implements.
+
+ 1. For flat 2-dimensional coordinates:
+
+ Parameters:
+ x (float): The x coordinate of the target location
+ y (float): The y coordinate of the target location
+ sep (float): The separation distance
+
+ 2. For 3-dimensional Cartesian coordinates:
+
+ Parameters:
+ x (float): The x coordinate of the target location
+ y (float): The y coordinate of the target location
+ z (float): The z coordinate of the target location
+ sep (float): The separation distance
+
+ 3. For spherical coordinates:
+
+ Parameters:
+ ra (float or Angle): The right ascension of the target location
+ dec (float or Angle): The declination of the target location
+ c (CelestialCorod): A ``coord.CelestialCoord`` object in lieu of (ra, dec)
+ sep (float or Angle): The separation distance
+ ra_units (str): The units of ra if given as a float
+ dec_units (str): The units of dec if given as a float
+ sep_units (str): The units of sep if given as a float
+
+ 4. For spherical coordinates with distances:
+
+ Parameters:
+ ra (float or Angle): The right ascension of the target location
+ dec (float or Angle): The declination of the target location
+ c (CelestialCorod): A ``coord.CelestialCoord`` object in lieu of (ra, dec)
+ r (float): The distance to the target location
+ sep (float): The separation distance
+ ra_units (str): The units of ra if given as a float
+ dec_units (str): The units of dec if given as a float
+
+ In all cases, for parameters that are angles (ra, dec, sep for 'spherical'), you may either
+ provide this quantity as a ``coord.Angle`` instance, or you may provide ra_units, dec_units
+ or sep_units respectively to specify which angular units are providing.
+
+ Finally, in cases where ra, dec are allowed, you may instead provide a
+ ``coord.CelestialCoord`` instance as the first argument to specify both RA and Dec.
+ """
+ x,y,z,sep = parse_xyzsep(args, kwargs, self._coords)
+ if self.min_size == 0:
+ # If min_size == 0, then regular method is already exact.
+ ind = self._get_near(x, y, z, sep)
+ else:
+ # Expand the radius by the minimum size of the cells.
+ sep1 = sep + self.min_size
+ # Get those indices
+ ind = self._get_near(x, y, z, sep1)
+ # Now check the actual radii of these points using the catalog x,y,z values.
+ rsq = (self.cat.x[ind]-x)**2 + (self.cat.y[ind]-y)**2
+ if self._coords != _lib.Flat:
+ rsq += (self.cat.z[ind]-z)**2
+ # Select the ones with r < sep
+ near = rsq < sep**2
+ ind = ind[near]
+ # It comes back unsorted, so sort it. (Not really required, but nicer output.)
+ return np.sort(ind)
+
+ def _get_near(self, x, y, z, sep):
+ # If self.min_size > 0, these results may be approximate, since the tree will have
+ # grouped points within this separation together.
+ # First count how many there are, so we can allocate the array for the indices.
+ n = self._count_near(x, y, z, sep)
+ ind = np.empty(n, dtype=int)
+ # Now fill the array with the indices of the nearby points.
+ _lib.FieldGetNear(self.data, x, y, z, sep, self._d, self._coords, lp(ind), n)
+ return ind
+
+[docs] @depr_pos_kwargs
+ def run_kmeans(self, npatch, *, max_iter=200, tol=1.e-5, init='tree', alt=False, rng=None):
+ r"""Use k-means algorithm to set patch labels for a field.
+
+ The k-means algorithm (cf. https://en.wikipedia.org/wiki/K-means_clustering) identifies
+ a center position for each patch. Each point is then assigned to the patch whose center
+ is closest. The centers are then updated to be the mean position of all the points
+ assigned to the patch. This process is repeated until the center locations have converged.
+
+ The process tends to converge relatively quickly. The convergence criterion we use
+ is a tolerance on the rms shift in the centroid positions as a fraction of the overall
+ size of the whole field. This is settable as ``tol`` (default 1.e-5). You can also
+ set the maximum number of iterations to allow as ``max_iter`` (default 200).
+
+ The upshot of the k-means process is to minimize the total within-cluster sum of squares
+ (WCSS), also known as the "inertia" of each patch. This tends to produce patches with
+ more or less similar inertia, which make them useful for jackknife or other sampling
+ estimates of the errors in the correlation functions.
+
+ More specifically, if the points :math:`j` have vector positions :math:`\vec x_j`,
+ and we define patches :math:`S_i` to comprise disjoint subsets of the :math:`j`
+ values, then the inertia :math:`I_i` of each patch is defined as:
+
+ .. math::
+
+ I_i = \sum_{j \in S_i} \left| \vec x_j - \vec \mu_i \right|^2,
+
+ where :math:`\vec \mu_i` is the center of each patch:
+
+ .. math::
+
+ \vec \mu_i \equiv \frac{\sum_{j \in S_i} \vec x_j}{N_i},
+
+ and :math:`N_i` is the number of points assigned to patch :math:`S_i`.
+ The k-means algorithm finds a solution that is a local minimum in the total inertia,
+ :math:`\sum_i I_i`.
+
+ In addition to the normal k-means algorithm, we also offer an alternate algorithm, which
+ can produce slightly better patches for the purpose of patch-based covariance estimation.
+ The ideal patch definition for such use would be to minimize the standard deviation (std)
+ of the inertia of each patch, not the total (or mean) inertia. It turns out that it is
+ difficult to devise an algorithm that literally does this, since it has a tendancy to
+ become unstable and not converge.
+
+ However, adding a penalty term to the patch assignment step of the normal k-means
+ algorithm turns out to work reasonably well. The penalty term we use is :math:`f I_i`,
+ where :math:`f` is a scaling constant (see below). When doing the assignment step we assign
+ each point :math:`j` to the patch :math:`i` that gives the minimum penalized distance
+
+ .. math::
+
+ d_{ij}^{\prime\;\! 2} = \left| \vec x_j - \mu_i \right|^2 + f I_i.
+
+ The penalty term means that patches with less inertia get more points on the next
+ iteration, and vice versa, which tends to equalize the inertia values somewhat.
+ The resulting patches have significantly lower std inertia, but typically only slightly
+ higher total inertia.
+
+ For the scaling constant, :math:`f`, we chose
+
+ .. math::
+
+ f = \frac{3}{\langle N_i\rangle},
+
+ three times the inverse of the mean number of points in each patch.
+
+ The :math:`1/\langle N_i\rangle` factor makes the two terms of comparable magnitude
+ near the edges of the patches, so patches still get most of the points near their previous
+ centers, even if they already have larger than average inertia, but some of the points in
+ the outskirts of the patch might switch to a nearby patch with smaller inertia. The
+ factor of 3 is purely empirical, and was found to give good results in terms of std
+ inertia on some test data (the DES SV field).
+
+ The alternate algorithm is available by specifying ``alt=True``. Despite it typically
+ giving better patch centers than the standard algorithm, we don't make it the default,
+ because it may be possible for the iteration to become unstable, leading to some patches
+ with no points in them. (This happened in our tests when the arbitrary factor in the
+ scaling constant was 5 instead of 3, but I could not prove that 3 would always avoid this
+ failure mode.) If this happens for you, your best bet is probably to switch to the
+ standard algorithm, which can never suffer from this problem.
+
+ Parameters:
+ npatch (int): How many patches to generate
+ max_iter (int): How many iterations at most to run. (default: 200)
+ tol (float): Tolerance in the rms centroid shift to consider as converged
+ as a fraction of the total field size. (default: 1.e-5)
+ init (str): Initialization method. Options are:
+
+ - 'tree' (default) = Use the normal tree structure of the
+ field, traversing down to a level where there are npatch
+ cells, and use the centroids of these cells as the initial
+ centers. This is almost always the best choice.
+ - 'random' = Use npatch random points as the intial centers.
+ - 'kmeans++' = Use the k-means++ algorithm.
+ cf. https://en.wikipedia.org/wiki/K-means%2B%2B
+
+ alt (bool): Use the alternate assignment algorithm to minimize the standard
+ deviation of the inertia rather than the total inertia (aka WCSS).
+ (default: False)
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for random
+ number generation. (default: None)
+
+ Returns:
+ Tuple containing
+
+ - patches (array): An array of patch labels, all integers from 0..npatch-1.
+ Size is self.ntot.
+ - centers (array): An array of center coordinates used to make the patches.
+ Shape is (npatch, 2) for flat geometries or (npatch, 3) for 3d or
+ spherical geometries. In the latter case, the centers represent
+ (x,y,z) coordinates on the unit sphere.
+ """
+ centers = self.kmeans_initialize_centers(npatch, init=init, rng=rng)
+ self.kmeans_refine_centers(centers, max_iter=max_iter, tol=tol, alt=alt)
+ patches = self.kmeans_assign_patches(centers)
+ return patches, centers
+
+[docs] @depr_pos_kwargs
+ def kmeans_initialize_centers(self, npatch, init='tree', *, rng=None):
+ """Use the field's tree structure to assign good initial centers for a K-Means run.
+
+ The classic K-Means algorithm involves starting with random points as the initial
+ centers of the patches. This has a tendency to result in rather poor results in
+ terms of having similar sized patches at the end. Specifically, the standard deviation
+ of the inertia at the local minimum that the K-Means algorithm settles into tends to be
+ fairly high for typical geometries.
+
+ A better approach is to use the existing tree structure to start out with centers that
+ are fairly evenly spread out through the field. This algorithm traverses the tree
+ until we get to a level that has enough cells for the requested number of patches.
+ Then it uses the centroids of these cells as the initial patch centers.
+
+ Parameters:
+ npatch (int): How many patches to generate initial centers for
+ init (str): Initialization method. Options are:
+
+ - 'tree' (default) = Use the normal tree structure of the
+ field, traversing down to a level where there are npatch
+ cells, and use the centroids of these cells as the initial
+ centers. This is almost always the best choice.
+ - 'random' = Use npatch random points as the intial centers.
+ - 'kmeans++' = Use the k-means++ algorithm.
+ cf. https://en.wikipedia.org/wiki/K-means%2B%2B
+
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for random
+ number generation. (default: None)
+
+ Returns:
+ An array of center coordinates.
+ Shape is (npatch, 2) for flat geometries or (npatch, 3) for 3d or
+ spherical geometries. In the latter case, the centers represent
+ (x,y,z) coordinates on the unit sphere.
+ """
+ if npatch > self.ntot:
+ raise ValueError("Invalid npatch. Cannot be greater than self.ntot.")
+ if npatch < 1:
+ raise ValueError("Invalid npatch. Cannot be less than 1.")
+ if self._coords == _lib.Flat:
+ centers = np.empty((npatch, 2))
+ else:
+ centers = np.empty((npatch, 3))
+ seed = 0 if rng is None else int(rng.random_sample() * 2**63)
+ if init == 'tree':
+ _lib.KMeansInitTree(self.data, dp(centers), int(npatch), self._d, self._coords, seed)
+ elif init == 'random':
+ _lib.KMeansInitRand(self.data, dp(centers), int(npatch), self._d, self._coords, seed)
+ elif init == 'kmeans++':
+ _lib.KMeansInitKMPP(self.data, dp(centers), int(npatch), self._d, self._coords, seed)
+ else:
+ raise ValueError("Invalid init: %s. "%init +
+ "Must be one of 'tree', 'random', or 'kmeans++.'")
+
+ return centers
+
+[docs] @depr_pos_kwargs
+ def kmeans_refine_centers(self, centers, *, max_iter=200, tol=1.e-5, alt=False):
+ """Fast implementation of the K-Means algorithm
+
+ The standard K-Means algorithm is as follows
+ (cf. https://en.wikipedia.org/wiki/K-means_clustering):
+
+ 1. Choose centers somehow. Traditionally, this is done by just selecting npatch random
+ points from the full set, but we do this more smartly in `kmeans_initialize_centers`.
+ 2. For each point, measure the distance to each current patch center, and assign it to the
+ patch that has the closest center.
+ 3. Update all the centers to be the centroid of the points assigned to each patch.
+ 4. Repeat 2, 3 until the rms shift in the centers is less than some tolerance or the
+ maximum number of iterations is reached.
+ 5. Assign the corresponding patch label to each point (`kmeans_assign_patches`).
+
+ In TreeCorr, we use the tree structure to massively increase the speed of steps 2 and 3.
+ For a given cell, we know both its center and its size, so we can quickly check whether
+ all the points in the cell are closer to one center than another. This lets us quickly
+ cull centers from consideration as we traverse the tree. Once we get to a cell where only
+ one center can be closest for any of the points in it, we stop traversing and assign the
+ whole cell to that patch.
+
+ Further, it is also fast to update the new centroid, since the sum of all the positions
+ for a cell is just N times the cell's centroid.
+
+ As a result, this algorithm typically takes a fraction of a second for ~a million points.
+ Indeed most of the time spent in the full kmeans calculation is in building the tree
+ in the first place, rather than actually running the kmeans code. With the alternate
+ algorithm (``alt=True``), the time is only slightly slower from having to calculate
+ the sizes at each step.
+
+ Parameters:
+ centers (array): An array of center coordinates. (modified by this function)
+ Shape is (npatch, 2) for flat geometries or (npatch, 3) for 3d or
+ spherical geometries. In the latter case, the centers represent
+ (x,y,z) coordinates on the unit sphere.
+ max_iter (int): How many iterations at most to run. (default: 200)
+ tol (float): Tolerance in the rms centroid shift to consider as converged
+ as a fraction of the total field size. (default: 1.e-5)
+ alt (bool): Use the alternate assignment algorithm to minimize the standard
+ deviation of the inertia rather than the total inertia (aka WCSS).
+ (default: False)
+ """
+ npatch = centers.shape[0]
+ _lib.KMeansRun(self.data, dp(centers), npatch, int(max_iter), float(tol),
+ bool(alt), self._d, self._coords)
+
+[docs] def kmeans_assign_patches(self, centers):
+ """Assign patch numbers to each point according to the given centers.
+
+ This is final step in the full K-Means algorithm. It assignes patch numbers to each
+ point in the field according to which center is closest.
+
+ Parameters:
+ centers (array): An array of center coordinates.
+ Shape is (npatch, 2) for flat geometries or (npatch, 3) for 3d or
+ spherical geometries. In the latter case, the centers represent
+ (x,y,z) coordinates on the unit sphere.
+
+ Returns:
+ An array of patch labels, all integers from 0..npatch-1. Size is self.ntot.
+ """
+ patches = np.empty(self.ntot, dtype=int)
+ npatch = centers.shape[0]
+ centers = np.ascontiguousarray(centers)
+ _lib.KMeansAssign(self.data, dp(centers), npatch,
+ lp(patches), self.ntot, self._d, self._coords)
+ return patches
+
+
+[docs]class NField(Field):
+ r"""This class stores the positions and number of objects in a tree structure from which it is
+ efficient to compute correlation functions.
+
+ An NField is typically created from a Catalog object using
+
+ >>> nfield = cat.getNField(min_size=min_size, max_size=max_size)
+
+ Parameters:
+ cat (Catalog): The catalog from which to make the field.
+ min_size (float): The minimum radius cell required (usually min_sep). (default: 0)
+ max_size (float): The maximum radius cell required (usually max_sep). (default: None)
+ split_method (str): Which split method to use ('mean', 'median', 'middle', or 'random').
+ (default: 'mean')
+ brute (bool): Whether to force traversal to the leaves for this field.
+ (default: False)
+ min_top (int): The minimum number of top layers to use when setting up the field.
+ (default: :math:`\max(3, \log_2(N_{\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the field.
+ (default: 10)
+ coords (str): The kind of coordinate system to use. (default: cat.coords)
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for random
+ number generation. (default: None)
+ logger (Logger): A logger file if desired. (default: None)
+ """
+ @depr_pos_kwargs
+ def __init__(self, cat, *, min_size=0, max_size=None, split_method='mean', brute=False,
+ min_top=None, max_top=10, coords=None, rng=None, logger=None):
+ if logger:
+ if cat.name != '':
+ logger.info('Building NField from cat %s',cat.name)
+ else:
+ logger.info('Building NField')
+
+ self._cat = weakref.ref(cat)
+ self.ntot = cat.ntot
+ self.min_size = float(min_size) if not brute else 0.
+ self.max_size = float(max_size) if max_size is not None else np.inf
+ self.split_method = split_method
+ self._sm = _parse_split_method(split_method)
+ self._d = 1 # NData
+ self.brute = bool(brute)
+ self.min_top, self.max_top = self._determine_top(min_top, max_top)
+ self.coords = coords if coords is not None else cat.coords
+ self._coords = coord_enum(self.coords) # These are the C++-layer enums
+ seed = 0 if rng is None else int(rng.random_sample() * 2**63)
+
+ self.data = _lib.BuildNField(dp(cat.x), dp(cat.y), dp(cat.z),
+ dp(cat.w), dp(cat.wpos), cat.ntot,
+ self.min_size, self.max_size, self._sm, seed,
+ self.brute, self.min_top, self.max_top, self._coords)
+ if logger:
+ logger.debug('Finished building NField (%s)',self.coords)
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+
+ # In case __init__ failed to get that far
+ if hasattr(self,'data'): # pragma: no branch
+ # I don't get this, but sometimes it gets here when the ffi.lock is already locked.
+ # When that happens, this will freeze in a `with ffi._lock` line in the ffi api.py.
+ # So, don't do that, and just accept the memory leak instead.
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyNField(self.data, self._coords)
+
+
+[docs]class KField(Field):
+ r"""This class stores the values of a scalar field (kappa in the weak lensing context) in a
+ tree structure from which it is efficient to compute correlation functions.
+
+ A KField is typically created from a Catalog object using
+
+ >>> kfield = cat.getKField(min_size, max_size, b)
+
+ Parameters:
+ cat (Catalog): The catalog from which to make the field.
+ min_size (float): The minimum radius cell required (usually min_sep). (default: 0)
+ max_size (float): The maximum radius cell required (usually max_sep). (default: None)
+ split_method (str): Which split method to use ('mean', 'median', 'middle', or 'random').
+ (default: 'mean')
+ brute (bool): Whether to force traversal to the leaves for this field.
+ (default: False)
+ min_top (int): The minimum number of top layers to use when setting up the field.
+ (default: :math:`\max(3, \log_2(N_{\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the field.
+ (default: 10)
+ coords (str): The kind of coordinate system to use. (default: cat.coords)
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for random
+ number generation. (default: None)
+ logger (Logger): A logger file if desired. (default: None)
+ """
+ @depr_pos_kwargs
+ def __init__(self, cat, *, min_size=0, max_size=None, split_method='mean', brute=False,
+ min_top=None, max_top=10, coords=None, rng=None, logger=None):
+ if logger:
+ if cat.name != '':
+ logger.info('Building KField from cat %s',cat.name)
+ else:
+ logger.info('Building KField')
+
+ self._cat = weakref.ref(cat)
+ self.ntot = cat.ntot
+ self.min_size = float(min_size) if not brute else 0.
+ self.max_size = float(max_size) if max_size is not None else np.inf
+ self.split_method = split_method
+ self._sm = _parse_split_method(split_method)
+ self._d = 2 # KData
+ self.brute = bool(brute)
+ self.min_top, self.max_top = self._determine_top(min_top, max_top)
+ self.coords = coords if coords is not None else cat.coords
+ self._coords = coord_enum(self.coords) # These are the C++-layer enums
+ seed = 0 if rng is None else int(rng.random_sample() * 2**63)
+
+ self.data = _lib.BuildKField(dp(cat.x), dp(cat.y), dp(cat.z),
+ dp(cat.k),
+ dp(cat.w), dp(cat.wpos), cat.ntot,
+ self.min_size, self.max_size, self._sm, seed,
+ self.brute, self.min_top, self.max_top, self._coords)
+ if logger:
+ logger.debug('Finished building KField (%s)',self.coords)
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+
+ # In case __init__ failed to get that far
+ if hasattr(self,'data'): # pragma: no branch
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyKField(self.data, self._coords)
+
+
+[docs]class GField(Field):
+ r"""This class stores the values of a spinor field (gamma in the weak lensing context) in a
+ tree structure from which it is efficient to compute correlation functions.
+
+ A GField is typically created from a Catalog object using
+
+ >>> gfield = cat.getGField(min_size, max_size, b)
+
+ Parameters:
+ cat (Catalog): The catalog from which to make the field.
+ min_size (float): The minimum radius cell required (usually min_sep). (default: 0)
+ max_size (float): The maximum radius cell required (usually max_sep). (default: None)
+ split_method (str): Which split method to use ('mean', 'median', 'middle', or 'random').
+ (default: 'mean')
+ brute (bool): Whether to force traversal to the leaves for this field.
+ (default: False)
+ min_top (int): The minimum number of top layers to use when setting up the field.
+ (default: :math:`\max(3, \log_2(N_{\rm cpu}))`)
+ max_top (int): The maximum number of top layers to use when setting up the field.
+ (default: 10)
+ coords (str): The kind of coordinate system to use. (default: cat.coords)
+ rng (RandomState): If desired, a numpy.random.RandomState instance to use for random
+ number generation. (default: None)
+ logger (Logger): A logger file if desired. (default: None)
+ """
+ @depr_pos_kwargs
+ def __init__(self, cat, *, min_size=0, max_size=None, split_method='mean', brute=False,
+ min_top=None, max_top=10, coords=None, rng=None, logger=None):
+ if logger:
+ if cat.name != '':
+ logger.info('Building GField from cat %s',cat.name)
+ else:
+ logger.info('Building GField')
+
+ self._cat = weakref.ref(cat)
+ self.ntot = cat.ntot
+ self.min_size = float(min_size) if not brute else 0.
+ self.max_size = float(max_size) if max_size is not None else np.inf
+ self.split_method = split_method
+ self._sm = _parse_split_method(split_method)
+ self._d = 3 # GData
+ self.brute = bool(brute)
+ self.min_top, self.max_top = self._determine_top(min_top, max_top)
+ self.coords = coords if coords is not None else cat.coords
+ self._coords = coord_enum(self.coords) # These are the C++-layer enums
+ seed = 0 if rng is None else int(rng.random_sample() * 2**63)
+
+ self.data = _lib.BuildGField(dp(cat.x), dp(cat.y), dp(cat.z),
+ dp(cat.g1), dp(cat.g2),
+ dp(cat.w), dp(cat.wpos), cat.ntot,
+ self.min_size, self.max_size, self._sm, seed,
+ self.brute, self.min_top, self.max_top, self._coords)
+ if logger:
+ logger.debug('Finished building GField (%s)',self.coords)
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+
+ # In case __init__ failed to get that far
+ if hasattr(self,'data'): # pragma: no branch
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyGField(self.data, self._coords)
+
+
+[docs]class SimpleField(object):
+ """A SimpleField is like a Field, but only stores the leaves as a list, skipping all the
+ tree stuff.
+
+ Again, this is an abstract base class, which cannot be instantiated. You should
+ make one of the concrete subclasses:
+
+ - NSimpleField describes a field of objects to be counted only.
+ - KSimpleField describes a field of points sampling a scalar field.
+ - GSimpleField describes a field of points sampling a spinor field.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+ """
+ def __init__(self):
+ raise NotImplementedError(
+ "SimpleField is an abstract base class. It cannot be instantiated.")
+
+
+[docs]class NSimpleField(SimpleField):
+ """This class stores the positions as a list, skipping all the tree stuff.
+
+ An NSimpleField is typically created from a Catalog object using
+
+ >>> nfield = cat.getNSimpleField()
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat (Catalog): The catalog from which to make the field.
+ logger (Logger): A logger file if desired. (default: None)
+ """
+ @depr_pos_kwargs
+ def __init__(self, cat, *, logger=None):
+ if logger:
+ if cat.name != '':
+ logger.info('Building NSimpleField from cat %s',cat.name)
+ else:
+ logger.info('Building NSimpleField')
+ self._d = 1 # NData
+ self.coords = cat.coords
+ self._coords = coord_enum(self.coords) # These are the C++-layer enums
+
+ self.data = _lib.BuildNSimpleField(dp(cat.x), dp(cat.y), dp(cat.z),
+ dp(cat.w), dp(cat.wpos), cat.ntot,
+ self._coords)
+ if logger:
+ logger.debug('Finished building NSimpleField (%s)',self.coords)
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+
+ # In case __init__ failed to get that far
+ if hasattr(self,'data'): # pragma: no branch
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyNSimpleField(self.data, self._coords)
+
+
+[docs]class KSimpleField(SimpleField):
+ """This class stores the kappa field as a list, skipping all the tree stuff.
+
+ A KSimpleField is typically created from a Catalog object using
+
+ >>> kfield = cat.getKSimpleField()
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat (Catalog): The catalog from which to make the field.
+ logger (Logger): A logger file if desired. (default: None)
+ """
+ @depr_pos_kwargs
+ def __init__(self, cat, *, logger=None):
+ if logger:
+ if cat.name != '':
+ logger.info('Building KSimpleField from cat %s',cat.name)
+ else:
+ logger.info('Building KSimpleField')
+ self._d = 2 # KData
+ self.coords = cat.coords
+ self._coords = coord_enum(self.coords) # These are the C++-layer enums
+
+ self.data = _lib.BuildKSimpleField(dp(cat.x), dp(cat.y), dp(cat.z),
+ dp(cat.k),
+ dp(cat.w), dp(cat.wpos), cat.ntot,
+ self._coords)
+ if logger:
+ logger.debug('Finished building KSimpleField (%s)',self.coords)
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+
+ # In case __init__ failed to get that far
+ if hasattr(self,'data'): # pragma: no branch
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyKSimpleField(self.data, self._coords)
+
+
+[docs]class GSimpleField(SimpleField):
+ """This class stores the shear field as a list, skipping all the tree stuff.
+
+ A GSimpleField is typically created from a Catalog object using
+
+ >>> gfield = cat.getGSimpleField()
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat (Catalog): The catalog from which to make the field.
+ logger (Logger): A logger file if desired. (default: None)
+ """
+ @depr_pos_kwargs
+ def __init__(self, cat, *, logger=None):
+ if logger:
+ if cat.name != '':
+ logger.info('Building GSimpleField from cat %s',cat.name)
+ else:
+ logger.info('Building GSimpleField')
+ self._d = 3 # GData
+ self.coords = cat.coords
+ self._coords = coord_enum(self.coords) # These are the C++-layer enums
+
+ self.data = _lib.BuildGSimpleField(dp(cat.x), dp(cat.y), dp(cat.z),
+ dp(cat.g1), dp(cat.g2),
+ dp(cat.w), dp(cat.wpos), cat.ntot,
+ self._coords)
+ if logger:
+ logger.debug('Finished building KSimpleField (%s)',self.coords)
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+
+ # In case __init__ failed to get that far
+ if hasattr(self,'data'): # pragma: no branch
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyGSimpleField(self.data, self._coords)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: ggcorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarG
+from .binnedcorr2 import BinnedCorr2
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class GGCorrelation(BinnedCorr2):
+ r"""This class handles the calculation and storage of a 2-point shear-shear correlation
+ function.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+
+ In addition, the following attributes are numpy arrays of length (nbins):
+
+ Attributes:
+
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ meanr: The (weighted) mean value of r for the pairs in each bin.
+ If there are no pairs in a bin, then exp(logr) will be used instead.
+ meanlogr: The (weighted) mean value of log(r) for the pairs in each bin.
+ If there are no pairs in a bin, then logr will be used instead.
+ xip: The correlation function, :math:`\xi_+(r)`.
+ xim: The correlation function, :math:`\xi_-(r)`.
+ xip_im: The imaginary part of :math:`\xi_+(r)`.
+ xim_im: The imaginary part of :math:`\xi_-(r)`.
+ varxip: An estimate of the variance of :math:`\xi_+(r)`
+ varxim: An estimate of the variance of :math:`\xi_-(r)`
+ weight: The total weight in each bin.
+ npairs: The number of pairs going into each bin (including pairs where one or
+ both objects have w=0).
+ cov: An estimate of the full covariance matrix for the data vector with
+ :math:`\xi_+` first and then :math:`\xi_-`.
+
+ .. note::
+
+ The default method for estimating the variance and covariance attributes (``varxip``,
+ ``varxim``, and ``cov``) is 'shot', which only includes the shape noise propagated into
+ the final correlation. This does not include sample variance, so it is always an
+ underestimate of the actual variance. To get better estimates, you need to set
+ ``var_method`` to something else and use patches in the input catalog(s).
+ cf. `Covariance Estimates`.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_auto` and/or
+ `process_cross`, then the units will not be applied to ``meanr`` or ``meanlogr`` until
+ the `finalize` function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> gg = treecorr.GGCorrelation(config)
+ >>> gg.process(cat) # For auto-correlation.
+ >>> gg.process(cat1,cat2) # For cross-correlation.
+ >>> gg.write(file_name) # Write out to a file.
+ >>> xip = gg.xip # Or access the correlation function directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr2`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr2` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `GGCorrelation`. See class doc for details.
+ """
+ BinnedCorr2.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 3 # GData
+ self._ro._d2 = 3 # GData
+ self.xip = np.zeros_like(self.rnom, dtype=float)
+ self.xim = np.zeros_like(self.rnom, dtype=float)
+ self.xip_im = np.zeros_like(self.rnom, dtype=float)
+ self.xim_im = np.zeros_like(self.rnom, dtype=float)
+ self.varxip = np.zeros_like(self.rnom, dtype=float)
+ self.varxim = np.zeros_like(self.rnom, dtype=float)
+ self.meanr = np.zeros_like(self.rnom, dtype=float)
+ self.meanlogr = np.zeros_like(self.rnom, dtype=float)
+ self.weight = np.zeros_like(self.rnom, dtype=float)
+ self.npairs = np.zeros_like(self.rnom, dtype=float)
+ self.logger.debug('Finished building GGCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr2(
+ self._d1, self._d2, self._bintype,
+ self._min_sep,self._max_sep,self._nbins,self._bin_size,self.b,
+ self.min_rpar, self.max_rpar, self.xperiod, self.yperiod, self.zperiod,
+ dp(self.xip),dp(self.xip_im),dp(self.xim),dp(self.xim_im),
+ dp(self.meanr),dp(self.meanlogr),dp(self.weight),dp(self.npairs))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr2(self.corr, self._d1, self._d2, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `GGCorrelation` instances are equal"""
+ return (isinstance(other, GGCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.min_rpar == other.min_rpar and
+ self.max_rpar == other.max_rpar and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meanr, other.meanr) and
+ np.array_equal(self.meanlogr, other.meanlogr) and
+ np.array_equal(self.xip, other.xip) and
+ np.array_equal(self.xim, other.xim) and
+ np.array_equal(self.xip_im, other.xip_im) and
+ np.array_equal(self.xim_im, other.xim_im) and
+ np.array_equal(self.varxip, other.varxip) and
+ np.array_equal(self.varxim, other.varxim) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.npairs, other.npairs))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = GGCorrelation.__new__(GGCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_auto(self, cat, *, metric=None, num_threads=None):
+ """Process a single catalog, accumulating the auto-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat (Catalog): The catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat.name == '':
+ self.logger.info('Starting process GG auto-correlations')
+ else:
+ self.logger.info('Starting process GG auto-correlations for cat %s.',cat.name)
+
+ self._set_metric(metric, cat.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ field = cat.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=bool(self.brute),
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',field.nTopLevelNodes)
+ _lib.ProcessAuto2(self.corr, field.data, self.output_dots,
+ field._d, self._coords, self._bintype, self._metric)
+
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process GG cross-correlations')
+ else:
+ self.logger.info('Starting process GG cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross2(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+
+[docs] @depr_pos_kwargs
+ def process_pairwise(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation, only using
+ the corresponding pairs of objects in each catalog.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `process` for
+ details. (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ import warnings
+ warnings.warn("The process_pairwise function is slated to be removed in a future version. "+
+ "If you are actually using this function usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process GG pairwise-correlations')
+ else:
+ self.logger.info('Starting process GG pairwise-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+
+ f1 = cat1.getGSimpleField()
+ f2 = cat2.getGSimpleField()
+
+ _lib.ProcessPair(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ In this case, this is the concatenation of self.xip and self.xim (raveled if necessary).
+ """
+ return np.concatenate([self.xip.ravel(), self.xim.ravel()])
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ This is the weight array corresponding to `getStat`. In this case, the weight is
+ duplicated to account for both xip and xim returned as part of getStat().
+ """
+ return np.concatenate([self.weight.ravel(), self.weight.ravel()])
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.xip[mask1] /= self.weight[mask1]
+ self.xim[mask1] /= self.weight[mask1]
+ self.xip_im[mask1] /= self.weight[mask1]
+ self.xim_im[mask1] /= self.weight[mask1]
+ self.meanr[mask1] /= self.weight[mask1]
+ self.meanlogr[mask1] /= self.weight[mask1]
+
+ # Update the units of meanr, meanlogr
+ self._apply_units(mask1)
+
+ # Use meanr, meanlogr when available, but set to nominal when no pairs in bin.
+ self.meanr[mask2] = self.rnom[mask2]
+ self.meanlogr[mask2] = self.logr[mask2]
+
+[docs] def finalize(self, varg1, varg2):
+ """Finalize the calculation of the correlation function.
+
+ The `process_auto` and `process_cross` commands accumulate values in each bin,
+ so they can be called multiple times if appropriate. Afterwards, this command
+ finishes the calculation by dividing each column by the total weight.
+
+ Parameters:
+ varg1 (float): The shear variance per component for the first field.
+ varg2 (float): The shear variance per component for the second field.
+ """
+ self._finalize()
+ self._var_num = 2. * varg1 * varg2
+ self.cov = self.estimate_cov(self.var_method)
+ self.varxip.ravel()[:] = self.cov.diagonal()[:self._nbins]
+ self.varxim.ravel()[:] = self.cov.diagonal()[self._nbins:]
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.xip.ravel()[:] = 0
+ self.xim.ravel()[:] = 0
+ self.xip_im.ravel()[:] = 0
+ self.xim_im.ravel()[:] = 0
+ self.meanr.ravel()[:] = 0
+ self.meanlogr.ravel()[:] = 0
+ self.weight.ravel()[:] = 0
+ self.npairs.ravel()[:] = 0
+
+[docs] def __iadd__(self, other):
+ """Add a second `GGCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `GGCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, GGCorrelation):
+ raise TypeError("Can only add another GGCorrelation object")
+ if not (self._nbins == other._nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep):
+ raise ValueError("GGCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords)
+ self.xip.ravel()[:] += other.xip.ravel()[:]
+ self.xim.ravel()[:] += other.xim.ravel()[:]
+ self.xip_im.ravel()[:] += other.xip_im.ravel()[:]
+ self.xim_im.ravel()[:] += other.xim_im.ravel()[:]
+ self.meanr.ravel()[:] += other.meanr.ravel()[:]
+ self.meanlogr.ravel()[:] += other.meanlogr.ravel()[:]
+ self.weight.ravel()[:] += other.weight.ravel()[:]
+ self.npairs.ravel()[:] += other.npairs.ravel()[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.xip for c in others], axis=0, out=self.xip)
+ np.sum([c.xim for c in others], axis=0, out=self.xim)
+ np.sum([c.xip_im for c in others], axis=0, out=self.xip_im)
+ np.sum([c.xim_im for c in others], axis=0, out=self.xim_im)
+ np.sum([c.meanr for c in others], axis=0, out=self.meanr)
+ np.sum([c.meanlogr for c in others], axis=0, out=self.meanlogr)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.npairs for c in others], axis=0, out=self.npairs)
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2=None, *, metric=None, num_threads=None, comm=None, low_mem=False,
+ initialize=True, finalize=True):
+ """Compute the correlation function.
+
+ - If only 1 argument is given, then compute an auto-correlation function.
+ - If 2 arguments are given, then compute a cross-correlation function.
+
+ Both arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first G field.
+ cat2 (Catalog): A catalog or list of catalogs for the second G field, if any.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr2.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if cat2 is not None and not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+
+ if cat2 is None:
+ self._process_all_auto(cat1, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if cat2 is None:
+ varg1 = calculateVarG(cat1, low_mem=low_mem)
+ varg2 = varg1
+ self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
+ else:
+ varg1 = calculateVarG(cat1, low_mem=low_mem)
+ varg2 = calculateVarG(cat2, low_mem=low_mem)
+ self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
+ self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2))
+ self.finalize(varg1,varg2)
+
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ The output file will include the following columns:
+
+ ========= ========================================================
+ Column Description
+ ========= ========================================================
+ r_nom The nominal center of the bin in r
+ meanr The mean value :math:`\langle r \rangle` of pairs that
+ fell into each bin
+ meanlogr The mean value :math:`\langle \log(r) \rangle` of pairs
+ that fell into each bin
+ xip The real part of the :math:`\xi_+` correlation function
+ xim The real part of the :math:`\xi_-` correlation function
+ xip_im The imag part of the :math:`\xi_+` correlation function
+ xim_im The imag part of the :math:`\xi_-` correlation function
+ sigma_xip The sqrt of the variance estimate of :math:`\xi_+`
+ sigma_xim The sqrt of the variance estimate of :math:`\xi_-`
+ weight The total weight contributing to each bin
+ npairs The total number of pairs in each bin
+ ========= ========================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing GG correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return ['r_nom', 'meanr', 'meanlogr', 'xip', 'xim', 'xip_im', 'xim_im',
+ 'sigma_xip', 'sigma_xim', 'weight', 'npairs']
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.meanr, self.meanlogr,
+ self.xip, self.xim, self.xip_im, self.xim_im,
+ np.sqrt(self.varxip), np.sqrt(self.varxim),
+ self.weight, self.npairs ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `GGCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading GG correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ # Helper function used by _read
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ self.meanr = data['meanR'].reshape(s)
+ self.meanlogr = data['meanlogR'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meanr = data['meanr'].reshape(s)
+ self.meanlogr = data['meanlogr'].reshape(s)
+ self.xip = data['xip'].reshape(s)
+ self.xim = data['xim'].reshape(s)
+ self.xip_im = data['xip_im'].reshape(s)
+ self.xim_im = data['xim_im'].reshape(s)
+ # Read old output files without error.
+ if 'sigma_xi' in data.dtype.names: # pragma: no cover
+ self.varxip = data['sigma_xi'].reshape(s)**2
+ self.varxim = data['sigma_xi'].reshape(s)**2
+ else:
+ self.varxip = data['sigma_xip'].reshape(s)**2
+ self.varxim = data['sigma_xim'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.npairs = data['npairs'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+
+[docs] @depr_pos_kwargs
+ def calculateMapSq(self, *, R=None, m2_uform=None):
+ r"""Calculate the aperture mass statistics from the correlation function.
+
+ .. math::
+
+ \langle M_{ap}^2 \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{2R^2}
+ \left [ T_+\left(\frac{r}{R}\right) \xi_+(r) +
+ T_-\left(\frac{r}{R}\right) \xi_-(r) \right] \\
+ \langle M_\times^2 \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{2R^2}
+ \left[ T_+\left(\frac{r}{R}\right) \xi_+(r) -
+ T_-\left(\frac{r}{R}\right) \xi_-(r) \right]
+
+ The ``m2_uform`` parameter sets which definition of the aperture mass to use.
+ The default is to use 'Crittenden'.
+
+ If ``m2_uform`` is 'Crittenden':
+
+ .. math::
+
+ U(r) &= \frac{1}{2\pi} (1-r^2) \exp(-r^2/2) \\
+ Q(r) &= \frac{1}{4\pi} r^2 \exp(-r^2/2) \\
+ T_+(s) &= \frac{s^4 - 16s^2 + 32}{128} \exp(-s^2/4) \\
+ T_-(s) &= \frac{s^4}{128} \exp(-s^2/4) \\
+ rmax &= \infty
+
+ cf. Crittenden, et al (2002): ApJ, 568, 20
+
+ If ``m2_uform`` is 'Schneider':
+
+ .. math::
+
+ U(r) &= \frac{9}{\pi} (1-r^2) (1/3-r^2) \\
+ Q(r) &= \frac{6}{\pi} r^2 (1-r^2) \\
+ T_+(s) &= \frac{12}{5\pi} (2-15s^2) \arccos(s/2) \\
+ &\qquad + \frac{1}{100\pi} s \sqrt{4-s^2} (120 + 2320s^2 - 754s^4 + 132s^6 - 9s^8) \\
+ T_-(s) &= \frac{3}{70\pi} s^3 (4-s^2)^{7/2} \\
+ rmax &= 2R
+
+ cf. Schneider, et al (2002): A&A, 389, 729
+
+ .. note::
+
+ This function is only implemented for Log binning.
+
+
+ Parameters:
+ R (array): The R values at which to calculate the aperture mass statistics.
+ (default: None, which means use self.rnom)
+ m2_uform (str): Which form to use for the aperture mass, as described above.
+ (default: 'Crittenden'; this value can also be given in the
+ constructor in the config dict.)
+
+ Returns:
+ Tuple containing
+
+ - mapsq = array of :math:`\langle M_{ap}^2 \rangle(R)`
+ - mapsq_im = the imaginary part of mapsq, which is an estimate of
+ :math:`\langle M_{ap} M_\times \rangle(R)`
+ - mxsq = array of :math:`\langle M_\times^2 \rangle(R)`
+ - mxsq_im = the imaginary part of mxsq, which is an estimate of
+ :math:`\langle M_{ap} M_\times \rangle(R)`
+ - varmapsq = array of the variance estimate of either mapsq or mxsq
+ """
+ if m2_uform is None:
+ m2_uform = self.config.get('m2_uform', 'Crittenden')
+ if m2_uform not in ['Crittenden', 'Schneider']:
+ raise ValueError("Invalid m2_uform")
+ if self.bin_type != 'Log':
+ raise ValueError("calculateMapSq requires Log binning.")
+ if R is None:
+ R = self.rnom
+
+ # Make s a matrix, so we can eventually do the integral by doing a matrix product.
+ s = np.outer(1./R, self.meanr)
+ ssq = s*s
+ if m2_uform == 'Crittenden':
+ exp_factor = np.exp(-ssq/4.)
+ Tp = (32. + ssq*(-16. + ssq)) / 128. * exp_factor
+ Tm = ssq * ssq / 128. * exp_factor
+ else:
+ Tp = np.zeros_like(s)
+ Tm = np.zeros_like(s)
+ sa = s[s<2.]
+ ssqa = ssq[s<2.]
+ Tp[s<2.] = 12./(5.*np.pi) * (2.-15.*ssqa) * np.arccos(sa/2.)
+ Tp[s<2.] += 1./(100.*np.pi) * sa * np.sqrt(4.-ssqa) * (
+ 120. + ssqa*(2320. + ssqa*(-754. + ssqa*(132. - 9.*ssqa))))
+ Tm[s<2.] = 3./(70.*np.pi) * sa * ssqa * (4.-ssqa)**3.5
+ Tp *= ssq
+ Tm *= ssq
+
+ # Now do the integral by taking the matrix products.
+ # Note that dlogr = bin_size
+ Tpxip = Tp.dot(self.xip)
+ Tmxim = Tm.dot(self.xim)
+ mapsq = (Tpxip + Tmxim) * 0.5 * self.bin_size
+ mxsq = (Tpxip - Tmxim) * 0.5 * self.bin_size
+ Tpxip_im = Tp.dot(self.xip_im)
+ Tmxim_im = Tm.dot(self.xim_im)
+ mapsq_im = (Tpxip_im + Tmxim_im) * 0.5 * self.bin_size
+ mxsq_im = (Tpxip_im - Tmxim_im) * 0.5 * self.bin_size
+
+ # The variance of each of these is
+ # Var(<Map^2>(R)) = int_r=0..2R [1/4 s^4 dlogr^2 (T+(s)^2 + T-(s)^2) Var(xi)]
+ varmapsq = (Tp**2).dot(self.varxip) + (Tm**2).dot(self.varxim)
+ varmapsq *= 0.25 * self.bin_size**2
+
+ return mapsq, mapsq_im, mxsq, mxsq_im, varmapsq
+
+
+[docs] @depr_pos_kwargs
+ def calculateGamSq(self, *, R=None, eb=False):
+ r"""Calculate the tophat shear variance from the correlation function.
+
+ .. math::
+
+ \langle \gamma^2 \rangle(R) &= \int_0^{2R} \frac{r dr}{R^2} S_+(s) \xi_+(r) \\
+ \langle \gamma^2 \rangle_E(R) &= \int_0^{2R} \frac{r dr}{2 R^2}
+ \left[ S_+\left(\frac{r}{R}\right) \xi_+(r) +
+ S_-\left(\frac{r}{R}\right) \xi_-(r) \right] \\
+ \langle \gamma^2 \rangle_B(R) &= \int_0^{2R} \frac{r dr}{2 R^2}
+ \left[ S_+\left(\frac{r}{R}\right) \xi_+(r) -
+ S_-\left(\frac{r}{R}\right) \xi_-(r) \right] \\
+
+ S_+(s) &= \frac{1}{\pi} \left(4 \arccos(s/2) - s \sqrt{4-s^2} \right) \\
+ S_-(s) &= \begin{cases}
+ s<=2, & \frac{1}{\pi s^4} \left(s \sqrt{4-s^2} (6-s^2) - 8(3-s^2) \arcsin(s/2)\right)\\
+ s>=2, & \frac{1}{s^4} \left(4(s^2-3)\right)
+ \end{cases}
+
+ cf. Schneider, et al (2002): A&A, 389, 729
+
+ The default behavior is not to compute the E/B versions. They are calculated if
+ eb is set to True.
+
+ .. note::
+
+ This function is only implemented for Log binning.
+
+
+ Parameters:
+ R (array): The R values at which to calculate the shear variance.
+ (default: None, which means use self.rnom)
+ eb (bool): Whether to include the E/B decomposition as well as the total
+ :math:`\langle \gamma^2\rangle`. (default: False)
+
+ Returns:
+ Tuple containing
+
+ - gamsq = array of :math:`\langle \gamma^2 \rangle(R)`
+ - vargamsq = array of the variance estimate of gamsq
+ - gamsq_e (Only if eb is True) = array of :math:`\langle \gamma^2 \rangle_E(R)`
+ - gamsq_b (Only if eb is True) = array of :math:`\langle \gamma^2 \rangle_B(R)`
+ - vargamsq_e (Only if eb is True) = array of the variance estimate of
+ gamsq_e or gamsq_b
+ """
+ if self.bin_type != 'Log':
+ raise ValueError("calculateGamSq requires Log binning.")
+
+ if R is None:
+ R = self.rnom
+ s = np.outer(1./R, self.meanr)
+ ssq = s*s
+ Sp = np.zeros_like(s)
+ sa = s[s<2]
+ ssqa = ssq[s<2]
+ Sp[s<2.] = 1./np.pi * ssqa * (4.*np.arccos(sa/2.) - sa*np.sqrt(4.-ssqa))
+
+ # Now do the integral by taking the matrix products.
+ # Note that dlogr = bin_size
+ Spxip = Sp.dot(self.xip)
+ gamsq = Spxip * self.bin_size
+ vargamsq = (Sp**2).dot(self.varxip) * self.bin_size**2
+
+ # Stop here if eb is False
+ if not eb: return gamsq, vargamsq
+
+ Sm = np.empty_like(s)
+ Sm[s<2.] = 1./(ssqa*np.pi) * (sa*np.sqrt(4.-ssqa)*(6.-ssqa)
+ -8.*(3.-ssqa)*np.arcsin(sa/2.))
+ Sm[s>=2.] = 4.*(ssq[s>=2]-3.)/ssq[s>=2]
+ # This already includes the extra ssq factor.
+
+ Smxim = Sm.dot(self.xim)
+ gamsq_e = (Spxip + Smxim) * 0.5 * self.bin_size
+ gamsq_b = (Spxip - Smxim) * 0.5 * self.bin_size
+ vargamsq_e = (Sp**2).dot(self.varxip) + (Sm**2).dot(self.varxim)
+ vargamsq_e *= 0.25 * self.bin_size**2
+
+ return gamsq, vargamsq, gamsq_e, gamsq_b, vargamsq_e
+
+
+[docs] @depr_pos_kwargs
+ def writeMapSq(self, file_name, *, R=None, m2_uform=None, file_type=None, precision=None):
+ r"""Write the aperture mass statistics based on the correlation function to the
+ file, file_name.
+
+ See `calculateMapSq` for an explanation of the ``m2_uform`` parameter.
+
+ The output file will include the following columns:
+
+ ========= ==========================================================
+ Column Description
+ ========= ==========================================================
+ R The aperture radius
+ Mapsq The real part of :math:`\langle M_{ap}^2\rangle`
+ (cf. `calculateMapSq`)
+ Mxsq The real part of :math:`\langle M_\times^2\rangle`
+ MMxa The imag part of :math:`\langle M_{ap}^2\rangle`:
+ an estimator of :math:`\langle M_{ap} M_\times\rangle`
+ MMxa The imag part of :math:`\langle M_\times^2\rangle`:
+ an estimator of :math:`\langle M_{ap} M_\times\rangle`
+ sig_map The sqrt of the variance estimate of
+ :math:`\langle M_{ap}^2\rangle`
+ Gamsq The tophat shear variance :math:`\langle \gamma^2\rangle`
+ (cf. `calculateGamSq`)
+ sig_gam The sqrt of the variance estimate of
+ :math:`\langle \gamma^2\rangle`
+ ========= ==========================================================
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ R (array): The R values at which to calculate the statistics.
+ (default: None, which means use self.rnom)
+ m2_uform (str): Which form to use for the aperture mass. (default: 'Crittenden';
+ this value can also be given in the constructor in the config dict.)
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ """
+ self.logger.info('Writing Map^2 from GG correlations to %s',file_name)
+
+ if R is None:
+ R = self.rnom
+ mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = self.calculateMapSq(R=R, m2_uform=m2_uform)
+ gamsq, vargamsq = self.calculateGamSq(R=R)
+ if precision is None:
+ precision = self.config.get('precision', 4)
+
+ col_names = ['R','Mapsq','Mxsq','MMxa','MMxb','sig_map','Gamsq','sig_gam']
+ columns = [ R,
+ mapsq, mxsq, mapsq_im, -mxsq_im, np.sqrt(varmapsq),
+ gamsq, np.sqrt(vargamsq) ]
+ with make_writer(file_name, precision, file_type, logger=self.logger) as writer:
+ writer.write(col_names, columns)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: nnncorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarG
+from .binnedcorr3 import BinnedCorr3
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class GGGCorrelation(BinnedCorr3):
+ r"""This class handles the calculation and storage of a 3-point shear-shear-shear correlation
+ function.
+
+ We use the "natural components" of the shear 3-point function described by Schneider &
+ Lombardi (2003) [Astron.Astrophys. 397 (2003) 809-818]. In this paradigm, the shears
+ are projected relative to some point defined by the geometry of the triangle. They
+ give several reasonable choices for this point. We choose the triangle's centroid as the
+ "most natural" point, as many simple shear fields have purely real :math:`\Gamma_0` using
+ this definition. It is also a fairly simple point to calculate in the code compared to
+ some of the other options they offer, so projections relative to it are fairly efficient.
+
+ There are 4 complex-valued 3-point shear corrletion functions defined for triples of shear
+ values projected relative to the line joining the location of the shear to the cenroid of
+ the triangle:
+
+ .. math::
+
+ \Gamma_0 &= \langle \gamma(\mathbf{x1}) \gamma(\mathbf{x2}) \gamma(\mathbf{x3}) \rangle \\
+ \Gamma_1 &= \langle \gamma(\mathbf{x1})^* \gamma(\mathbf{x2}) \gamma(\mathbf{x3}) \rangle \\
+ \Gamma_2 &= \langle \gamma(\mathbf{x1}) \gamma(\mathbf{x2})^* \gamma(\mathbf{x3}) \rangle \\
+ \Gamma_3 &= \langle \gamma(\mathbf{x1}) \gamma(\mathbf{x2}) \gamma(\mathbf{x3})^* \rangle \\
+
+ where :math:`\mathbf{x1}, \mathbf{x2}, \mathbf{x3}` are the corners of the triange opposite
+ sides d1, d2, d3 respectively, where d1 > d2 > d3, and :math:`{}^*` indicates complex
+ conjugation.
+
+ See the doc string of `BinnedCorr3` for a description of how the triangles
+ are binned.
+
+ This class only holds one set of these :math:`\Gamma` functions, which means that it is only
+ directly applicable for computing auto-correlations. To describe a cross-correlation of one
+ shear field with another, you need three sets of these functions. To describe a three-way
+ cross-correlation of three different shear fields, you need six. These use cases are
+ enabled by the class `GGGCrossCorrelation` which holds six instances of this class to keep
+ track of all the various triangles. See that class for more details.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr where r = d2
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+ nubins: The number of bins in u where u = d3/d2
+ ubin_size: The size of the bins in u
+ min_u: The minimum u being considered
+ max_u: The maximum u being considered
+ nvbins: The number of bins in v where v = +-(d1-d2)/d3
+ vbin_size: The size of the bins in v
+ min_v: The minimum v being considered
+ max_v: The maximum v being considered
+ logr1d: The nominal centers of the nbins bins in log(r).
+ u1d: The nominal centers of the nubins bins in u.
+ v1d: The nominal centers of the nvbins bins in v.
+
+ In addition, the following attributes are numpy arrays whose shape is (nbins, nubins, nvbins):
+
+ Attributes:
+ logr: The nominal center of each bin in log(r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ u: The nominal center of each bin in u.
+ v: The nominal center of each bin in v.
+ meand1: The (weighted) mean value of d1 for the triangles in each bin.
+ meanlogd1: The mean value of log(d1) for the triangles in each bin.
+ meand2: The (weighted) mean value of d2 (aka r) for the triangles in each bin.
+ meanlogd2: The mean value of log(d2) for the triangles in each bin.
+ meand2: The (weighted) mean value of d3 for the triangles in each bin.
+ meanlogd2: The mean value of log(d3) for the triangles in each bin.
+ meanu: The mean value of u for the triangles in each bin.
+ meanv: The mean value of v for the triangles in each bin.
+ gam0: The 0th "natural" correlation function, :math:`\Gamma_0(r,u,v)`.
+ gam1: The 1st "natural" correlation function, :math:`\Gamma_1(r,u,v)`.
+ gam2: The 2nd "natural" correlation function, :math:`\Gamma_2(r,u,v)`.
+ gam3: The 3rd "natural" correlation function, :math:`\Gamma_3(r,u,v)`.
+ vargam0: The variance of :math:`\Gamma_0`, only including the shot noise
+ propagated into the final correlation. This (and the related values for
+ 1,2,3) does not include sample variance, so it is always an underestimate
+ of the actual variance.
+ vargam1: The variance of :math:`\Gamma_1`.
+ vargam2: The variance of :math:`\Gamma_2`.
+ vargam3: The variance of :math:`\Gamma_3`.
+ weight: The total weight in each bin.
+ ntri: The number of triangles going into each bin (including those where one or
+ more objects have w=0).
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_auto` and/or
+ `process_cross`, then the units will not be applied to ``meanr`` or ``meanlogr`` until
+ the `finalize` function is called.
+
+ The typical usage pattern is as follows::
+
+ >>> ggg = treecorr.GGGCorrelation(config)
+ >>> ggg.process(cat) # For auto-correlation.
+ >>> ggg.process(cat1,cat2,cat3) # For cross-correlation.
+ >>> ggg.write(file_name) # Write out to a file.
+ >>> gam0 = ggg.gam0, etc. # To access gamma values directly.
+ >>> gam0r = ggg.gam0r # You can also access real and imag parts separately.
+ >>> gam0i = ggg.gam0i
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr3`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr3` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `GGGCorrelation`. See class doc for details.
+ """
+ BinnedCorr3.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 3 # GData
+ self._ro._d2 = 3 # GData
+ self._ro._d3 = 3 # GData
+ shape = self.logr.shape
+ self.gam0r = np.zeros(shape, dtype=float)
+ self.gam1r = np.zeros(shape, dtype=float)
+ self.gam2r = np.zeros(shape, dtype=float)
+ self.gam3r = np.zeros(shape, dtype=float)
+ self.gam0i = np.zeros(shape, dtype=float)
+ self.gam1i = np.zeros(shape, dtype=float)
+ self.gam2i = np.zeros(shape, dtype=float)
+ self.gam3i = np.zeros(shape, dtype=float)
+ self.vargam0 = np.zeros(shape, dtype=float)
+ self.vargam1 = np.zeros(shape, dtype=float)
+ self.vargam2 = np.zeros(shape, dtype=float)
+ self.vargam3 = np.zeros(shape, dtype=float)
+ self.meand1 = np.zeros(shape, dtype=float)
+ self.meanlogd1 = np.zeros(shape, dtype=float)
+ self.meand2 = np.zeros(shape, dtype=float)
+ self.meanlogd2 = np.zeros(shape, dtype=float)
+ self.meand3 = np.zeros(shape, dtype=float)
+ self.meanlogd3 = np.zeros(shape, dtype=float)
+ self.meanu = np.zeros(shape, dtype=float)
+ self.meanv = np.zeros(shape, dtype=float)
+ self.weight = np.zeros(shape, dtype=float)
+ self.ntri = np.zeros(shape, dtype=float)
+ self.logger.debug('Finished building GGGCorr')
+
+ @property
+ def gam0(self):
+ return self.gam0r + 1j * self.gam0i
+
+ @property
+ def gam1(self):
+ return self.gam1r + 1j * self.gam1i
+
+ @property
+ def gam2(self):
+ return self.gam2r + 1j * self.gam2i
+
+ @property
+ def gam3(self):
+ return self.gam3r + 1j * self.gam3i
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr3(
+ self._d1, self._d2, self._d3, self._bintype,
+ self._min_sep,self._max_sep,self.nbins,self._bin_size,self.b,
+ self.min_u,self.max_u,self.nubins,self.ubin_size,self.bu,
+ self.min_v,self.max_v,self.nvbins,self.vbin_size,self.bv,
+ self.xperiod, self.yperiod, self.zperiod,
+ dp(self.gam0r), dp(self.gam0i), dp(self.gam1r), dp(self.gam1i),
+ dp(self.gam2r), dp(self.gam2i), dp(self.gam3r), dp(self.gam3i),
+ dp(self.meand1), dp(self.meanlogd1), dp(self.meand2), dp(self.meanlogd2),
+ dp(self.meand3), dp(self.meanlogd3), dp(self.meanu), dp(self.meanv),
+ dp(self.weight), dp(self.ntri))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr3(self.corr, self._d1, self._d2, self._d3, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `GGGCorrelation` instances are equal"""
+ return (isinstance(other, GGGCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nubins == other.nubins and
+ self.ubin_size == other.ubin_size and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v and
+ self.nvbins == other.nvbins and
+ self.vbin_size == other.vbin_size and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meand1, other.meand1) and
+ np.array_equal(self.meanlogd1, other.meanlogd1) and
+ np.array_equal(self.meand2, other.meand2) and
+ np.array_equal(self.meanlogd2, other.meanlogd2) and
+ np.array_equal(self.meand3, other.meand3) and
+ np.array_equal(self.meanlogd3, other.meanlogd3) and
+ np.array_equal(self.meanu, other.meanu) and
+ np.array_equal(self.meanv, other.meanv) and
+ np.array_equal(self.gam0r, other.gam0r) and
+ np.array_equal(self.gam0i, other.gam0i) and
+ np.array_equal(self.gam1r, other.gam1r) and
+ np.array_equal(self.gam1i, other.gam1i) and
+ np.array_equal(self.gam2r, other.gam2r) and
+ np.array_equal(self.gam2i, other.gam2i) and
+ np.array_equal(self.gam3r, other.gam3r) and
+ np.array_equal(self.gam3i, other.gam3i) and
+ np.array_equal(self.vargam0, other.vargam0) and
+ np.array_equal(self.vargam1, other.vargam1) and
+ np.array_equal(self.vargam2, other.vargam2) and
+ np.array_equal(self.vargam3, other.vargam3) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.ntri, other.ntri))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = GGGCorrelation.__new__(GGGCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_auto(self, cat, *, metric=None, num_threads=None):
+ """Process a single catalog, accumulating the auto-correlation.
+
+ This accumulates the auto-correlation for the given catalog. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ Parameters:
+ cat (Catalog): The catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat.name == '':
+ self.logger.info('Starting process GGG auto-correlations')
+ else:
+ self.logger.info('Starting process GGG auto-correlations for cat %s.', cat.name)
+
+ self._set_metric(metric, cat.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ field = cat.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method, brute=bool(self.brute),
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',field.nTopLevelNodes)
+ _lib.ProcessAuto3(self.corr, field.data, self.output_dots,
+ field._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross12(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process two catalogs, accumulating the 3pt cross-correlation, where one of the
+ points in each triangle come from the first catalog, and two come from the second.
+
+ This accumulates the cross-correlation for the given catalogs as part of a larger
+ auto-correlation calculation. E.g. when splitting up a large catalog into patches,
+ this is appropriate to use for the cross correlation between different patches
+ as part of the complete auto-correlation of the full catalog.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process. (1 point in each triangle will come
+ from this catalog.)
+ cat2 (Catalog): The second catalog to process. (2 points in each triangle will come
+ from this catalog.)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process GGG (1-2) cross-correlations')
+ else:
+ self.logger.info('Starting process GGG (1-2) cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 3 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross12(self.corr, self.corr, self.corr,
+ f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords,
+ self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, cat3, *, metric=None, num_threads=None):
+ """Process a set of three catalogs, accumulating the 3pt cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs as part of a larger
+ auto-correlation calculation. E.g. when splitting up a large catalog into patches,
+ this is appropriate to use for the cross correlation between different patches
+ as part of the complete auto-correlation of the full catalog.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ cat3 (Catalog): The third catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '' and cat3.name == '':
+ self.logger.info('Starting process GGG cross-correlations')
+ else:
+ self.logger.info('Starting process GGG cross-correlations for cats %s, %s, %s.',
+ cat1.name, cat2.name, cat3.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords, cat3.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f3 = cat3.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 3,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 6 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross3(self.corr, self.corr, self.corr,
+ self.corr, self.corr, self.corr,
+ f1.data, f2.data, f3.data, self.output_dots,
+ f1._d, f2._d, f3._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.gam0r[mask1] /= self.weight[mask1]
+ self.gam0i[mask1] /= self.weight[mask1]
+ self.gam1r[mask1] /= self.weight[mask1]
+ self.gam1i[mask1] /= self.weight[mask1]
+ self.gam2r[mask1] /= self.weight[mask1]
+ self.gam2i[mask1] /= self.weight[mask1]
+ self.gam3r[mask1] /= self.weight[mask1]
+ self.gam3i[mask1] /= self.weight[mask1]
+ self.meand1[mask1] /= self.weight[mask1]
+ self.meanlogd1[mask1] /= self.weight[mask1]
+ self.meand2[mask1] /= self.weight[mask1]
+ self.meanlogd2[mask1] /= self.weight[mask1]
+ self.meand3[mask1] /= self.weight[mask1]
+ self.meanlogd3[mask1] /= self.weight[mask1]
+ self.meanu[mask1] /= self.weight[mask1]
+ self.meanv[mask1] /= self.weight[mask1]
+
+ # Update the units
+ self._apply_units(mask1)
+
+ # Use meanlogr when available, but set to nominal when no triangles in bin.
+ self.meand2[mask2] = self.rnom[mask2]
+ self.meanlogd2[mask2] = self.logr[mask2]
+ self.meanu[mask2] = self.u[mask2]
+ self.meanv[mask2] = self.v[mask2]
+ self.meand3[mask2] = self.u[mask2] * self.meand2[mask2]
+ self.meanlogd3[mask2] = np.log(self.meand3[mask2])
+ self.meand1[mask2] = np.abs(self.v[mask2]) * self.meand3[mask2] + self.meand2[mask2]
+ self.meanlogd1[mask2] = np.log(self.meand1[mask2])
+
+[docs] def finalize(self, varg1, varg2, varg3):
+ """Finalize the calculation of the correlation function.
+
+ The `process_auto` and `process_cross` commands accumulate values in each bin,
+ so they can be called multiple times if appropriate. Afterwards, this command
+ finishes the calculation by dividing by the total weight.
+
+ Parameters:
+ varg1 (float): The shear variance for the first field.
+ varg2 (float): The shear variance for the second field.
+ varg3 (float): The shear variance for the third field.
+ """
+ self._finalize()
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+ self._var_num = 4 * varg1 * varg2 * varg3
+ self.cov = self.estimate_cov(self.var_method)
+ # Note: diagonal should be very close to pure real. So ok to just copy real part.
+ diag = self.cov.diagonal()
+ # This should never trigger. If you find this assert to fail, please post an
+ # issue about it describing your use case that caused it to fail.
+ assert np.sum(diag.imag**2) <= 1.e-8 * np.sum(diag.real**2)
+ self.vargam0.ravel()[:] = self.cov.diagonal()[0:self._nbins].real
+ self.vargam1.ravel()[:] = self.cov.diagonal()[self._nbins:2*self._nbins].real
+ self.vargam2.ravel()[:] = self.cov.diagonal()[2*self._nbins:3*self._nbins].real
+ self.vargam3.ravel()[:] = self.cov.diagonal()[3*self._nbins:4*self._nbins].real
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.gam0r[:,:,:] = 0.
+ self.gam0i[:,:,:] = 0.
+ self.gam1r[:,:,:] = 0.
+ self.gam1i[:,:,:] = 0.
+ self.gam2r[:,:,:] = 0.
+ self.gam2i[:,:,:] = 0.
+ self.gam3r[:,:,:] = 0.
+ self.gam3i[:,:,:] = 0.
+ self.vargam0[:,:,:] = 0.
+ self.vargam1[:,:,:] = 0.
+ self.vargam2[:,:,:] = 0.
+ self.vargam3[:,:,:] = 0.
+ self.meand1[:,:,:] = 0.
+ self.meanlogd1[:,:,:] = 0.
+ self.meand2[:,:,:] = 0.
+ self.meanlogd2[:,:,:] = 0.
+ self.meand3[:,:,:] = 0.
+ self.meanlogd3[:,:,:] = 0.
+ self.meanu[:,:,:] = 0.
+ self.meanv[:,:,:] = 0.
+ self.weight[:,:,:] = 0.
+ self.ntri[:,:,:] = 0.
+
+[docs] def __iadd__(self, other):
+ """Add a second `GGGCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `GGGCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, GGGCorrelation):
+ raise TypeError("Can only add another GGGCorrelation object")
+ if not (self.nbins == other.nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.nubins == other.nubins and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nvbins == other.nvbins and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v):
+ raise ValueError("GGGCorrelation to be added is not compatible with this one.")
+
+ if not other.nonzero: return self
+ self._set_metric(other.metric, other.coords, other.coords, other.coords)
+ self.gam0r[:] += other.gam0r[:]
+ self.gam0i[:] += other.gam0i[:]
+ self.gam1r[:] += other.gam1r[:]
+ self.gam1i[:] += other.gam1i[:]
+ self.gam2r[:] += other.gam2r[:]
+ self.gam2i[:] += other.gam2i[:]
+ self.gam3r[:] += other.gam3r[:]
+ self.gam3i[:] += other.gam3i[:]
+ self.meand1[:] += other.meand1[:]
+ self.meanlogd1[:] += other.meanlogd1[:]
+ self.meand2[:] += other.meand2[:]
+ self.meanlogd2[:] += other.meanlogd2[:]
+ self.meand3[:] += other.meand3[:]
+ self.meanlogd3[:] += other.meanlogd3[:]
+ self.meanu[:] += other.meanu[:]
+ self.meanv[:] += other.meanv[:]
+ self.weight[:] += other.weight[:]
+ self.ntri[:] += other.ntri[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.gam0r for c in others], axis=0, out=self.gam0r)
+ np.sum([c.gam0i for c in others], axis=0, out=self.gam0i)
+ np.sum([c.gam1r for c in others], axis=0, out=self.gam1r)
+ np.sum([c.gam1i for c in others], axis=0, out=self.gam1i)
+ np.sum([c.gam2r for c in others], axis=0, out=self.gam2r)
+ np.sum([c.gam2i for c in others], axis=0, out=self.gam2i)
+ np.sum([c.gam3r for c in others], axis=0, out=self.gam3r)
+ np.sum([c.gam3i for c in others], axis=0, out=self.gam3i)
+ np.sum([c.meand1 for c in others], axis=0, out=self.meand1)
+ np.sum([c.meanlogd1 for c in others], axis=0, out=self.meanlogd1)
+ np.sum([c.meand2 for c in others], axis=0, out=self.meand2)
+ np.sum([c.meanlogd2 for c in others], axis=0, out=self.meanlogd2)
+ np.sum([c.meand3 for c in others], axis=0, out=self.meand3)
+ np.sum([c.meanlogd3 for c in others], axis=0, out=self.meanlogd3)
+ np.sum([c.meanu for c in others], axis=0, out=self.meanu)
+ np.sum([c.meanv for c in others], axis=0, out=self.meanv)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.ntri for c in others], axis=0, out=self.ntri)
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2=None, cat3=None, *, metric=None, num_threads=None,
+ comm=None, low_mem=False, initialize=True, finalize=True):
+ """Compute the 3pt correlation function.
+
+ - If only 1 argument is given, then compute an auto-correlation function.
+ - If 2 arguments are given, then compute a cross-correlation function with the
+ first catalog taking one corner of the triangles, and the second taking two corners.
+ - If 3 arguments are given, then compute a three-way cross-correlation function.
+
+ All arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ .. note::
+
+ For a correlation of multiple catalogs, it typically matters which corner of the
+ triangle comes from which catalog, which is not kept track of by this function.
+ The final accumulation will have d1 > d2 > d3 regardless of which input catalog
+ appears at each corner. The class which keeps track of which catalog appears
+ in each position in the triangle is `GGGCrossCorrelation`.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first G field.
+ cat2 (Catalog): A catalog or list of catalogs for the second G field.
+ (default: None)
+ cat3 (Catalog): A catalog or list of catalogs for the third G field.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr3.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if cat2 is not None and not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+ if cat3 is not None and not isinstance(cat3,list):
+ cat3 = cat3.get_patches(low_mem=low_mem)
+
+ if cat2 is None:
+ if cat3 is not None:
+ raise ValueError("For two catalog case, use cat1,cat2, not cat1,cat3")
+ self._process_all_auto(cat1, metric, num_threads, comm, low_mem)
+ elif cat3 is None:
+ self._process_all_cross12(cat1, cat2, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, cat3, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if cat2 is None:
+ varg1 = calculateVarG(cat1, low_mem=low_mem)
+ varg2 = varg1
+ varg3 = varg1
+ self.logger.info("varg = %f: sig_g = %f",varg1,math.sqrt(varg1))
+ elif cat3 is None:
+ varg1 = calculateVarG(cat1, low_mem=low_mem)
+ varg2 = calculateVarG(cat2, low_mem=low_mem)
+ varg3 = varg2
+ self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1))
+ self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2))
+ else:
+ varg1 = calculateVarG(cat1, low_mem=low_mem)
+ varg2 = calculateVarG(cat2, low_mem=low_mem)
+ varg3 = calculateVarG(cat3, low_mem=low_mem)
+ self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1))
+ self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2))
+ self.logger.info("varg3 = %f: sig_g = %f",varg3,math.sqrt(varg3))
+ self.finalize(varg1,varg2,varg3)
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ In this case, the concatenation of gam0.ravel(), gam1.ravel(), gam2.ravel(), gam3.ravel().
+
+ .. note::
+
+ This is a complex array, unlike most other statistics.
+ The computed covariance matrix will be complex, although since it is Hermitian the
+ diagonal is real, so the resulting vargam0, etc. will all be real arrays.
+ """
+ return np.concatenate([self.gam0.ravel(), self.gam1.ravel(),
+ self.gam2.ravel(), self.gam3.ravel()])
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ In this case, 4 copies of self.weight.ravel().
+ """
+ return np.concatenate([self.weight.ravel()] * 4)
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ As described in the doc string for `GGGCorrelation`, we use the "natural components" of
+ the shear 3-point function described by Schneider & Lombardi (2003) using the triangle
+ centroid as the projection point. There are 4 complex-valued natural components, so there
+ are 8 columns in the output file.
+
+ The output file will include the following columns:
+
+ ========== =============================================================
+ Column Description
+ ========== =============================================================
+ r_nom The nominal center of the bin in r = d2 where d1 > d2 > d3
+ u_nom The nominal center of the bin in u = d3/d2
+ v_nom The nominal center of the bin in v = +-(d1-d2)/d3
+ meand1 The mean value :math:`\langle d1\rangle` of triangles that
+ fell into each bin
+ meanlogd1 The mean value :math:`\langle \log(d1)\rangle` of triangles
+ that fell into each bin
+ meand2 The mean value :math:`\langle d2\rangle` of triangles that
+ fell into each bin
+ meanlogd2 The mean value :math:`\langle \log(d2)\rangle` of triangles
+ that fell into each bin
+ meand3 The mean value :math:`\langle d3\rangle` of triangles that
+ fell into each bin
+ meanlogd3 The mean value :math:`\langle \log(d3)\rangle` of triangles
+ that fell into each bin
+ meanu The mean value :math:`\langle u\rangle` of triangles that
+ fell into each bin
+ meanv The mean value :math:`\langle v\rangle` of triangles that
+ fell into each bi.
+ gam0r The real part of the estimator of :math:`\Gamma_0(r,u,v)`
+ gam0i The imag part of the estimator of :math:`\Gamma_0(r,u,v)`
+ gam1r The real part of the estimator of :math:`\Gamma_1(r,u,v)`
+ gam1i The imag part of the estimator of :math:`\Gamma_1(r,u,v)`
+ gam2r The real part of the estimator of :math:`\Gamma_2(r,u,v)`
+ gam2i The imag part of the estimator of :math:`\Gamma_2(r,u,v)`
+ gam3r The real part of the estimator of :math:`\Gamma_3(r,u,v)`
+ gam3i The imag part of the estimator of :math:`\Gamma_3(r,u,v)`
+ sigma_gam0 The sqrt of the variance estimate of :math:`\Gamma_0`
+ sigma_gam1 The sqrt of the variance estimate of :math:`\Gamma_1`
+ sigma_gam2 The sqrt of the variance estimate of :math:`\Gamma_2`
+ sigma_gam3 The sqrt of the variance estimate of :math:`\Gamma_3`
+ weight The total weight of triangles contributing to each bin.
+ ntri The number of triangles contributing to each bin.
+ ========== =============================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing GGG correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return [ 'r_nom', 'u_nom', 'v_nom', 'meand1', 'meanlogd1', 'meand2', 'meanlogd2',
+ 'meand3', 'meanlogd3', 'meanu', 'meanv',
+ 'gam0r', 'gam0i', 'gam1r', 'gam1i', 'gam2r', 'gam2i', 'gam3r', 'gam3i',
+ 'sigma_gam0', 'sigma_gam1', 'sigma_gam2', 'sigma_gam3', 'weight', 'ntri' ]
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.u, self.v,
+ self.meand1, self.meanlogd1, self.meand2, self.meanlogd2,
+ self.meand3, self.meanlogd3, self.meanu, self.meanv,
+ self.gam0r, self.gam0i, self.gam1r, self.gam1i,
+ self.gam2r, self.gam2i, self.gam3r, self.gam3i,
+ np.sqrt(self.vargam0), np.sqrt(self.vargam1), np.sqrt(self.vargam2),
+ np.sqrt(self.vargam3), self.weight, self.ntri ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `GGGCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading GGG correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self._ro.u = data['u_nom'].reshape(s)
+ self._ro.v = data['v_nom'].reshape(s)
+ self.meand1 = data['meand1'].reshape(s)
+ self.meanlogd1 = data['meanlogd1'].reshape(s)
+ self.meand2 = data['meand2'].reshape(s)
+ self.meanlogd2 = data['meanlogd2'].reshape(s)
+ self.meand3 = data['meand3'].reshape(s)
+ self.meanlogd3 = data['meanlogd3'].reshape(s)
+ self.meanu = data['meanu'].reshape(s)
+ self.meanv = data['meanv'].reshape(s)
+ self.gam0r = data['gam0r'].reshape(s)
+ self.gam0i = data['gam0i'].reshape(s)
+ self.gam1r = data['gam1r'].reshape(s)
+ self.gam1i = data['gam1i'].reshape(s)
+ self.gam2r = data['gam2r'].reshape(s)
+ self.gam2i = data['gam2i'].reshape(s)
+ self.gam3r = data['gam3r'].reshape(s)
+ self.gam3i = data['gam3i'].reshape(s)
+ # Read old output files without error.
+ if 'sigma_gam' in data.dtype.names: # pragma: no cover
+ self.vargam0 = data['sigma_gam'].reshape(s)**2
+ self.vargam1 = data['sigma_gam'].reshape(s)**2
+ self.vargam2 = data['sigma_gam'].reshape(s)**2
+ self.vargam3 = data['sigma_gam'].reshape(s)**2
+ else:
+ self.vargam0 = data['sigma_gam0'].reshape(s)**2
+ self.vargam1 = data['sigma_gam1'].reshape(s)**2
+ self.vargam2 = data['sigma_gam2'].reshape(s)**2
+ self.vargam3 = data['sigma_gam3'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.ntri = data['ntri'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+ self.npatch3 = params.get('npatch3', 1)
+
+ @classmethod
+ def _calculateT(cls, s, t, k1, k2, k3):
+ # First calculate q values:
+ q1 = (s+t)/3.
+ q2 = q1-t
+ q3 = q1-s
+
+ # |qi|^2 shows up a lot, so save these.
+ # The a stands for "absolute", and the ^2 part is implicit.
+ a1 = np.abs(q1)**2
+ a2 = np.abs(q2)**2
+ a3 = np.abs(q3)**2
+ a123 = a1*a2*a3
+
+ # These combinations also appear multiple times.
+ # The b doesn't stand for anything. It's just the next letter after a.
+ b1 = np.conjugate(q1)**2*q2*q3
+ b2 = np.conjugate(q2)**2*q1*q3
+ b3 = np.conjugate(q3)**2*q1*q2
+
+ if k1==1 and k2==1 and k3==1:
+
+ # Some factors we use multiple times
+ expfactor = -np.exp(-(a1 + a2 + a3)/2)
+
+ # JBJ Equation 51
+ # Note that we actually accumulate the Gammas with a different choice for
+ # alpha_i. We accumulate the shears relative to the q vectors, not relative to s.
+ # cf. JBJ Equation 41 and footnote 3. The upshot is that we multiply JBJ's formulae
+ # be (q1q2q3)^2 / |q1q2q3|^2 for T0 and (q1*q2q3)^2/|q1q2q3|^2 for T1.
+ # Then T0 becomes
+ # T0 = -(|q1 q2 q3|^2)/24 exp(-(|q1|^2+|q2|^2+|q3|^2)/2)
+ T0 = expfactor * a123 / 24
+
+ # JBJ Equation 52
+ # After the phase adjustment, T1 becomes:
+ # T1 = -[(|q1 q2 q3|^2)/24
+ # - (q1*^2 q2 q3)/9
+ # + (q1*^4 q2^2 q3^2 + 2 |q2 q3|^2 q1*^2 q2 q3)/(|q1 q2 q3|^2)/27
+ # ] exp(-(|q1|^2+|q2|^2+|q3|^2)/2)
+ T1 = expfactor * (a123 / 24 - b1 / 9 + (b1**2 + 2*a2*a3*b1) / (a123 * 27))
+ T2 = expfactor * (a123 / 24 - b2 / 9 + (b2**2 + 2*a1*a3*b2) / (a123 * 27))
+ T3 = expfactor * (a123 / 24 - b3 / 9 + (b3**2 + 2*a1*a2*b3) / (a123 * 27))
+
+ else:
+ # SKL Equation 63:
+ k1sq = k1*k1
+ k2sq = k2*k2
+ k3sq = k3*k3
+ Theta2 = ((k1sq*k2sq + k1sq*k3sq + k2sq*k3sq)/3.)**0.5
+ k1sq /= Theta2 # These are now what SKL calls theta_i^2 / Theta^2
+ k2sq /= Theta2
+ k3sq /= Theta2
+ Theta4 = Theta2*Theta2
+ Theta6 = Theta4*Theta2
+ S = k1sq * k2sq * k3sq
+
+ # SKL Equation 64:
+ Z = ((2*k2sq + 2*k3sq - k1sq) * a1 +
+ (2*k3sq + 2*k1sq - k2sq) * a2 +
+ (2*k1sq + 2*k2sq - k3sq) * a3) / (6*Theta2)
+ expfactor = -S * np.exp(-Z) / Theta4
+
+ # SKL Equation 65:
+ f1 = (k2sq+k3sq)/2 + (k2sq-k3sq)*(q2-q3)/(6*q1)
+ f2 = (k3sq+k1sq)/2 + (k3sq-k1sq)*(q3-q1)/(6*q2)
+ f3 = (k1sq+k2sq)/2 + (k1sq-k2sq)*(q1-q2)/(6*q3)
+ f1c = np.conjugate(f1)
+ f2c = np.conjugate(f2)
+ f3c = np.conjugate(f3)
+
+ # SKL Equation 69:
+ g1 = k2sq*k3sq + (k3sq-k2sq)*k1sq*(q2-q3)/(3*q1)
+ g2 = k3sq*k1sq + (k1sq-k3sq)*k2sq*(q3-q1)/(3*q2)
+ g3 = k1sq*k2sq + (k2sq-k1sq)*k3sq*(q1-q2)/(3*q3)
+ g1c = np.conjugate(g1)
+ g2c = np.conjugate(g2)
+ g3c = np.conjugate(g3)
+
+ # SKL Equation 62:
+ T0 = expfactor * a123 * f1c**2 * f2c**2 * f3c**2 / (24.*Theta6)
+
+ # SKL Equation 68:
+ T1 = expfactor * (
+ a123 * f1**2 * f2c**2 * f3c**2 / (24*Theta6) -
+ b1 * f1*f2c*f3c*g1c / (9*Theta4) +
+ (b1**2 * g1c**2 + 2*k2sq*k3sq*a2*a3*b1 * f2c * f3c) / (a123 * 27*Theta2))
+ T2 = expfactor * (
+ a123 * f1c**2 * f2**2 * f3c**2 / (24*Theta6) -
+ b2 * f1c*f2*f3c*g2c / (9*Theta4) +
+ (b2**2 * g2c**2 + 2*k1sq*k3sq*a1*a3*b2 * f1c * f3c) / (a123 * 27*Theta2))
+ T3 = expfactor * (
+ a123 * f1c**2 * f2c**2 * f3**2 / (24*Theta6) -
+ b3 * f1c*f2c*f3*g3c / (9*Theta4) +
+ (b3**2 * g3c**2 + 2*k1sq*k2sq*a1*a2*b3 * f1c * f2c) / (a123 * 27*Theta2))
+
+ return T0, T1, T2, T3
+
+[docs] @depr_pos_kwargs
+ def calculateMap3(self, *, R=None, k2=1, k3=1):
+ r"""Calculate the skewness of the aperture mass from the correlation function.
+
+ The equations for this come from Jarvis, Bernstein & Jain (2004, MNRAS, 352).
+ See their section 3, especially equations 51 and 52 for the :math:`T_i` functions,
+ equations 60 and 61 for the calculation of :math:`\langle \cal M^3 \rangle` and
+ :math:`\langle \cal M^2 M^* \rangle`, and equations 55-58 for how to convert
+ these to the return values.
+
+ If k2 or k3 != 1, then this routine calculates the generalization of the skewness
+ proposed by Schneider, Kilbinger & Lombardi (2005, A&A, 431):
+ :math:`\langle M_{ap}^3(R, k_2 R, k_3 R)\rangle` and related values.
+
+ If k2 = k3 = 1 (the default), then there are only 4 combinations of Map and Mx
+ that are relevant:
+
+ - map3 = :math:`\langle M_{ap}^3(R)\rangle`
+ - map2mx = :math:`\langle M_{ap}^2(R) M_\times(R)\rangle`,
+ - mapmx2 = :math:`\langle M_{ap}(R) M_\times(R)\rangle`
+ - mx3 = :math:`\langle M_{\rm \times}^3(R)\rangle`
+
+ However, if k2 or k3 != 1, then there are 8 combinations:
+
+ - map3 = :math:`\langle M_{ap}(R) M_{ap}(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mapmapmx = :math:`\langle M_{ap}(R) M_{ap}(k_2 R) M_\times(k_3 R)\rangle`
+ - mapmxmap = :math:`\langle M_{ap}(R) M_\times(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mxmapmap = :math:`\langle M_\times(R) M_{ap}(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mxmxmap = :math:`\langle M_\times(R) M_\times(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mxmapmx = :math:`\langle M_\times(R) M_{ap}(k_2 R) M_\times(k_3 R)\rangle`
+ - mapmxmx = :math:`\langle M_{ap}(R) M_\times(k_2 R) M_\times(k_3 R)\rangle`
+ - mx3 = :math:`\langle M_\times(R) M_\times(k_2 R) M_\times(k_3 R)\rangle`
+
+ To accommodate this full generality, we always return all 8 values, along with the
+ estimated variance (which is equal for each), even when k2 = k3 = 1.
+
+ .. note::
+
+ The formulae for the ``m2_uform`` = 'Schneider' definition of the aperture mass,
+ described in the documentation of `calculateMapSq`, are not known, so that is not an
+ option here. The calculations here use the definition that corresponds to
+ ``m2_uform`` = 'Crittenden'.
+
+ Parameters:
+ R (array): The R values at which to calculate the aperture mass statistics.
+ (default: None, which means use self.rnom1d)
+ k2 (float): If given, the ratio R2/R1 in the SKL formulae. (default: 1)
+ k3 (float): If given, the ratio R3/R1 in the SKL formulae. (default: 1)
+
+ Returns:
+ Tuple containing:
+
+ - map3 = array of :math:`\langle M_{ap}(R) M_{ap}(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mapmapmx = array of :math:`\langle M_{ap}(R) M_{ap}(k_2 R) M_\times(k_3 R)\rangle`
+ - mapmxmap = array of :math:`\langle M_{ap}(R) M_\times(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mxmapmap = array of :math:`\langle M_\times(R) M_{ap}(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mxmxmap = array of :math:`\langle M_\times(R) M_\times(k_2 R) M_{ap}(k_3 R)\rangle`
+ - mxmapmx = array of :math:`\langle M_\times(R) M_{ap}(k_2 R) M_\times(k_3 R)\rangle`
+ - mapmxmx = array of :math:`\langle M_{ap}(R) M_\times(k_2 R) M_\times(k_3 R)\rangle`
+ - mx3 = array of :math:`\langle M_\times(R) M_\times(k_2 R) M_\times(k_3 R)\rangle`
+ - varmap3 = array of variance estimates of the above values
+ """
+ # As in the calculateMapSq function, we Make s and t matrices, so we can eventually do the
+ # integral by doing a matrix product.
+ if R is None:
+ R = self.rnom1d
+
+ # Pick s = d2, so dlogs is bin_size
+ s = d2 = np.outer(1./R, self.meand2.ravel())
+
+ # We take t = d3, but we need the x and y components. (relative to s along x axis)
+ # cf. Figure 1 in JBJ.
+ # d1^2 = d2^2 + d3^2 - 2 d2 d3 cos(theta1)
+ # tx = d3 cos(theta1) = (d2^2 + d3^2 - d1^2)/2d2
+ # Simplify this using u=d3/d2 and v=(d1-d2)/d3
+ # = (d3^2 - (d1+d2)(d1-d2)) / 2d2
+ # = d3 (d3 - (d1+d2)v) / 2d2
+ # = d3 (u - (2+uv)v)/2
+ # = d3 (u - 2v - uv^2)/2
+ # = d3 (u(1-v^2)/2 - v)
+ # Note that v here is really |v|. We'll account for the sign of v in ty.
+ d3 = np.outer(1./R, self.meand3.ravel())
+ d1 = np.outer(1./R, self.meand1.ravel())
+ u = self.meanu.ravel()
+ v = self.meanv.ravel()
+ tx = d3*(0.5*u*(1-v**2) - np.abs(v))
+ # This form tends to be more stable near potentially degenerate triangles
+ # than tx = (d2*d2 + d3*d3 - d1*d1) / (2*d2)
+ # However, add a check to make sure.
+ bad = (tx <= -d3) | (tx >= d3)
+ if np.any(bad): # pragma: no cover
+ self.logger.warning("Warning: Detected some invalid triangles when computing Map^3")
+ self.logger.warning("Excluding these triangles from the integral.")
+ self.logger.debug("N bad points = %s",np.sum(bad))
+ self.logger.debug("d1[bad] = %s",d1[bad])
+ self.logger.debug("d2[bad] = %s",d2[bad])
+ self.logger.debug("d3[bad] = %s",d3[bad])
+ self.logger.debug("tx[bad] = %s",tx[bad])
+ bad = np.where(bad)
+ tx[bad] = 0 # for now to avoid nans
+ ty = np.sqrt(d3**2 - tx**2)
+ ty[:,self.meanv.ravel() > 0] *= -1.
+ t = tx + 1j * ty
+
+ # Next we need to construct the T values.
+ T0, T1, T2, T3 = self._calculateT(s,t,1.,k2,k3)
+
+ # Finally, account for the Jacobian in d^2t: jac = |J(tx, ty; u, v)|,
+ # since our Gammas are accumulated in s, u, v, not s, tx, ty.
+ # u = d3/d2, v = (d1-d2)/d3
+ # tx = d3 (u - 2v - uv^2)/2
+ # = s/2 (u^2 - 2uv - u^2v^2)
+ # dtx/du = s (u - v - uv^2)
+ # dtx/dv = -us (1 + uv)
+ # ty = sqrt(d3^2 - tx^2) = sqrt(u^2 s^2 - tx^2)
+ # dty/du = s^2 u/2ty (1-v^2) (2 + 3uv - u^2 + u^2v^2)
+ # dty/dv = s^2 u^2/2ty (1 + uv) (u - uv^2 - 2uv)
+ #
+ # After some algebra...
+ #
+ # J = s^3 u^2 (1+uv) / ty
+ # = d3^2 d1 / ty
+ #
+ jac = np.abs(d3*d3*d1/ty)
+ jac[bad] = 0. # Exclude any bad triangles from the integral.
+ d2t = jac * self.ubin_size * self.vbin_size / (2.*np.pi)
+ sds = s * s * self.bin_size # Remember bin_size is dln(s)
+ # Note: these are really d2t/2piR^2 and sds/R^2, which are what actually show up
+ # in JBJ equations 45 and 50.
+
+ T0 *= sds * d2t
+ T1 *= sds * d2t
+ T2 *= sds * d2t
+ T3 *= sds * d2t
+
+ # Now do the integral by taking the matrix products.
+ gam0 = self.gam0.ravel()
+ gam1 = self.gam1.ravel()
+ gam2 = self.gam2.ravel()
+ gam3 = self.gam3.ravel()
+ vargam0 = self.vargam0.ravel()
+ vargam1 = self.vargam1.ravel()
+ vargam2 = self.vargam2.ravel()
+ vargam3 = self.vargam3.ravel()
+ mmm = T0.dot(gam0)
+ mcmm = T1.dot(gam1)
+ mmcm = T2.dot(gam2)
+ mmmc = T3.dot(gam3)
+
+ # These accumulate the coefficients that are being dotted to gam0,1,2,3 respectively.
+ # Below, we will take the abs^2 and dot it to gam0,1,2,3 in each case to compute the
+ # total variance.
+ # Note: This assumes that gam0, gam1, gam2, gam3 have no covariance.
+ # This is not technically true, but I think it's approximately ok.
+ var0 = T0.copy()
+ var1 = T1.copy()
+ var2 = T2.copy()
+ var3 = T3.copy()
+
+ if k2 == 1 and k3 == 1:
+ mmm *= 6
+ mcmm += mmcm
+ mcmm += mmmc
+ mcmm *= 2
+ mmcm = mmmc = mcmm
+ var0 *= 6
+ var1 *= 6
+ var2 *= 6
+ var3 *= 6
+ else:
+ # Repeat the above for the other permutations
+ for (_k1, _k2, _k3, _mcmm, _mmcm, _mmmc) in [
+ (1,k3,k2,mcmm,mmmc,mmcm),
+ (k2,1,k3,mmcm,mcmm,mmmc),
+ (k2,k3,1,mmcm,mmmc,mcmm),
+ (k3,1,k2,mmmc,mcmm,mmcm),
+ (k3,k2,1,mmmc,mmcm,mcmm) ]:
+ T0, T1, T2, T3 = self._calculateT(s,t,_k1,_k2,_k3)
+ T0 *= sds * d2t
+ T1 *= sds * d2t
+ T2 *= sds * d2t
+ T3 *= sds * d2t
+ # Relies on numpy array overloading += to actually update in place.
+ mmm += T0.dot(gam0)
+ _mcmm += T1.dot(gam1)
+ _mmcm += T2.dot(gam2)
+ _mmmc += T3.dot(gam3)
+ var0 += T0
+ var1 += T1
+ var2 += T2
+ var3 += T3
+
+ map3 = 0.25 * np.real(mcmm + mmcm + mmmc + mmm)
+ mapmapmx = 0.25 * np.imag(mcmm + mmcm - mmmc + mmm)
+ mapmxmap = 0.25 * np.imag(mcmm - mmcm + mmmc + mmm)
+ mxmapmap = 0.25 * np.imag(-mcmm + mmcm + mmmc + mmm)
+ mxmxmap = 0.25 * np.real(mcmm + mmcm - mmmc - mmm)
+ mxmapmx = 0.25 * np.real(mcmm - mmcm + mmmc - mmm)
+ mapmxmx = 0.25 * np.real(-mcmm + mmcm + mmmc - mmm)
+ mx3 = 0.25 * np.imag(mcmm + mmcm + mmmc - mmm)
+
+ var0 /= 4
+ var1 /= 4
+ var2 /= 4
+ var3 /= 4
+
+ # Now finally add up the coefficient squared times each vargam element.
+ var = np.abs(var0**2).dot(vargam0)
+ var += np.abs(var1**2).dot(vargam1)
+ var += np.abs(var2**2).dot(vargam2)
+ var += np.abs(var3**2).dot(vargam3)
+
+ return map3, mapmapmx, mapmxmap, mxmapmap, mxmxmap, mxmapmx, mapmxmx, mx3, var
+
+[docs] @depr_pos_kwargs
+ def writeMap3(self, file_name, *, R=None, file_type=None, precision=None):
+ r"""Write the aperture mass skewness based on the correlation function to the
+ file, file_name.
+
+ The output file will include the following columns:
+
+ ========== ==========================================================
+ Column Description
+ ========== ==========================================================
+ R The aperture radius
+ Map3 An estimate of :math:`\langle M_{ap}^3\rangle(R)`
+ (cf. `calculateMap3`)
+ Map2Mx An estimate of :math:`\langle M_{ap}^2 M_\times\rangle(R)`
+ MapMx2 An estimate of :math:`\langle M_{ap} M_\times^2\rangle(R)`
+ Mx3 An estimate of :math:`\langle M_\times^3\rangle(R)`
+ sig_map The sqrt of the variance estimate of each of these
+ ========== ==========================================================
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ R (array): The R values at which to calculate the statistics.
+ (default: None, which means use self.rnom)
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ """
+ self.logger.info('Writing Map^3 from GGG correlations to %s',file_name)
+
+ if R is None:
+ R = self.rnom1d
+ stats = self.calculateMap3(R=R)
+ if precision is None:
+ precision = self.config.get('precision', 4)
+
+ col_names = ['R','Map3','Map2Mx', 'MapMx2', 'Mx3','sig_map']
+ columns = [ R, stats[0], stats[1], stats[4], stats[7], np.sqrt(stats[8]) ]
+ with make_writer(file_name, precision, file_type, logger=self.logger) as writer:
+ writer.write(col_names, columns)
+
+
+[docs]class GGGCrossCorrelation(BinnedCorr3):
+ r"""This class handles the calculation a 3-point shear-shear-shear cross-correlation
+ function.
+
+ For 3-point cross correlations, it matters which of the two or three fields falls on
+ each corner of the triangle. E.g. is field 1 on the corner opposite d1 (the longest
+ size of the triangle) or is it field 2 (or 3) there? This is in contrast to the 2-point
+ correlation where the symmetry of the situation means that it doesn't matter which point
+ is identified with each field. This makes it significantly more complicated to keep track
+ of all the relevant information for a 3-point cross correlation function.
+
+ The `GGGCorrelation` class holds a single set of :math:`\Gamma` functions describing all
+ possible triangles, parameterized according to their relative side lengths ordered as
+ d1 > d2 > d3.
+
+ For a cross-correlation of two fields: G1 - G1 - G2 (i.e. the G1 field is at two of the
+ corners and G2 is at one corner), then we need three sets of these :math:`\Gamma` functions
+ to capture all of the triangles, since the G2 points may be opposite d1 or d2 or d3.
+ For a cross-correlation of three fields: G1 - G2 - G3, we need six sets, to account for
+ all of the possible permutations relative to the triangle sides.
+
+ Therefore, this class holds 6 instances of `GGGCorrelation`, which in turn hold the
+ information about triangles in each of the relevant configurations. We name these:
+
+ Attributes:
+ g1g2g3: Triangles where G1 is opposite d1, G2 is opposite d2, G3 is opposite d3.
+ g1g3g2: Triangles where G1 is opposite d1, G3 is opposite d2, G2 is opposite d3.
+ g2g1g3: Triangles where G2 is opposite d1, G1 is opposite d2, G3 is opposite d3.
+ g2g3g1: Triangles where G2 is opposite d1, G3 is opposite d2, G1 is opposite d3.
+ g3g1g2: Triangles where G3 is opposite d1, G1 is opposite d2, G2 is opposite d3.
+ g3g2g1: Triangles where G3 is opposite d1, G2 is opposite d2, G1 is opposite d3.
+
+ If for instance G2 and G3 are the same field, then e.g. g1g2g3 and g1g3g2 will have
+ the same values.
+
+ Ojects of this class also hold the following attributes, which are identical in each of
+ the above GGGCorrelation instances.
+
+ Attributes:
+ nbins: The number of bins in logr where r = d2
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+ nubins: The number of bins in u where u = d3/d2
+ ubin_size: The size of the bins in u
+ min_u: The minimum u being considered
+ max_u: The maximum u being considered
+ nvbins: The number of bins in v where v = +-(d1-d2)/d3
+ vbin_size: The size of the bins in v
+ min_v: The minimum v being considered
+ max_v: The maximum v being considered
+ logr1d: The nominal centers of the nbins bins in log(r).
+ u1d: The nominal centers of the nubins bins in u.
+ v1d: The nominal centers of the nvbins bins in v.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_cross` directly,
+ then the units will not be applied to ``meanr`` or ``meanlogr`` until the `finalize`
+ function is called.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr3`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr3` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `GGGCrossCorrelation`. See class doc for details.
+ """
+ BinnedCorr3.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 3 # GData
+ self._ro._d2 = 3 # GData
+ self._ro._d3 = 3 # GData
+
+ self.g1g2g3 = GGGCorrelation(config, logger=logger, **kwargs)
+ self.g1g3g2 = GGGCorrelation(config, logger=logger, **kwargs)
+ self.g2g1g3 = GGGCorrelation(config, logger=logger, **kwargs)
+ self.g2g3g1 = GGGCorrelation(config, logger=logger, **kwargs)
+ self.g3g1g2 = GGGCorrelation(config, logger=logger, **kwargs)
+ self.g3g2g1 = GGGCorrelation(config, logger=logger, **kwargs)
+ self._all = [self.g1g2g3, self.g1g3g2, self.g2g1g3, self.g2g3g1, self.g3g1g2, self.g3g2g1]
+
+ self.logger.debug('Finished building GGGCrossCorr')
+
+[docs] def __eq__(self, other):
+ """Return whether two `GGGCrossCorrelation` instances are equal"""
+ return (isinstance(other, GGGCrossCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nubins == other.nubins and
+ self.ubin_size == other.ubin_size and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v and
+ self.nvbins == other.nvbins and
+ self.vbin_size == other.vbin_size and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ self.g1g2g3 == other.g1g2g3 and
+ self.g1g3g2 == other.g1g3g2 and
+ self.g2g1g3 == other.g2g1g3 and
+ self.g2g3g1 == other.g2g3g1 and
+ self.g3g1g2 == other.g3g1g2 and
+ self.g3g2g1 == other.g3g2g1)
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = GGGCrossCorrelation.__new__(GGGCrossCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, GGGCorrelation):
+ ret.__dict__[key] = item.copy()
+ else:
+ ret.__dict__[key] = item
+ # This needs to be the new list:
+ ret._all = [ret.g1g2g3, ret.g1g3g2, ret.g2g1g3, ret.g2g3g1, ret.g3g1g2, ret.g3g2g1]
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_cross12(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process two catalogs, accumulating the 3pt cross-correlation, where one of the
+ points in each triangle come from the first catalog, and two come from the second.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ .. note::
+
+ This only adds to the attributes g1g2g3, g2g1g3, g2g3g1, not the ones where
+ 3 comes before 2. When running this via the regular `process` method, it will
+ combine them at the end to make sure g1g2g3 == g1g3g2, etc. for a complete
+ calculation of the 1-2 cross-correlation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process. (1 point in each triangle will come
+ from this catalog.)
+ cat2 (Catalog): The second catalog to process. (2 points in each triangle will come
+ from this catalog.)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process GGG (1-2) cross-correlations')
+ else:
+ self.logger.info('Starting process GGG (1-2) cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ for ggg in self._all:
+ ggg._set_metric(self.metric, self.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 3 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross12(self.g1g2g3.corr, self.g2g1g3.corr, self.g2g3g1.corr,
+ f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords,
+ self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, cat3, *, metric=None, num_threads=None):
+ """Process a set of three catalogs, accumulating the 3pt cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ cat3 (Catalog): The third catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '' and cat3.name == '':
+ self.logger.info('Starting process GGG cross-correlations')
+ else:
+ self.logger.info('Starting process GGG cross-correlations for cats %s, %s, %s.',
+ cat1.name, cat2.name, cat3.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords, cat3.coords)
+ for ggg in self._all:
+ ggg._set_metric(self.metric, self.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f3 = cat3.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 3,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross3(self.g1g2g3.corr, self.g1g3g2.corr,
+ self.g2g1g3.corr, self.g2g3g1.corr,
+ self.g3g1g2.corr, self.g3g2g1.corr,
+ f1.data, f2.data, f3.data, self.output_dots,
+ f1._d, f2._d, f3._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ for ggg in self._all:
+ ggg._finalize()
+
+[docs] def finalize(self, varg1, varg2, varg3):
+ """Finalize the calculation of the correlation function.
+
+ The `process_cross` command accumulate values in each bin, so they can be called
+ multiple times if appropriate. Afterwards, this command finishes the calculation
+ by dividing by the total weight.
+
+ Parameters:
+ varg1 (float): The shear variance for the first field that was correlated.
+ varg2 (float): The shear variance for the second field that was correlated.
+ varg3 (float): The shear variance for the third field that was correlated.
+ """
+ self.g1g2g3.finalize(varg1,varg2,varg3)
+ self.g1g3g2.finalize(varg1,varg3,varg2)
+ self.g2g1g3.finalize(varg2,varg1,varg3)
+ self.g2g3g1.finalize(varg2,varg3,varg1)
+ self.g3g1g2.finalize(varg3,varg1,varg2)
+ self.g3g2g1.finalize(varg3,varg2,varg1)
+
+ @property
+ def nonzero(self):
+ """Return if there are any values accumulated yet. (i.e. ntri > 0)
+ """
+ return any([ggg.nonzero for ggg in self._all])
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ for ggg in self._all:
+ ggg._clear()
+
+[docs] def __iadd__(self, other):
+ """Add a second `GGGCrossCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `GGGCrossCorrelation` objects should not have had
+ `finalize` called yet. Then, after adding them together, you should call `finalize`
+ on the sum.
+ """
+ if not isinstance(other, GGGCrossCorrelation):
+ raise TypeError("Can only add another GGGCrossCorrelation object")
+ self.g1g2g3 += other.g1g2g3
+ self.g1g3g2 += other.g1g3g2
+ self.g2g1g3 += other.g2g1g3
+ self.g2g3g1 += other.g2g3g1
+ self.g3g1g2 += other.g3g1g2
+ self.g3g2g1 += other.g3g2g1
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ for i, ggg in enumerate(self._all):
+ ggg._sum([c._all[i] for c in others])
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2, cat3=None, *, metric=None, num_threads=None,
+ comm=None, low_mem=False, initialize=True, finalize=True):
+ """Accumulate the cross-correlation of the points in the given Catalogs: cat1, cat2, cat3.
+
+ - If 2 arguments are given, then compute a cross-correlation function with the
+ first catalog taking one corner of the triangles, and the second taking two corners.
+ - If 3 arguments are given, then compute a three-way cross-correlation function.
+
+ All arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first G field.
+ cat2 (Catalog): A catalog or list of catalogs for the second G field.
+ cat3 (Catalog): A catalog or list of catalogs for the third G field.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr3.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+ self._process12 = False
+
+ if not isinstance(cat1,list): cat1 = cat1.get_patches()
+ if not isinstance(cat2,list): cat2 = cat2.get_patches()
+ if cat3 is not None and not isinstance(cat3,list): cat3 = cat3.get_patches()
+
+ if cat3 is None:
+ self._process12 = True
+ self._process_all_cross12(cat1, cat2, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, cat3, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if self._process12:
+ # Then some of the processing involved a cross12 calculation.
+ # This means that spots 2 and 3 should not be distinguished.
+ # Combine the relevant arrays.
+ self.g1g2g3 += self.g1g3g2
+ self.g2g1g3 += self.g3g1g2
+ self.g2g3g1 += self.g3g2g1
+ # Copy back by doing clear and +=.
+ # This makes sure the coords and metric are set properly.
+ self.g1g3g2.clear()
+ self.g3g1g2.clear()
+ self.g3g2g1.clear()
+ self.g1g3g2 += self.g1g2g3
+ self.g3g1g2 += self.g2g1g3
+ self.g3g2g1 += self.g2g3g1
+
+ varg1 = calculateVarG(cat1, low_mem=low_mem)
+ varg2 = calculateVarG(cat2, low_mem=low_mem)
+ self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1))
+ self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2))
+ if cat3 is None:
+ varg3 = varg2
+ else:
+ varg3 = calculateVarG(cat3, low_mem=low_mem)
+ self.logger.info("varg3 = %f: sig_g = %f",varg3,math.sqrt(varg3))
+ self.finalize(varg1,varg2,varg3)
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ In this case, the concatenation of zeta.ravel() for each combination in the following
+ order: g1g2g3, g1g3g2, g2g1g3, g2g3g1, g3g1g2, g3g2g1.
+ """
+ return np.concatenate([ggg.getStat() for ggg in self._all])
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ In this case, the concatenation of getWeight() for each combination in the following
+ order: g1g2g3, g1g3g2, g2g1g3, g2g3g1, g3g1g2, g3g2g1.
+ """
+ return np.concatenate([ggg.getWeight() for ggg in self._all])
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the cross-correlation functions to the file, file_name.
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing GGG cross-correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ names = [ 'g1g2g3', 'g1g3g2', 'g2g1g3', 'g2g3g1', 'g3g1g2', 'g3g2g1' ]
+ for name, corr in zip(names, self._all):
+ corr._write(writer, name, write_patch_results)
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `GGGCrossCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading GGG cross-correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ names = [ 'g1g2g3', 'g1g3g2', 'g2g1g3', 'g2g3g1', 'g3g1g2', 'g3g2g1' ]
+ for name, corr in zip(names, self._all):
+ corr._read(reader, name)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: kgcorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarG, calculateVarK
+from .binnedcorr2 import BinnedCorr2
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class KGCorrelation(BinnedCorr2):
+ r"""This class handles the calculation and storage of a 2-point kappa-shear correlation
+ function.
+
+ .. note::
+
+ While we use the term kappa (:math:`\kappa`) here and the letter K in various places,
+ in fact any scalar field will work here. For example, you can use this to compute
+ correlations of some survey property, such as seeing, with shear, where "kappa" would
+ really be the measured property, e.g. the observed sizes of the stars.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+
+ In addition, the following attributes are numpy arrays of length (nbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ meanr: The (weighted) mean value of r for the pairs in each bin.
+ If there are no pairs in a bin, then exp(logr) will be used instead.
+ meanlogr: The (weighted) mean value of log(r) for the pairs in each bin.
+ If there are no pairs in a bin, then logr will be used instead.
+ xi: The correlation function, :math:`\xi(r) = \langle \kappa\, \gamma_T\rangle`.
+ xi_im: The imaginary part of :math:`\xi(r)`.
+ varxi: An estimate of the variance of :math:`\xi`
+ weight: The total weight in each bin.
+ npairs: The number of pairs going into each bin (including pairs where one or
+ both objects have w=0).
+ cov: An estimate of the full covariance matrix.
+
+ .. note::
+
+ The default method for estimating the variance and covariance attributes (``varxi``,
+ and ``cov``) is 'shot', which only includes the shape noise propagated into the final
+ correlation. This does not include sample variance, so it is always an underestimate of
+ the actual variance. To get better estimates, you need to set ``var_method`` to something
+ else and use patches in the input catalog(s). cf. `Covariance Estimates`.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_cross`,
+ then the units will not be applied to ``meanr`` or ``meanlogr`` until the `finalize`
+ function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> kg = treecorr.KGCorrelation(config)
+ >>> kg.process(cat1,cat2) # Calculate the cross-correlation
+ >>> kg.write(file_name) # Write out to a file.
+ >>> xi = kg.xi # Or access the correlation function directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr2`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr2` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `KGCorrelation`. See class doc for details.
+ """
+ BinnedCorr2.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 2 # KData
+ self._ro._d2 = 3 # GData
+ self.xi = np.zeros_like(self.rnom, dtype=float)
+ self.xi_im = np.zeros_like(self.rnom, dtype=float)
+ self.varxi = np.zeros_like(self.rnom, dtype=float)
+ self.meanr = np.zeros_like(self.rnom, dtype=float)
+ self.meanlogr = np.zeros_like(self.rnom, dtype=float)
+ self.weight = np.zeros_like(self.rnom, dtype=float)
+ self.npairs = np.zeros_like(self.rnom, dtype=float)
+ self.logger.debug('Finished building KGCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr2(
+ self._d1, self._d2, self._bintype,
+ self._min_sep,self._max_sep,self._nbins,self._bin_size,self.b,
+ self.min_rpar, self.max_rpar, self.xperiod, self.yperiod, self.zperiod,
+ dp(self.xi),dp(self.xi_im), dp(None), dp(None),
+ dp(self.meanr),dp(self.meanlogr),dp(self.weight),dp(self.npairs))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr2(self.corr, self._d1, self._d2, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `KGCorrelation` instances are equal"""
+ return (isinstance(other, KGCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.min_rpar == other.min_rpar and
+ self.max_rpar == other.max_rpar and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meanr, other.meanr) and
+ np.array_equal(self.meanlogr, other.meanlogr) and
+ np.array_equal(self.xi, other.xi) and
+ np.array_equal(self.xi_im, other.xi_im) and
+ np.array_equal(self.varxi, other.varxi) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.npairs, other.npairs))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = KGCorrelation.__new__(KGCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process KG cross-correlations')
+ else:
+ self.logger.info('Starting process KG cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross2(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_pairwise(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation, only using
+ the corresponding pairs of objects in each catalog.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ import warnings
+ warnings.warn("The process_pairwise function is slated to be removed in a future version. "+
+ "If you are actually using this function usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process KG pairwise-correlations')
+ else:
+ self.logger.info('Starting process KG pairwise-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+
+ f1 = cat1.getKSimpleField()
+ f2 = cat2.getGSimpleField()
+
+ _lib.ProcessPair(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.xi[mask1] /= self.weight[mask1]
+ self.xi_im[mask1] /= self.weight[mask1]
+ self.meanr[mask1] /= self.weight[mask1]
+ self.meanlogr[mask1] /= self.weight[mask1]
+
+ # Update the units of meanr, meanlogr
+ self._apply_units(mask1)
+
+ # Use meanr, meanlogr when available, but set to nominal when no pairs in bin.
+ self.meanr[mask2] = self.rnom[mask2]
+ self.meanlogr[mask2] = self.logr[mask2]
+
+[docs] def finalize(self, vark, varg):
+ """Finalize the calculation of the correlation function.
+
+ The `process_cross` command accumulates values in each bin, so it can be called
+ multiple times if appropriate. Afterwards, this command finishes the calculation
+ by dividing each column by the total weight.
+
+ Parameters:
+ vark (float): The kappa variance for the first field.
+ varg (float): The shear variance per component for the second field.
+ """
+ self._finalize()
+ self._var_num = vark * varg
+ self.cov = self.estimate_cov(self.var_method)
+ self.varxi.ravel()[:] = self.cov.diagonal()
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.xi.ravel()[:] = 0
+ self.xi_im.ravel()[:] = 0
+ self.meanr.ravel()[:] = 0
+ self.meanlogr.ravel()[:] = 0
+ self.weight.ravel()[:] = 0
+ self.npairs.ravel()[:] = 0
+
+[docs] def __iadd__(self, other):
+ """Add a second `KGCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `KGCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, KGCorrelation):
+ raise TypeError("Can only add another KGCorrelation object")
+ if not (self._nbins == other._nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep):
+ raise ValueError("KGCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords)
+ self.xi.ravel()[:] += other.xi.ravel()[:]
+ self.xi_im.ravel()[:] += other.xi_im.ravel()[:]
+ self.meanr.ravel()[:] += other.meanr.ravel()[:]
+ self.meanlogr.ravel()[:] += other.meanlogr.ravel()[:]
+ self.weight.ravel()[:] += other.weight.ravel()[:]
+ self.npairs.ravel()[:] += other.npairs.ravel()[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.xi for c in others], axis=0, out=self.xi)
+ np.sum([c.xi_im for c in others], axis=0, out=self.xi_im)
+ np.sum([c.meanr for c in others], axis=0, out=self.meanr)
+ np.sum([c.meanlogr for c in others], axis=0, out=self.meanlogr)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.npairs for c in others], axis=0, out=self.npairs)
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2, *, metric=None, num_threads=None, comm=None, low_mem=False,
+ initialize=True, finalize=True):
+ """Compute the correlation function.
+
+ Both arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the K field.
+ cat2 (Catalog): A catalog or list of catalogs for the G field.
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr2.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+
+ self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ vark = calculateVarK(cat1, low_mem=low_mem)
+ varg = calculateVarG(cat2, low_mem=low_mem)
+ self.logger.info("vark = %f: sig_k = %f",vark,math.sqrt(vark))
+ self.logger.info("varg = %f: sig_sn (per component) = %f",varg,math.sqrt(varg))
+ self.finalize(vark,varg)
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ The output file will include the following columns:
+
+ ========== ========================================================
+ Column Description
+ ========== ========================================================
+ r_nom The nominal center of the bin in r
+ meanr The mean value :math:`\langle r\rangle` of pairs that
+ fell into each bin
+ meanlogr The mean value :math:`\langle \log(r)\rangle` of pairs
+ that fell into each bin
+ kgamT The real part of correlation function,
+ :math:`\langle \kappa\, \gamma_T\rangle`
+ kgamX The imag part of correlation function,
+ :math:`\langle \kappa\, \gamma_\times\rangle`
+ sigma The sqrt of the variance estimate of both of these
+ weight The total weight contributing to each bin
+ npairs The total number of pairs in each bin
+ ========== ========================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing KG correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return ['r_nom','meanr','meanlogr','kgamT','kgamX','sigma','weight','npairs']
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.meanr, self.meanlogr,
+ self.xi, self.xi_im, np.sqrt(self.varxi),
+ self.weight, self.npairs ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `KGCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading KG correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ self.meanr = data['meanR'].reshape(s)
+ self.meanlogr = data['meanlogR'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meanr = data['meanr'].reshape(s)
+ self.meanlogr = data['meanlogr'].reshape(s)
+ self.xi = data['kgamT'].reshape(s)
+ self.xi_im = data['kgamX'].reshape(s)
+ self.varxi = data['sigma'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.npairs = data['npairs'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: kkcorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarK
+from .binnedcorr2 import BinnedCorr2
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class KKCorrelation(BinnedCorr2):
+ r"""This class handles the calculation and storage of a 2-point kappa-kappa correlation
+ function.
+
+ .. note::
+
+ While we use the term kappa (:math:`\kappa`) here and the letter K in various places,
+ in fact any scalar field will work here. For example, you can use this to compute
+ correlations of the CMB temperature fluctuations, where "kappa" would really be
+ :math:`\Delta T`.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+
+ In addition, the following attributes are numpy arrays of length (nbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ meanr: The (weighted) mean value of r for the pairs in each bin.
+ If there are no pairs in a bin, then exp(logr) will be used instead.
+ meanlogr: The (weighted) mean value of log(r) for the pairs in each bin.
+ If there are no pairs in a bin, then logr will be used instead.
+ xi: The correlation function, :math:`\xi(r)`
+ varxi: An estimate of the variance of :math:`\xi`
+ weight: The total weight in each bin.
+ npairs: The number of pairs going into each bin (including pairs where one or
+ both objects have w=0).
+ cov: An estimate of the full covariance matrix.
+
+ .. note::
+
+ The default method for estimating the variance and covariance attributes (``varxi``,
+ and ``cov``) is 'shot', which only includes the shot noise propagated into the final
+ correlation. This does not include sample variance, so it is always an underestimate of
+ the actual variance. To get better estimates, you need to set ``var_method`` to something
+ else and use patches in the input catalog(s). cf. `Covariance Estimates`.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_auto` and/or
+ `process_cross`, then the units will not be applied to ``meanr`` or ``meanlogr`` until
+ the `finalize` function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> kk = treecorr.KKCorrelation(config)
+ >>> kk.process(cat) # For auto-correlation.
+ >>> kk.process(cat1,cat2) # For cross-correlation.
+ >>> kk.write(file_name) # Write out to a file.
+ >>> xi = kk.xi # Or access the correlation function directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr2`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr2` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `KKCorrelation`. See class doc for details.
+ """
+ BinnedCorr2.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 2 # KData
+ self._ro._d2 = 2 # KData
+ self.xi = np.zeros_like(self.rnom, dtype=float)
+ self.varxi = np.zeros_like(self.rnom, dtype=float)
+ self.meanr = np.zeros_like(self.rnom, dtype=float)
+ self.meanlogr = np.zeros_like(self.rnom, dtype=float)
+ self.weight = np.zeros_like(self.rnom, dtype=float)
+ self.npairs = np.zeros_like(self.rnom, dtype=float)
+ self.logger.debug('Finished building KKCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr2(
+ self._d1, self._d2, self._bintype,
+ self._min_sep,self._max_sep,self._nbins,self._bin_size,self.b,
+ self.min_rpar, self.max_rpar, self.xperiod, self.yperiod, self.zperiod,
+ dp(self.xi), dp(None), dp(None), dp(None),
+ dp(self.meanr),dp(self.meanlogr),dp(self.weight),dp(self.npairs))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr2(self.corr, self._d1, self._d2, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `KKCorrelation` instances are equal"""
+ return (isinstance(other, KKCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.min_rpar == other.min_rpar and
+ self.max_rpar == other.max_rpar and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meanr, other.meanr) and
+ np.array_equal(self.meanlogr, other.meanlogr) and
+ np.array_equal(self.xi, other.xi) and
+ np.array_equal(self.varxi, other.varxi) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.npairs, other.npairs))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = KKCorrelation.__new__(KKCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_auto(self, cat, *, metric=None, num_threads=None):
+ """Process a single catalog, accumulating the auto-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat (Catalog): The catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat.name == '':
+ self.logger.info('Starting process KK auto-correlations')
+ else:
+ self.logger.info('Starting process KK auto-correlations for cat %s.', cat.name)
+
+ self._set_metric(metric, cat.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ field = cat.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method, brute=bool(self.brute),
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',field.nTopLevelNodes)
+ _lib.ProcessAuto2(self.corr, field.data, self.output_dots,
+ field._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process KK cross-correlations')
+ else:
+ self.logger.info('Starting process KK cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross2(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_pairwise(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation, only using
+ the corresponding pairs of objects in each catalog.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ import warnings
+ warnings.warn("The process_pairwise function is slated to be removed in a future version. "+
+ "If you are actually using this function usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process KK pairwise-correlations')
+ else:
+ self.logger.info('Starting process KK pairwise-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+
+ f1 = cat1.getKSimpleField()
+ f2 = cat2.getKSimpleField()
+
+ _lib.ProcessPair(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.xi[mask1] /= self.weight[mask1]
+ self.meanr[mask1] /= self.weight[mask1]
+ self.meanlogr[mask1] /= self.weight[mask1]
+
+ # Update the units of meanlogr
+ self._apply_units(mask1)
+
+ # Use meanlogr when available, but set to nominal when no pairs in bin.
+ self.meanr[mask2] = self.rnom[mask2]
+ self.meanlogr[mask2] = self.logr[mask2]
+
+[docs] def finalize(self, vark1, vark2):
+ """Finalize the calculation of the correlation function.
+
+ The `process_auto` and `process_cross` commands accumulate values in each bin,
+ so they can be called multiple times if appropriate. Afterwards, this command
+ finishes the calculation by dividing each column by the total weight.
+
+ Parameters:
+ vark1 (float): The kappa variance for the first field.
+ vark2 (float): The kappa variance for the second field.
+ """
+ self._finalize()
+ self._var_num = vark1 * vark2
+ self.cov = self.estimate_cov(self.var_method)
+ self.varxi.ravel()[:] = self.cov.diagonal()
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.xi.ravel()[:] = 0
+ self.meanr.ravel()[:] = 0
+ self.meanlogr.ravel()[:] = 0
+ self.weight.ravel()[:] = 0
+ self.npairs.ravel()[:] = 0
+
+[docs] def __iadd__(self, other):
+ """Add a second `KKCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `KKCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, KKCorrelation):
+ raise TypeError("Can only add another KKCorrelation object")
+ if not (self._nbins == other._nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep):
+ raise ValueError("KKCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords)
+ self.xi.ravel()[:] += other.xi.ravel()[:]
+ self.meanr.ravel()[:] += other.meanr.ravel()[:]
+ self.meanlogr.ravel()[:] += other.meanlogr.ravel()[:]
+ self.weight.ravel()[:] += other.weight.ravel()[:]
+ self.npairs.ravel()[:] += other.npairs.ravel()[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.xi for c in others], axis=0, out=self.xi)
+ np.sum([c.meanr for c in others], axis=0, out=self.meanr)
+ np.sum([c.meanlogr for c in others], axis=0, out=self.meanlogr)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.npairs for c in others], axis=0, out=self.npairs)
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2=None, *, metric=None, num_threads=None, comm=None, low_mem=False,
+ initialize=True, finalize=True):
+ """Compute the correlation function.
+
+ - If only 1 argument is given, then compute an auto-correlation function.
+ - If 2 arguments are given, then compute a cross-correlation function.
+
+ Both arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first K field.
+ cat2 (Catalog): A catalog or list of catalogs for the second K field, if any.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr2.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if cat2 is not None and not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+
+ if cat2 is None:
+ self._process_all_auto(cat1, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if cat2 is None:
+ vark1 = calculateVarK(cat1, low_mem=low_mem)
+ vark2 = vark1
+ self.logger.info("vark = %f: sig_k = %f",vark1,math.sqrt(vark1))
+ else:
+ vark1 = calculateVarK(cat1, low_mem=low_mem)
+ vark2 = calculateVarK(cat2, low_mem=low_mem)
+ self.logger.info("vark1 = %f: sig_k = %f",vark1,math.sqrt(vark1))
+ self.logger.info("vark2 = %f: sig_k = %f",vark2,math.sqrt(vark2))
+ self.finalize(vark1,vark2)
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ The output file will include the following columns:
+
+ ========== ========================================================
+ Column Description
+ ========== ========================================================
+ r_nom The nominal center of the bin in r
+ meanr The mean value :math:`\langle r \rangle` of pairs that
+ fell into each bin
+ meanlogr The mean value :math:`\langle \log(r) \rangle` of pairs
+ that fell into each bin
+ xi The estimate of the correlation function xi(r)
+ sigma_xi The sqrt of the variance estimate of xi(r)
+ weight The total weight contributing to each bin
+ npairs The total number of pairs in each bin
+ ========== ========================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing KK correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return ['r_nom','meanr','meanlogr','xi','sigma_xi','weight','npairs']
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.meanr, self.meanlogr,
+ self.xi, np.sqrt(self.varxi), self.weight, self.npairs ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `KKCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading KK correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ self.meanr = data['meanR'].reshape(s)
+ self.meanlogr = data['meanlogR'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meanr = data['meanr'].reshape(s)
+ self.meanlogr = data['meanlogr'].reshape(s)
+ self.xi = data['xi'].reshape(s)
+ self.varxi = data['sigma_xi'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.npairs = data['npairs'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: nnncorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarK
+from .binnedcorr3 import BinnedCorr3
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class KKKCorrelation(BinnedCorr3):
+ r"""This class handles the calculation and storage of a 3-point kappa-kappa-kappa correlation
+ function.
+
+ .. note::
+
+ While we use the term kappa (:math:`\kappa`) here and the letter K in various places,
+ in fact any scalar field will work here. For example, you can use this to compute
+ correlations of the CMB temperature fluctuations, where "kappa" would really be
+ :math:`\Delta T`.
+
+ See the doc string of `BinnedCorr3` for a description of how the triangles are binned.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr where r = d2
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+ nubins: The number of bins in u where u = d3/d2
+ ubin_size: The size of the bins in u
+ min_u: The minimum u being considered
+ max_u: The maximum u being considered
+ nvbins: The number of bins in v where v = +-(d1-d2)/d3
+ vbin_size: The size of the bins in v
+ min_v: The minimum v being considered
+ max_v: The maximum v being considered
+ logr1d: The nominal centers of the nbins bins in log(r).
+ u1d: The nominal centers of the nubins bins in u.
+ v1d: The nominal centers of the nvbins bins in v.
+
+ In addition, the following attributes are numpy arrays whose shape is (nbins, nubins, nvbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ u: The nominal center of the bin in u.
+ v: The nominal center of the bin in v.
+ meand1: The (weighted) mean value of d1 for the triangles in each bin.
+ meanlogd1: The mean value of log(d1) for the triangles in each bin.
+ meand2: The (weighted) mean value of d2 (aka r) for the triangles in each bin.
+ meanlogd2: The mean value of log(d2) for the triangles in each bin.
+ meand2: The (weighted) mean value of d3 for the triangles in each bin.
+ meanlogd2: The mean value of log(d3) for the triangles in each bin.
+ meanu: The mean value of u for the triangles in each bin.
+ meanv: The mean value of v for the triangles in each bin.
+ zeta: The correlation function, :math:`\zeta(r,u,v)`.
+ varzeta: The variance of :math:`\zeta`, only including the shot noise propagated into
+ the final correlation. This does not include sample variance, so it is always
+ an underestimate of the actual variance.
+ weight: The total weight in each bin.
+ ntri: The number of triangles going into each bin (including those where one or
+ more objects have w=0).
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_auto` and/or
+ `process_cross`, then the units will not be applied to ``meanr`` or ``meanlogr`` until
+ the `finalize` function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> kkk = treecorr.KKKCorrelation(config)
+ >>> kkk.process(cat) # For auto-correlation.
+ >>> kkk.process(cat1,cat2,cat3) # For cross-correlation.
+ >>> kkk.write(file_name) # Write out to a file.
+ >>> zeta = kkk.zeta # To access zeta directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr3`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr3` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `KKKCorrelation`. See class doc for details.
+ """
+ BinnedCorr3.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 2 # KData
+ self._ro._d2 = 2 # KData
+ self._ro._d3 = 2 # KData
+ shape = self.logr.shape
+ self.zeta = np.zeros(shape, dtype=float)
+ self.varzeta = np.zeros(shape, dtype=float)
+ self.meand1 = np.zeros(shape, dtype=float)
+ self.meanlogd1 = np.zeros(shape, dtype=float)
+ self.meand2 = np.zeros(shape, dtype=float)
+ self.meanlogd2 = np.zeros(shape, dtype=float)
+ self.meand3 = np.zeros(shape, dtype=float)
+ self.meanlogd3 = np.zeros(shape, dtype=float)
+ self.meanu = np.zeros(shape, dtype=float)
+ self.meanv = np.zeros(shape, dtype=float)
+ self.weight = np.zeros(shape, dtype=float)
+ self.ntri = np.zeros(shape, dtype=float)
+ self.logger.debug('Finished building KKKCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr3(
+ self._d1, self._d2, self._d3, self._bintype,
+ self._min_sep,self._max_sep,self.nbins,self._bin_size,self.b,
+ self.min_u,self.max_u,self.nubins,self.ubin_size,self.bu,
+ self.min_v,self.max_v,self.nvbins,self.vbin_size,self.bv,
+ self.xperiod, self.yperiod, self.zperiod,
+ dp(self.zeta), dp(None), dp(None), dp(None),
+ dp(None), dp(None), dp(None), dp(None),
+ dp(self.meand1), dp(self.meanlogd1), dp(self.meand2), dp(self.meanlogd2),
+ dp(self.meand3), dp(self.meanlogd3), dp(self.meanu), dp(self.meanv),
+ dp(self.weight), dp(self.ntri))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr3(self.corr, self._d1, self._d2, self._d3, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `KKKCorrelation` instances are equal"""
+ return (isinstance(other, KKKCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nubins == other.nubins and
+ self.ubin_size == other.ubin_size and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v and
+ self.nvbins == other.nvbins and
+ self.vbin_size == other.vbin_size and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meand1, other.meand1) and
+ np.array_equal(self.meanlogd1, other.meanlogd1) and
+ np.array_equal(self.meand2, other.meand2) and
+ np.array_equal(self.meanlogd2, other.meanlogd2) and
+ np.array_equal(self.meand3, other.meand3) and
+ np.array_equal(self.meanlogd3, other.meanlogd3) and
+ np.array_equal(self.meanu, other.meanu) and
+ np.array_equal(self.meanv, other.meanv) and
+ np.array_equal(self.zeta, other.zeta) and
+ np.array_equal(self.varzeta, other.varzeta) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.ntri, other.ntri))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = KKKCorrelation.__new__(KKKCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_auto(self, cat, *, metric=None, num_threads=None):
+ """Process a single catalog, accumulating the auto-correlation.
+
+ This accumulates the auto-correlation for the given catalog. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ Parameters:
+ cat (Catalog): The catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat.name == '':
+ self.logger.info('Starting process KKK auto-correlations')
+ else:
+ self.logger.info('Starting process KKK auto-correlations for cat %s.', cat.name)
+
+ self._set_metric(metric, cat.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ field = cat.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method, brute=bool(self.brute),
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',field.nTopLevelNodes)
+ _lib.ProcessAuto3(self.corr, field.data, self.output_dots,
+ field._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross12(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process two catalogs, accumulating the 3pt cross-correlation, where one of the
+ points in each triangle come from the first catalog, and two come from the second.
+
+ This accumulates the cross-correlation for the given catalogs as part of a larger
+ auto-correlation calculation. E.g. when splitting up a large catalog into patches,
+ this is appropriate to use for the cross correlation between different patches
+ as part of the complete auto-correlation of the full catalog.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process. (1 point in each triangle will come
+ from this catalog.)
+ cat2 (Catalog): The second catalog to process. (2 points in each triangle will come
+ from this catalog.)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process KKK (1-2) cross-correlations')
+ else:
+ self.logger.info('Starting process KKK (1-2) cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 3 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross12(self.corr, self.corr, self.corr,
+ f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords,
+ self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, cat3, *, metric=None, num_threads=None):
+ """Process a set of three catalogs, accumulating the 3pt cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs as part of a larger
+ auto-correlation calculation. E.g. when splitting up a large catalog into patches,
+ this is appropriate to use for the cross correlation between different patches
+ as part of the complete auto-correlation of the full catalog.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ cat3 (Catalog): The third catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '' and cat3.name == '':
+ self.logger.info('Starting process KKK cross-correlations')
+ else:
+ self.logger.info('Starting process KKK cross-correlations for cats %s, %s, %s.',
+ cat1.name, cat2.name, cat3.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords, cat3.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f3 = cat3.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 3,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 6 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross3(self.corr, self.corr, self.corr,
+ self.corr, self.corr, self.corr,
+ f1.data, f2.data, f3.data, self.output_dots,
+ f1._d, f2._d, f3._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.zeta[mask1] /= self.weight[mask1]
+ self.meand1[mask1] /= self.weight[mask1]
+ self.meanlogd1[mask1] /= self.weight[mask1]
+ self.meand2[mask1] /= self.weight[mask1]
+ self.meanlogd2[mask1] /= self.weight[mask1]
+ self.meand3[mask1] /= self.weight[mask1]
+ self.meanlogd3[mask1] /= self.weight[mask1]
+ self.meanu[mask1] /= self.weight[mask1]
+ self.meanv[mask1] /= self.weight[mask1]
+
+ # Update the units
+ self._apply_units(mask1)
+
+ # Use meanlogr when available, but set to nominal when no triangles in bin.
+ self.meand2[mask2] = self.rnom[mask2]
+ self.meanlogd2[mask2] = self.logr[mask2]
+ self.meanu[mask2] = self.u[mask2]
+ self.meanv[mask2] = self.v[mask2]
+ self.meand3[mask2] = self.u[mask2] * self.meand2[mask2]
+ self.meanlogd3[mask2] = np.log(self.meand3[mask2])
+ self.meand1[mask2] = self.v[mask2] * self.meand3[mask2] + self.meand2[mask2]
+ self.meanlogd1[mask2] = np.log(self.meand1[mask2])
+
+[docs] def finalize(self, vark1, vark2, vark3):
+ """Finalize the calculation of the correlation function.
+
+ The `process_auto` and `process_cross` commands accumulate values in each bin,
+ so they can be called multiple times if appropriate. Afterwards, this command
+ finishes the calculation by dividing by the total weight.
+
+ Parameters:
+ vark1 (float): The kappa variance for the first field.
+ vark2 (float): The kappa variance for the second field.
+ vark3 (float): The kappa variance for the third field.
+ """
+ self._finalize()
+ self._var_num = vark1 * vark2 * vark3
+ self.cov = self.estimate_cov(self.var_method)
+ self.varzeta.ravel()[:] = self.cov.diagonal()
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.zeta[:,:,:] = 0.
+ self.varzeta[:,:,:] = 0.
+ self.meand1[:,:,:] = 0.
+ self.meanlogd1[:,:,:] = 0.
+ self.meand2[:,:,:] = 0.
+ self.meanlogd2[:,:,:] = 0.
+ self.meand3[:,:,:] = 0.
+ self.meanlogd3[:,:,:] = 0.
+ self.meanu[:,:,:] = 0.
+ self.meanv[:,:,:] = 0.
+ self.weight[:,:,:] = 0.
+ self.ntri[:,:,:] = 0.
+
+[docs] def __iadd__(self, other):
+ """Add a second `KKKCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `KKKCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, KKKCorrelation):
+ raise TypeError("Can only add another KKKCorrelation object")
+ if not (self.nbins == other.nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.nubins == other.nubins and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nvbins == other.nvbins and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v):
+ raise ValueError("KKKCorrelation to be added is not compatible with this one.")
+
+ if not other.nonzero: return self
+ self._set_metric(other.metric, other.coords, other.coords, other.coords)
+ self.zeta[:] += other.zeta[:]
+ self.meand1[:] += other.meand1[:]
+ self.meanlogd1[:] += other.meanlogd1[:]
+ self.meand2[:] += other.meand2[:]
+ self.meanlogd2[:] += other.meanlogd2[:]
+ self.meand3[:] += other.meand3[:]
+ self.meanlogd3[:] += other.meanlogd3[:]
+ self.meanu[:] += other.meanu[:]
+ self.meanv[:] += other.meanv[:]
+ self.weight[:] += other.weight[:]
+ self.ntri[:] += other.ntri[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.zeta for c in others], axis=0, out=self.zeta)
+ np.sum([c.meand1 for c in others], axis=0, out=self.meand1)
+ np.sum([c.meanlogd1 for c in others], axis=0, out=self.meanlogd1)
+ np.sum([c.meand2 for c in others], axis=0, out=self.meand2)
+ np.sum([c.meanlogd2 for c in others], axis=0, out=self.meanlogd2)
+ np.sum([c.meand3 for c in others], axis=0, out=self.meand3)
+ np.sum([c.meanlogd3 for c in others], axis=0, out=self.meanlogd3)
+ np.sum([c.meanu for c in others], axis=0, out=self.meanu)
+ np.sum([c.meanv for c in others], axis=0, out=self.meanv)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.ntri for c in others], axis=0, out=self.ntri)
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2=None, cat3=None, *, metric=None, num_threads=None,
+ comm=None, low_mem=False, initialize=True, finalize=True):
+ """Compute the 3pt correlation function.
+
+ - If only 1 argument is given, then compute an auto-correlation function.
+ - If 2 arguments are given, then compute a cross-correlation function with the
+ first catalog taking one corner of the triangles, and the second taking two corners.
+ - If 3 arguments are given, then compute a three-way cross-correlation function.
+
+ All arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ .. note::
+
+ For a correlation of multiple catalogs, it typically matters which corner of the
+ triangle comes from which catalog, which is not kept track of by this function.
+ The final accumulation will have d1 > d2 > d3 regardless of which input catalog
+ appears at each corner. The class which keeps track of which catalog appears
+ in each position in the triangle is `KKKCrossCorrelation`.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first K field.
+ cat2 (Catalog): A catalog or list of catalogs for the second K field.
+ (default: None)
+ cat3 (Catalog): A catalog or list of catalogs for the third K field.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr3.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if cat2 is not None and not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+ if cat3 is not None and not isinstance(cat3,list):
+ cat3 = cat3.get_patches(low_mem=low_mem)
+
+ if cat2 is None:
+ if cat3 is not None:
+ raise ValueError("For two catalog case, use cat1,cat2, not cat1,cat3")
+ self._process_all_auto(cat1, metric, num_threads, comm, low_mem)
+ elif cat3 is None:
+ self._process_all_cross12(cat1, cat2, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, cat3, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if cat2 is None:
+ vark1 = calculateVarK(cat1, low_mem=low_mem)
+ vark2 = vark1
+ vark3 = vark1
+ self.logger.info("vark = %f: sig_k = %f",vark1,math.sqrt(vark1))
+ elif cat3 is None:
+ vark1 = calculateVarK(cat1, low_mem=low_mem)
+ vark2 = calculateVarK(cat2, low_mem=low_mem)
+ vark3 = vark2
+ self.logger.info("vark1 = %f: sig_k = %f",vark1,math.sqrt(vark1))
+ self.logger.info("vark2 = %f: sig_k = %f",vark2,math.sqrt(vark2))
+ else:
+ vark1 = calculateVarK(cat1, low_mem=low_mem)
+ vark2 = calculateVarK(cat2, low_mem=low_mem)
+ vark3 = calculateVarK(cat3, low_mem=low_mem)
+ self.logger.info("vark1 = %f: sig_k = %f",vark1,math.sqrt(vark1))
+ self.logger.info("vark2 = %f: sig_k = %f",vark2,math.sqrt(vark2))
+ self.logger.info("vark3 = %f: sig_k = %f",vark3,math.sqrt(vark3))
+ self.finalize(vark1,vark2,vark3)
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ The output file will include the following columns:
+
+ ========== =============================================================
+ Column Description
+ ========== =============================================================
+ r_nom The nominal center of the bin in r = d2 where d1 > d2 > d3
+ u_nom The nominal center of the bin in u = d3/d2
+ v_nom The nominal center of the bin in v = +-(d1-d2)/d3
+ meand1 The mean value :math:`\langle d1\rangle` of triangles that
+ fell into each bin
+ meanlogd1 The mean value :math:`\langle \log(d1)\rangle` of triangles
+ that fell into each bin
+ meand2 The mean value :math:`\langle d2\rangle` of triangles that
+ fell into each bin
+ meanlogd2 The mean value :math:`\langle \log(d2)\rangle` of triangles
+ that fell into each bin
+ meand3 The mean value :math:`\langle d3\rangle` of triangles that
+ fell into each bin
+ meanlogd3 The mean value :math:`\langle \log(d3)\rangle` of triangles
+ that fell into each bin
+ meanu The mean value :math:`\langle u\rangle` of triangles that
+ fell into each bin
+ meanv The mean value :math:`\langle v\rangle` of triangles that
+ fell into each bin
+ zeta The estimator of :math:`\zeta(r,u,v)`
+ sigma_zeta The sqrt of the variance estimate of :math:`\zeta`
+ weight The total weight of triangles contributing to each bin
+ ntri The number of triangles contributing to each bin
+ ========== =============================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing KKK correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return [ 'r_nom', 'u_nom', 'v_nom',
+ 'meand1', 'meanlogd1', 'meand2', 'meanlogd2',
+ 'meand3', 'meanlogd3', 'meanu', 'meanv',
+ 'zeta', 'sigma_zeta', 'weight', 'ntri' ]
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.u, self.v,
+ self.meand1, self.meanlogd1, self.meand2, self.meanlogd2,
+ self.meand3, self.meanlogd3, self.meanu, self.meanv,
+ self.zeta, np.sqrt(self.varzeta), self.weight, self.ntri ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `KKKCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading KKK correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meand1 = data['meand1'].reshape(s)
+ self.meanlogd1 = data['meanlogd1'].reshape(s)
+ self.meand2 = data['meand2'].reshape(s)
+ self.meanlogd2 = data['meanlogd2'].reshape(s)
+ self.meand3 = data['meand3'].reshape(s)
+ self.meanlogd3 = data['meanlogd3'].reshape(s)
+ self.meanu = data['meanu'].reshape(s)
+ self.meanv = data['meanv'].reshape(s)
+ self.zeta = data['zeta'].reshape(s)
+ self.varzeta = data['sigma_zeta'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.ntri = data['ntri'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+ self.npatch3 = params.get('npatch3', 1)
+
+
+[docs]class KKKCrossCorrelation(BinnedCorr3):
+ r"""This class handles the calculation a 3-point kappa-kappa-kappa cross-correlation
+ function.
+
+ For 3-point cross correlations, it matters which of the two or three fields falls on
+ each corner of the triangle. E.g. is field 1 on the corner opposite d1 (the longest
+ size of the triangle) or is it field 2 (or 3) there? This is in contrast to the 2-point
+ correlation where the symmetry of the situation means that it doesn't matter which point
+ is identified with each field. This makes it significantly more complicated to keep track
+ of all the relevant information for a 3-point cross correlation function.
+
+ The `KKKCorrelation` class holds a single :math:`\zeta` functions describing all
+ possible triangles, parameterized according to their relative side lengths ordered as
+ d1 > d2 > d3.
+
+ For a cross-correlation of two fields: K1 - K1 - K2 (i.e. the K1 field is at two of the
+ corners and K2 is at one corner), then we need three these :math:`\zeta` functions
+ to capture all of the triangles, since the K2 points may be opposite d1 or d2 or d3.
+ For a cross-correlation of three fields: K1 - K2 - K3, we need six sets, to account for
+ all of the possible permutations relative to the triangle sides.
+
+ Therefore, this class holds 6 instances of `KKKCorrelation`, which in turn hold the
+ information about triangles in each of the relevant configurations. We name these:
+
+ Attributes:
+ k1k2k3: Triangles where K1 is opposite d1, K2 is opposite d2, K3 is opposite d3.
+ k1k3k2: Triangles where K1 is opposite d1, K3 is opposite d2, K2 is opposite d3.
+ k2k1k3: Triangles where K2 is opposite d1, K1 is opposite d2, K3 is opposite d3.
+ k2k3k1: Triangles where K2 is opposite d1, K3 is opposite d2, K1 is opposite d3.
+ k3k1k2: Triangles where K3 is opposite d1, K1 is opposite d2, K2 is opposite d3.
+ k3k2k1: Triangles where K3 is opposite d1, K2 is opposite d2, K1 is opposite d3.
+
+ If for instance K2 and K3 are the same field, then e.g. k1k2k3 and k1k3k2 will have
+ the same values.
+
+ Ojects of this class also hold the following attributes, which are identical in each of
+ the above KKKCorrelation instances.
+
+ Attributes:
+ nbins: The number of bins in logr where r = d2
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+ nubins: The number of bins in u where u = d3/d2
+ ubin_size: The size of the bins in u
+ min_u: The minimum u being considered
+ max_u: The maximum u being considered
+ nvbins: The number of bins in v where v = +-(d1-d2)/d3
+ vbin_size: The size of the bins in v
+ min_v: The minimum v being considered
+ max_v: The maximum v being considered
+ logr1d: The nominal centers of the nbins bins in log(r).
+ u1d: The nominal centers of the nubins bins in u.
+ v1d: The nominal centers of the nvbins bins in v.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_cross` directly,
+ then the units will not be applied to ``meanr`` or ``meanlogr`` until the `finalize`
+ function is called.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr3`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr3` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `KKKCrossCorrelation`. See class doc for details.
+ """
+ BinnedCorr3.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 2 # KData
+ self._ro._d2 = 2 # KData
+ self._ro._d3 = 2 # KData
+
+ self.k1k2k3 = KKKCorrelation(config, logger=logger, **kwargs)
+ self.k1k3k2 = KKKCorrelation(config, logger=logger, **kwargs)
+ self.k2k1k3 = KKKCorrelation(config, logger=logger, **kwargs)
+ self.k2k3k1 = KKKCorrelation(config, logger=logger, **kwargs)
+ self.k3k1k2 = KKKCorrelation(config, logger=logger, **kwargs)
+ self.k3k2k1 = KKKCorrelation(config, logger=logger, **kwargs)
+ self._all = [self.k1k2k3, self.k1k3k2, self.k2k1k3, self.k2k3k1, self.k3k1k2, self.k3k2k1]
+
+ self.logger.debug('Finished building KKKCrossCorr')
+
+[docs] def __eq__(self, other):
+ """Return whether two `KKKCrossCorrelation` instances are equal"""
+ return (isinstance(other, KKKCrossCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nubins == other.nubins and
+ self.ubin_size == other.ubin_size and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v and
+ self.nvbins == other.nvbins and
+ self.vbin_size == other.vbin_size and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ self.k1k2k3 == other.k1k2k3 and
+ self.k1k3k2 == other.k1k3k2 and
+ self.k2k1k3 == other.k2k1k3 and
+ self.k2k3k1 == other.k2k3k1 and
+ self.k3k1k2 == other.k3k1k2 and
+ self.k3k2k1 == other.k3k2k1)
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = KKKCrossCorrelation.__new__(KKKCrossCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, KKKCorrelation):
+ ret.__dict__[key] = item.copy()
+ else:
+ ret.__dict__[key] = item
+ # This needs to be the new list:
+ ret._all = [ret.k1k2k3, ret.k1k3k2, ret.k2k1k3, ret.k2k3k1, ret.k3k1k2, ret.k3k2k1]
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_cross12(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process two catalogs, accumulating the 3pt cross-correlation, where one of the
+ points in each triangle come from the first catalog, and two come from the second.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ .. note::
+
+ This only adds to the attributes k1k2k3, k2k1k3, k2k3k1, not the ones where
+ 3 comes before 2. When running this via the regular `process` method, it will
+ combine them at the end to make sure k1k2k3 == k1k3k2, etc. for a complete
+ calculation of the 1-2 cross-correlation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process. (1 point in each triangle will come
+ from this catalog.)
+ cat2 (Catalog): The second catalog to process. (2 points in each triangle will come
+ from this catalog.)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process KKK (1-2) cross-correlations')
+ else:
+ self.logger.info('Starting process KKK (1-2) cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ for kkk in self._all:
+ kkk._set_metric(self.metric, self.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 3 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross12(self.k1k2k3.corr, self.k2k1k3.corr, self.k2k3k1.corr,
+ f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords,
+ self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, cat3, *, metric=None, num_threads=None):
+ """Process a set of three catalogs, accumulating the 3pt cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ cat3 (Catalog): The third catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '' and cat3.name == '':
+ self.logger.info('Starting process KKK cross-correlations')
+ else:
+ self.logger.info('Starting process KKK cross-correlations for cats %s, %s, %s.',
+ cat1.name, cat2.name, cat3.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords, cat3.coords)
+ for kkk in self._all:
+ kkk._set_metric(self.metric, self.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f3 = cat3.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 3,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross3(self.k1k2k3.corr, self.k1k3k2.corr,
+ self.k2k1k3.corr, self.k2k3k1.corr,
+ self.k3k1k2.corr, self.k3k2k1.corr,
+ f1.data, f2.data, f3.data, self.output_dots,
+ f1._d, f2._d, f3._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ for kkk in self._all:
+ kkk._finalize()
+
+[docs] def finalize(self, vark1, vark2, vark3):
+ """Finalize the calculation of the correlation function.
+
+ The `process_cross` command accumulate values in each bin, so they can be called
+ multiple times if appropriate. Afterwards, this command finishes the calculation
+ by dividing by the total weight.
+
+ Parameters:
+ vark1 (float): The kappa variance for the first field that was correlated.
+ vark2 (float): The kappa variance for the second field that was correlated.
+ vark3 (float): The kappa variance for the third field that was correlated.
+ """
+ self.k1k2k3.finalize(vark1,vark2,vark3)
+ self.k1k3k2.finalize(vark1,vark3,vark2)
+ self.k2k1k3.finalize(vark2,vark1,vark3)
+ self.k2k3k1.finalize(vark2,vark3,vark1)
+ self.k3k1k2.finalize(vark3,vark1,vark2)
+ self.k3k2k1.finalize(vark3,vark2,vark1)
+
+ @property
+ def nonzero(self):
+ """Return if there are any values accumulated yet. (i.e. ntri > 0)
+ """
+ return any([kkk.nonzero for kkk in self._all])
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ for kkk in self._all:
+ kkk._clear()
+
+[docs] def __iadd__(self, other):
+ """Add a second `KKKCrossCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `KKKCrossCorrelation` objects should not have had
+ `finalize` called yet. Then, after adding them together, you should call `finalize`
+ on the sum.
+ """
+ if not isinstance(other, KKKCrossCorrelation):
+ raise TypeError("Can only add another KKKCrossCorrelation object")
+ self.k1k2k3 += other.k1k2k3
+ self.k1k3k2 += other.k1k3k2
+ self.k2k1k3 += other.k2k1k3
+ self.k2k3k1 += other.k2k3k1
+ self.k3k1k2 += other.k3k1k2
+ self.k3k2k1 += other.k3k2k1
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ for i, kkk in enumerate(self._all):
+ kkk._sum([c._all[i] for c in others])
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2, cat3=None, *, metric=None, num_threads=None,
+ comm=None, low_mem=False, initialize=True, finalize=True):
+ """Accumulate the cross-correlation of the points in the given Catalogs: cat1, cat2, cat3.
+
+ - If 2 arguments are given, then compute a cross-correlation function with the
+ first catalog taking one corner of the triangles, and the second taking two corners.
+ - If 3 arguments are given, then compute a three-way cross-correlation function.
+
+ All arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first K field.
+ cat2 (Catalog): A catalog or list of catalogs for the second K field.
+ cat3 (Catalog): A catalog or list of catalogs for the third K field.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr3.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+ self._process12 = False
+
+ if not isinstance(cat1,list): cat1 = cat1.get_patches()
+ if not isinstance(cat2,list): cat2 = cat2.get_patches()
+ if cat3 is not None and not isinstance(cat3,list): cat3 = cat3.get_patches()
+
+ if cat3 is None:
+ self._process12 = True
+ self._process_all_cross12(cat1, cat2, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, cat3, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if self._process12:
+ # Then some of the processing involved a cross12 calculation.
+ # This means that spots 2 and 3 should not be distinguished.
+ # Combine the relevant arrays.
+ self.k1k2k3 += self.k1k3k2
+ self.k2k1k3 += self.k3k1k2
+ self.k2k3k1 += self.k3k2k1
+ # Copy back by doing clear and +=.
+ # This makes sure the coords and metric are set properly.
+ self.k1k3k2.clear()
+ self.k3k1k2.clear()
+ self.k3k2k1.clear()
+ self.k1k3k2 += self.k1k2k3
+ self.k3k1k2 += self.k2k1k3
+ self.k3k2k1 += self.k2k3k1
+
+ vark1 = calculateVarK(cat1, low_mem=low_mem)
+ vark2 = calculateVarK(cat2, low_mem=low_mem)
+ self.logger.info("vark1 = %f: sig_k = %f",vark1,math.sqrt(vark1))
+ self.logger.info("vark2 = %f: sig_k = %f",vark2,math.sqrt(vark2))
+ if cat3 is None:
+ vark3 = vark2
+ else:
+ vark3 = calculateVarK(cat3, low_mem=low_mem)
+ self.logger.info("vark3 = %f: sig_k = %f",vark3,math.sqrt(vark3))
+ self.finalize(vark1,vark2,vark3)
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ In this case, the concatenation of zeta.ravel() for each combination in the following
+ order: k1k2k3, k1k3k2, k2k1k3, k2k3k1, k3k1k2, k3k2k1.
+ """
+ return np.concatenate([kkk.zeta.ravel() for kkk in self._all])
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ In this case, the concatenation of getWeight() for each combination in the following
+ order: k1k2k3, k1k3k2, k2k1k3, k2k3k1, k3k1k2, k3k2k1.
+ """
+ return np.concatenate([kkk.getWeight() for kkk in self._all])
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the cross-correlation functions to the file, file_name.
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing KKK cross-correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ writer = make_writer(file_name, precision, file_type, self.logger)
+ with writer:
+ names = [ 'k1k2k3', 'k1k3k2', 'k2k1k3', 'k2k3k1', 'k3k1k2', 'k3k2k1' ]
+ for name, corr in zip(names, self._all):
+ corr._write(writer, name, write_patch_results)
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `KKKCrossCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading KKK cross-correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ names = [ 'k1k2k3', 'k1k3k2', 'k2k1k3', 'k2k3k1', 'k3k1k2', 'k3k2k1' ]
+ for name, corr in zip(names, self._all):
+ corr._read(reader, name)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: ngcorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarG
+from .binnedcorr2 import BinnedCorr2
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class NGCorrelation(BinnedCorr2):
+ r"""This class handles the calculation and storage of a 2-point count-shear correlation
+ function. This is the tangential shear profile around lenses, commonly referred to as
+ galaxy-galaxy lensing.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+
+ In addition, the following attributes are numpy arrays of length (nbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ meanr: The (weighted) mean value of r for the pairs in each bin.
+ If there are no pairs in a bin, then exp(logr) will be used instead.
+ meanlogr: The (weighted) mean value of log(r) for the pairs in each bin.
+ If there are no pairs in a bin, then logr will be used instead.
+ xi: The correlation function, :math:`\xi(r) = \langle \gamma_T\rangle`.
+ xi_im: The imaginary part of :math:`\xi(r)`.
+ varxi: An estimate of the variance of :math:`\xi`
+ weight: The total weight in each bin.
+ npairs: The number of pairs going into each bin (including pairs where one or
+ both objects have w=0).
+ cov: An estimate of the full covariance matrix.
+ raw_xi: The raw value of xi, uncorrected by an RG calculation. cf. `calculateXi`
+ raw_xi_im: The raw value of xi_im, uncorrected by an RG calculation. cf. `calculateXi`
+ raw_varxi: The raw value of varxi, uncorrected by an RG calculation. cf. `calculateXi`
+
+ .. note::
+
+ The default method for estimating the variance and covariance attributes (``varxi``,
+ and ``cov``) is 'shot', which only includes the shape noise propagated into
+ the final correlation. This does not include sample variance, so it is always an
+ underestimate of the actual variance. To get better estimates, you need to set
+ ``var_method`` to something else and use patches in the input catalog(s).
+ cf. `Covariance Estimates`.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_cross`,
+ then the units will not be applied to ``meanr`` or ``meanlogr`` until the `finalize`
+ function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> ng = treecorr.NGCorrelation(config)
+ >>> ng.process(cat1,cat2) # Compute the cross-correlation.
+ >>> ng.write(file_name) # Write out to a file.
+ >>> xi = gg.xi # Or access the correlation function directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr2`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr2` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `NGCorrelation`. See class doc for details.
+ """
+ BinnedCorr2.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 1 # NData
+ self._ro._d2 = 3 # GData
+ self.xi = np.zeros_like(self.rnom, dtype=float)
+ self.xi_im = np.zeros_like(self.rnom, dtype=float)
+ self.varxi = np.zeros_like(self.rnom, dtype=float)
+ self.meanr = np.zeros_like(self.rnom, dtype=float)
+ self.meanlogr = np.zeros_like(self.rnom, dtype=float)
+ self.weight = np.zeros_like(self.rnom, dtype=float)
+ self.npairs = np.zeros_like(self.rnom, dtype=float)
+ self.raw_xi = self.xi
+ self.raw_xi_im = self.xi_im
+ self.raw_varxi = self.varxi
+ self._rg = None
+ self.logger.debug('Finished building NGCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr2(
+ self._d1, self._d2, self._bintype,
+ self._min_sep,self._max_sep,self._nbins,self._bin_size,self.b,
+ self.min_rpar, self.max_rpar, self.xperiod, self.yperiod, self.zperiod,
+ dp(self.raw_xi),dp(self.raw_xi_im), dp(None), dp(None),
+ dp(self.meanr),dp(self.meanlogr),dp(self.weight),dp(self.npairs))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr2(self.corr, self._d1, self._d2, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `NGCorrelation` instances are equal"""
+ return (isinstance(other, NGCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.min_rpar == other.min_rpar and
+ self.max_rpar == other.max_rpar and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meanr, other.meanr) and
+ np.array_equal(self.meanlogr, other.meanlogr) and
+ np.array_equal(self.xi, other.xi) and
+ np.array_equal(self.xi_im, other.xi_im) and
+ np.array_equal(self.varxi, other.varxi) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.npairs, other.npairs))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = NGCorrelation.__new__(NGCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ if self.xi is self.raw_xi:
+ ret.raw_xi = ret.xi
+ ret.raw_xi_im = ret.xi_im
+ ret.raw_varxi = ret.varxi
+ else:
+ ret.raw_xi = self.raw_xi.copy()
+ ret.raw_xi_im = self.raw_xi_im.copy()
+ ret.raw_varxi = self.raw_varxi.copy()
+ if self._rg is not None:
+ ret._rg = self._rg.copy()
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NG cross-correlations')
+ else:
+ self.logger.info('Starting process NG cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getGField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross2(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_pairwise(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation, only using
+ the corresponding pairs of objects in each catalog.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ import warnings
+ warnings.warn("The process_pairwise function is slated to be removed in a future version. "+
+ "If you are actually using this function usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NG pairwise-correlations')
+ else:
+ self.logger.info('Starting process NG pairwise-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+
+ f1 = cat1.getNSimpleField()
+ f2 = cat2.getGSimpleField()
+
+ _lib.ProcessPair(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.raw_xi[mask1] /= self.weight[mask1]
+ self.raw_xi_im[mask1] /= self.weight[mask1]
+ self.meanr[mask1] /= self.weight[mask1]
+ self.meanlogr[mask1] /= self.weight[mask1]
+
+ # Update the units of meanr, meanlogr
+ self._apply_units(mask1)
+
+ # Use meanr, meanlogr when available, but set to nominal when no pairs in bin.
+ self.meanr[mask2] = self.rnom[mask2]
+ self.meanlogr[mask2] = self.logr[mask2]
+
+[docs] def finalize(self, varg):
+ """Finalize the calculation of the correlation function.
+
+ The `process_cross` command accumulates values in each bin, so it can be called
+ multiple times if appropriate. Afterwards, this command finishes the calculation
+ by dividing each column by the total weight.
+
+ Parameters:
+ varg (float): The shear variance per component for the second field.
+ """
+ self._finalize()
+ self._var_num = varg
+ self.cov = self.estimate_cov(self.var_method)
+ self.raw_varxi.ravel()[:] = self.cov.diagonal()
+
+ self.xi = self.raw_xi
+ self.xi_im = self.raw_xi_im
+ self.varxi = self.raw_varxi
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.raw_xi.ravel()[:] = 0
+ self.raw_xi_im.ravel()[:] = 0
+ self.raw_varxi.ravel()[:] = 0
+ self.meanr.ravel()[:] = 0
+ self.meanlogr.ravel()[:] = 0
+ self.weight.ravel()[:] = 0
+ self.npairs.ravel()[:] = 0
+ self.xi = self.raw_xi
+ self.xi_im = self.raw_xi_im
+ self.varxi = self.raw_varxi
+
+[docs] def __iadd__(self, other):
+ """Add a second `NGCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `NGCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, NGCorrelation):
+ raise TypeError("Can only add another NGCorrelation object")
+ if not (self._nbins == other._nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep):
+ raise ValueError("NGCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords)
+ self.raw_xi.ravel()[:] += other.raw_xi.ravel()[:]
+ self.raw_xi_im.ravel()[:] += other.raw_xi_im.ravel()[:]
+ self.meanr.ravel()[:] += other.meanr.ravel()[:]
+ self.meanlogr.ravel()[:] += other.meanlogr.ravel()[:]
+ self.weight.ravel()[:] += other.weight.ravel()[:]
+ self.npairs.ravel()[:] += other.npairs.ravel()[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.raw_xi for c in others], axis=0, out=self.raw_xi)
+ np.sum([c.raw_xi_im for c in others], axis=0, out=self.raw_xi_im)
+ np.sum([c.meanr for c in others], axis=0, out=self.meanr)
+ np.sum([c.meanlogr for c in others], axis=0, out=self.meanlogr)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.npairs for c in others], axis=0, out=self.npairs)
+ self.xi = self.raw_xi
+ self.xi_im = self.raw_xi_im
+ self.varxi = self.raw_varxi
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2, *, metric=None, num_threads=None, comm=None, low_mem=False,
+ initialize=True, finalize=True):
+ """Compute the correlation function.
+
+ Both arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the N field.
+ cat2 (Catalog): A catalog or list of catalogs for the G field.
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr2.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+ self._rg = None
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+
+ self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ varg = calculateVarG(cat2, low_mem=low_mem)
+ self.logger.info("varg = %f: sig_sn (per component) = %f",varg,math.sqrt(varg))
+ self.finalize(varg)
+
+[docs] @depr_pos_kwargs
+ def calculateXi(self, *, rg=None):
+ r"""Calculate the correlation function possibly given another correlation function
+ that uses random points for the foreground objects.
+
+ - If rg is None, the simple correlation function :math:`\langle \gamma_T\rangle` is
+ returned.
+ - If rg is not None, then a compensated calculation is done:
+ :math:`\langle \gamma_T\rangle = (DG - RG)`, where DG represents the mean shear
+ around the lenses and RG represents the mean shear around random points.
+
+ After calling this function, the attributes ``xi``, ``xi_im``, ``varxi``, and ``cov`` will
+ correspond to the compensated values (if rg is provided). The raw, uncompensated values
+ are available as ``rawxi``, ``raw_xi_im``, and ``raw_varxi``.
+
+ Parameters:
+ rg (NGCorrelation): The cross-correlation using random locations as the lenses
+ (RG), if desired. (default: None)
+
+ Returns:
+ Tuple containing
+
+ - xi = array of the real part of :math:`\xi(R)`
+ - xi_im = array of the imaginary part of :math:`\xi(R)`
+ - varxi = array of the variance estimates of the above values
+ """
+ if rg is not None:
+ self.xi = self.raw_xi - rg.xi
+ self.xi_im = self.raw_xi_im - rg.xi_im
+ self._rg = rg
+
+ if rg.npatch1 not in (1,self.npatch1) or rg.npatch2 != self.npatch2:
+ raise RuntimeError("RG must be run with the same patches as DG")
+
+ if len(self.results) > 0:
+ # If there are any rg patch pairs that aren't in results (e.g. due to different
+ # edge effects among the various pairs in consideration), then we need to add
+ # some dummy results to make sure all the right pairs are computed when we make
+ # the vectors for the covariance matrix.
+ template = next(iter(self.results.values())) # Just need something to copy.
+ for ij in rg.results:
+ if ij in self.results: continue
+ new_cij = template.copy()
+ new_cij.xi.ravel()[:] = 0
+ new_cij.weight.ravel()[:] = 0
+ self.results[ij] = new_cij
+
+ self.cov = self.estimate_cov(self.var_method)
+ self.varxi.ravel()[:] = self.cov.diagonal()
+ else:
+ self.varxi = self.raw_varxi + rg.varxi
+ else:
+ self.xi = self.raw_xi
+ self.xi_im = self.raw_xi_im
+ self.varxi = self.raw_varxi
+
+ return self.xi, self.xi_im, self.varxi
+
+ def _calculate_xi_from_pairs(self, pairs):
+ self._sum([self.results[ij] for ij in pairs])
+ self._finalize()
+ if self._rg is not None:
+ # If rg has npatch1 = 1, adjust pairs appropriately
+ if self._rg.npatch1 == 1:
+ pairs = [(0,ij[1]) for ij in pairs if ij[0] == ij[1]]
+ # Make sure all ij are in the rg results (some might be missing, which is ok)
+ pairs = [ij for ij in pairs if self._rg._ok[ij[0],ij[1]]]
+ self._rg._calculate_xi_from_pairs(pairs)
+ self.xi -= self._rg.xi
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, rg=None, file_type=None, precision=None,
+ write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ - If rg is None, the simple correlation function :math:`\langle \gamma_T\rangle` is used.
+ - If rg is not None, then a compensated calculation is done:
+ :math:`\langle \gamma_T\rangle = (DG - RG)`, where DG represents the mean shear
+ around the lenses and RG represents the mean shear around random points.
+
+ The output file will include the following columns:
+
+ ========== =============================================================
+ Column Description
+ ========== =============================================================
+ r_nom The nominal center of the bin in r
+ meanr The mean value :math:`\langle r \rangle` of pairs that fell
+ into each bin
+ meanlogr The mean value :math:`\langle \log(r) \rangle` of pairs that
+ fell into each bin
+ gamT The real part of the mean tangential shear,
+ :math:`\langle \gamma_T \rangle(r)`
+ gamX The imag part of the mean tangential shear,
+ :math:`\langle \gamma_\times \rangle(r)`
+ sigma The sqrt of the variance estimate of either of these
+ weight The total weight contributing to each bin
+ npairs The total number of pairs in each bin
+ ========== =============================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ rg (NGCorrelation): The cross-correlation using random locations as the lenses
+ (RG), if desired. (default: None)
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing NG correlations to %s',file_name)
+ self.calculateXi(rg=rg)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return ['r_nom','meanr','meanlogr','gamT','gamX','sigma','weight','npairs']
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.meanr, self.meanlogr,
+ self.xi, self.xi_im, np.sqrt(self.varxi), self.weight, self.npairs ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `NGCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading NG correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ self.meanr = data['meanR'].reshape(s)
+ self.meanlogr = data['meanlogR'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meanr = data['meanr'].reshape(s)
+ self.meanlogr = data['meanlogr'].reshape(s)
+ self.xi = data['gamT'].reshape(s)
+ self.xi_im = data['gamX'].reshape(s)
+ self.varxi = data['sigma'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.npairs = data['npairs'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.raw_xi = self.xi
+ self.raw_xi_im = self.xi_im
+ self.raw_varxi = self.varxi
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+
+[docs] @depr_pos_kwargs
+ def calculateNMap(self, *, R=None, rg=None, m2_uform=None):
+ r"""Calculate the aperture mass statistics from the correlation function.
+
+ .. math::
+
+ \langle N M_{ap} \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{R^2}
+ T_\times\left(\frac{r}{R}\right) \Re\xi(r) \\
+ \langle N M_{\times} \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{R^2}
+ T_\times\left(\frac{r}{R}\right) \Im\xi(r)
+
+ The ``m2_uform`` parameter sets which definition of the aperture mass to use.
+ The default is to use 'Crittenden'.
+
+ If ``m2_uform`` is 'Crittenden':
+
+ .. math::
+
+ U(r) &= \frac{1}{2\pi} (1-r^2) \exp(-r^2/2) \\
+ T_\times(s) &= \frac{s^2}{128} (12-s^2) \exp(-s^2/4)
+
+ cf. Crittenden, et al (2002): ApJ, 568, 20
+
+ If ``m2_uform`` is 'Schneider':
+
+ .. math::
+
+ U(r) &= \frac{9}{\pi} (1-r^2) (1/3-r^2) \\
+ T_\times(s) &= \frac{18}{\pi} s^2 \arccos(s/2) \\
+ &\qquad - \frac{3}{40\pi} s^3 \sqrt{4-s^2} (196 - 74s^2 + 14s^4 - s^6)
+
+ cf. Schneider, et al (2002): A&A, 389, 729
+
+ In neither case is this formula in the above papers, but the derivation is similar
+ to the derivations of :math:`T_+` and :math:`T_-` in Schneider et al. (2002).
+
+ Parameters:
+ R (array): The R values at which to calculate the aperture mass statistics.
+ (default: None, which means use self.rnom)
+ rg (NGCorrelation): The cross-correlation using random locations as the lenses
+ (RG), if desired. (default: None)
+ m2_uform (str): Which form to use for the aperture mass, as described above.
+ (default: 'Crittenden'; this value can also be given in the
+ constructor in the config dict.)
+
+ Returns:
+ Tuple containing
+
+ - nmap = array of :math:`\langle N M_{ap} \rangle(R)`
+ - nmx = array of :math:`\langle N M_{\times} \rangle(R)`
+ - varnmap = array of variance estimates of the above values
+ """
+ if m2_uform is None:
+ m2_uform = self.config.get('m2_uform','Crittenden')
+ if m2_uform not in ['Crittenden', 'Schneider']:
+ raise ValueError("Invalid m2_uform")
+ if R is None:
+ R = self.rnom
+
+ # Make s a matrix, so we can eventually do the integral by doing a matrix product.
+ s = np.outer(1./R, self.meanr)
+ ssq = s*s
+ if m2_uform == 'Crittenden':
+ exp_factor = np.exp(-ssq/4.)
+ Tx = ssq * (12. - ssq) / 128. * exp_factor
+ else:
+ Tx = np.zeros_like(s)
+ sa = s[s<2.]
+ ssqa = ssq[s<2.]
+ Tx[s<2.] = 196. + ssqa*(-74. + ssqa*(14. - ssqa))
+ Tx[s<2.] *= -3./(40.*np.pi) * sa * ssqa * np.sqrt(4.-sa**2)
+ Tx[s<2.] += 18./np.pi * ssqa * np.arccos(sa/2.)
+ Tx *= ssq
+
+ xi, xi_im, varxi = self.calculateXi(rg=rg)
+
+ # Now do the integral by taking the matrix products.
+ # Note that dlogr = bin_size
+ Txxi = Tx.dot(xi)
+ Txxi_im = Tx.dot(xi_im)
+ nmap = Txxi * self.bin_size
+ nmx = Txxi_im * self.bin_size
+
+ # The variance of each of these is
+ # Var(<NMap>(R)) = int_r=0..2R [s^4 dlogr^2 Tx(s)^2 Var(xi)]
+ varnmap = (Tx**2).dot(varxi) * self.bin_size**2
+
+ return nmap, nmx, varnmap
+
+[docs] @depr_pos_kwargs
+ def writeNMap(self, file_name, *, R=None, rg=None, m2_uform=None, file_type=None,
+ precision=None):
+ r"""Write the cross correlation of the foreground galaxy counts with the aperture mass
+ based on the correlation function to the file, file_name.
+
+ If rg is provided, the compensated calculation will be used for :math:`\xi`.
+
+ See `calculateNMap` for an explanation of the ``m2_uform`` parameter.
+
+ The output file will include the following columns:
+
+ ========== =========================================================
+ Column Description
+ ========== =========================================================
+ R The radius of the aperture.
+ NMap An estimate of :math:`\langle N_{ap} M_{ap} \rangle(R)`
+ NMx An estimate of :math:`\langle N_{ap} M_\times \rangle(R)`
+ sig_nmap The sqrt of the variance estimate of either of these
+ ========== =========================================================
+
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ R (array): The R values at which to calculate the aperture mass statistics.
+ (default: None, which means use self.rnom)
+ rg (NGCorrelation): The cross-correlation using random locations as the lenses
+ (RG), if desired. (default: None)
+ m2_uform (str): Which form to use for the aperture mass. (default: 'Crittenden';
+ this value can also be given in the constructor in the config dict.)
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ """
+ self.logger.info('Writing NMap from NG correlations to %s',file_name)
+ if R is None:
+ R = self.rnom
+
+ nmap, nmx, varnmap = self.calculateNMap(R=R, rg=rg, m2_uform=m2_uform)
+ if precision is None:
+ precision = self.config.get('precision', 4)
+
+ col_names = ['R','NMap','NMx','sig_nmap']
+ columns = [ R, nmap, nmx, np.sqrt(varnmap) ]
+ writer = make_writer(file_name, precision, file_type, logger=self.logger)
+ with writer:
+ writer.write(col_names, columns)
+
+[docs] @depr_pos_kwargs
+ def writeNorm(self, file_name, *, gg, dd, rr, R=None, dr=None, rg=None,
+ m2_uform=None, file_type=None, precision=None):
+ r"""Write the normalized aperture mass cross-correlation to the file, file_name.
+
+ The combination :math:`\langle N M_{ap}\rangle^2 / \langle M_{ap}^2\rangle
+ \langle N_{ap}^2\rangle` is related to :math:`r`, the galaxy-mass correlation
+ coefficient. Similarly, :math:`\langle N_{ap}^2\rangle / \langle M_{ap}^2\rangle`
+ is related to :math:`b`, the galaxy bias parameter. cf. Hoekstra et al, 2002:
+ http://adsabs.harvard.edu/abs/2002ApJ...577..604H
+
+ This function computes these combinations and outputs them to a file.
+
+ - if rg is provided, the compensated calculation will be used for
+ :math:`\langle N_{ap} M_{ap} \rangle`.
+ - if dr is provided, the compensated calculation will be used for
+ :math:`\langle N_{ap}^2 \rangle`.
+
+ See `calculateNMap` for an explanation of the ``m2_uform`` parameter.
+
+ The output file will include the following columns:
+
+ ========== =====================================================================
+ Column Description
+ ========== =====================================================================
+ R The radius of the aperture
+ NMap An estimate of :math:`\langle N_{ap} M_{ap} \rangle(R)`
+ NMx An estimate of :math:`\langle N_{ap} M_\times \rangle(R)`
+ sig_nmap The sqrt of the variance estimate of either of these
+ Napsq An estimate of :math:`\langle N_{ap}^2 \rangle(R)`
+ sig_napsq The sqrt of the variance estimate of :math:`\langle N_{ap}^2 \rangle`
+ Mapsq An estimate of :math:`\langle M_{ap}^2 \rangle(R)`
+ sig_mapsq The sqrt of the variance estimate of :math:`\langle M_{ap}^2 \rangle`
+ NMap_norm The ratio :math:`\langle N_{ap} M_{ap} \rangle^2 /`
+ :math:`\langle N_{ap}^2 \rangle \langle M_{ap}^2 \rangle`
+ sig_norm The sqrt of the variance estimate of this ratio
+ Nsq_Mapsq The ratio :math:`\langle N_{ap}^2 \rangle / \langle M_{ap}^2 \rangle`
+ sig_nn_mm The sqrt of the variance estimate of this ratio
+ ========== =====================================================================
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ gg (GGCorrelation): The auto-correlation of the shear field
+ dd (NNCorrelation): The auto-correlation of the lens counts (DD)
+ rr (NNCorrelation): The auto-correlation of the random field (RR)
+ R (array): The R values at which to calculate the aperture mass statistics.
+ (default: None, which means use self.rnom)
+ dr (NNCorrelation): The cross-correlation of the data with randoms (DR), if
+ desired, in which case the Landy-Szalay estimator will be
+ calculated. (default: None)
+ rd (NNCorrelation): The cross-correlation of the randoms with data (RD), if
+ desired. (default: None, which means use rd=dr)
+ rg (NGCorrelation): The cross-correlation using random locations as the lenses
+ (RG), if desired. (default: None)
+ m2_uform (str): Which form to use for the aperture mass. (default: 'Crittenden';
+ this value can also be given in the constructor in the config dict.)
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ """
+ self.logger.info('Writing Norm from NG correlations to %s',file_name)
+ if R is None:
+ R = self.rnom
+
+ nmap, nmx, varnmap = self.calculateNMap(R=R, rg=rg, m2_uform=m2_uform)
+ mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = gg.calculateMapSq(R=R, m2_uform=m2_uform)
+ nsq, varnsq = dd.calculateNapSq(R=R, rr=rr, dr=dr, m2_uform=m2_uform)
+
+ nmnorm = nmap**2 / (nsq * mapsq)
+ varnmnorm = nmnorm**2 * (4. * varnmap / nmap**2 + varnsq / nsq**2 + varmapsq / mapsq**2)
+ nnnorm = nsq / mapsq
+ varnnnorm = nnnorm**2 * (varnsq / nsq**2 + varmapsq / mapsq**2)
+ if precision is None:
+ precision = self.config.get('precision', 4)
+
+ col_names = [ 'R',
+ 'NMap','NMx','sig_nmap',
+ 'Napsq','sig_napsq','Mapsq','sig_mapsq',
+ 'NMap_norm','sig_norm','Nsq_Mapsq','sig_nn_mm' ]
+ columns = [ R,
+ nmap, nmx, np.sqrt(varnmap),
+ nsq, np.sqrt(varnsq), mapsq, np.sqrt(varmapsq),
+ nmnorm, np.sqrt(varnmnorm), nnnorm, np.sqrt(varnnnorm) ]
+ writer = make_writer(file_name, precision, file_type, logger=self.logger)
+ with writer:
+ writer.write(col_names, columns)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: nkcorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .catalog import calculateVarK
+from .binnedcorr2 import BinnedCorr2
+from .util import double_ptr as dp
+from .util import make_writer, make_reader
+from .util import depr_pos_kwargs
+
+
+[docs]class NKCorrelation(BinnedCorr2):
+ r"""This class handles the calculation and storage of a 2-point count-kappa correlation
+ function.
+
+ .. note::
+
+ While we use the term kappa (:math:`\kappa`) here and the letter K in various places,
+ in fact any scalar field will work here. For example, you can use this to compute
+ correlations of non-shear quantities, e.g. the sizes or concentrations of galaxies, around
+ a set of lenses, where "kappa" would be the measurements of these quantities.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+
+ In addition, the following attributes are numpy arrays of length (nbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ meanr: The (weighted) mean value of r for the pairs in each bin.
+ If there are no pairs in a bin, then exp(logr) will be used instead.
+ meanlogr: The (weighted) mean value of log(r) for the pairs in each bin.
+ If there are no pairs in a bin, then logr will be used instead.
+ xi: The correlation function, :math:`\xi(r) = \langle \kappa\rangle`.
+ varxi: An estimate of the variance of :math:`\xi`
+ weight: The total weight in each bin.
+ npairs: The number of pairs going into each bin (including pairs where one or
+ both objects have w=0).
+ cov: An estimate of the full covariance matrix.
+ raw_xi: The raw value of xi, uncorrected by an RK calculation. cf. `calculateXi`
+ raw_varxi: The raw value of varxi, uncorrected by an RK calculation. cf. `calculateXi`
+
+ .. note::
+
+ The default method for estimating the variance and covariance attributes (``varxi``,
+ and ``cov``) is 'shot', which only includes the shape noise propagated into
+ the final correlation. This does not include sample variance, so it is always an
+ underestimate of the actual variance. To get better estimates, you need to set
+ ``var_method`` to something else and use patches in the input catalog(s).
+ cf. `Covariance Estimates`.
+
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_cross`,
+ then the units will not be applied to ``meanr`` or ``meanlogr`` until the `finalize`
+ function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> nk = treecorr.NKCorrelation(config)
+ >>> nk.process(cat1,cat2) # Compute the cross-correlation function.
+ >>> nk.write(file_name) # Write out to a file.
+ >>> xi = nk.xi # Or access the correlation function directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr2`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr2` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `NKCorrelation`. See class doc for details.
+ """
+ BinnedCorr2.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 1 # NData
+ self._ro._d2 = 2 # KData
+ self.xi = np.zeros_like(self.rnom, dtype=float)
+ self.varxi = np.zeros_like(self.rnom, dtype=float)
+ self.meanr = np.zeros_like(self.rnom, dtype=float)
+ self.meanlogr = np.zeros_like(self.rnom, dtype=float)
+ self.weight = np.zeros_like(self.rnom, dtype=float)
+ self.npairs = np.zeros_like(self.rnom, dtype=float)
+ self.raw_xi = self.xi
+ self.raw_varxi = self.varxi
+ self._rk = None
+ self.logger.debug('Finished building NKCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr2(
+ self._d1, self._d2, self._bintype,
+ self._min_sep,self._max_sep,self._nbins,self._bin_size,self.b,
+ self.min_rpar, self.max_rpar, self.xperiod, self.yperiod, self.zperiod,
+ dp(self.raw_xi), dp(None), dp(None), dp(None),
+ dp(self.meanr),dp(self.meanlogr),dp(self.weight),dp(self.npairs))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr2(self.corr, self._d1, self._d2, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `NKCorrelation` instances are equal"""
+ return (isinstance(other, NKCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.min_rpar == other.min_rpar and
+ self.max_rpar == other.max_rpar and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ np.array_equal(self.meanr, other.meanr) and
+ np.array_equal(self.meanlogr, other.meanlogr) and
+ np.array_equal(self.xi, other.xi) and
+ np.array_equal(self.varxi, other.varxi) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.npairs, other.npairs))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = NKCorrelation.__new__(NKCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ if self.xi is self.raw_xi:
+ ret.raw_xi = ret.xi
+ ret.raw_varxi = ret.varxi
+ else:
+ ret.raw_xi = self.raw_xi.copy()
+ ret.raw_varxi = self.raw_varxi.copy()
+ if self._rk is not None:
+ ret._rk = self._rk.copy()
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NK cross-correlations')
+ else:
+ self.logger.info('Starting process NK cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getKField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross2(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+[docs] @depr_pos_kwargs
+ def process_pairwise(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation, only using
+ the corresponding pairs of objects in each catalog.
+
+ This accumulates the weighted sums into the bins, but does not finalize
+ the calculation by dividing by the total weight at the end. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ import warnings
+ warnings.warn("The process_pairwise function is slated to be removed in a future version. "+
+ "If you are actually using this function usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NK pairwise-correlations')
+ else:
+ self.logger.info('Starting process NK pairwise-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+
+ f1 = cat1.getNSimpleField()
+ f2 = cat2.getKSimpleField()
+
+ _lib.ProcessPair(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.raw_xi[mask1] /= self.weight[mask1]
+ self.meanr[mask1] /= self.weight[mask1]
+ self.meanlogr[mask1] /= self.weight[mask1]
+
+ # Update the units of meanr, meanlogr
+ self._apply_units(mask1)
+
+ # Use meanr, meanlogr when available, but set to nominal when no pairs in bin.
+ self.meanr[mask2] = self.rnom[mask2]
+ self.meanlogr[mask2] = self.logr[mask2]
+
+[docs] def finalize(self, vark):
+ """Finalize the calculation of the correlation function.
+
+ The `process_cross` command accumulates values in each bin, so it can be called
+ multiple times if appropriate. Afterwards, this command finishes the calculation
+ by dividing each column by the total weight.
+
+ Parameters:
+ vark: The kappa variance for the second field.
+ """
+ self._finalize()
+ self._var_num = vark
+ self.cov = self.estimate_cov(self.var_method)
+ self.raw_varxi.ravel()[:] = self.cov.diagonal()
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.raw_xi.ravel()[:] = 0
+ self.raw_varxi.ravel()[:] = 0
+ self.meanr.ravel()[:] = 0
+ self.meanlogr.ravel()[:] = 0
+ self.weight.ravel()[:] = 0
+ self.npairs.ravel()[:] = 0
+ self.xi = self.raw_xi
+ self.varxi = self.raw_varxi
+
+[docs] def __iadd__(self, other):
+ """Add a second `NKCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `NKCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, NKCorrelation):
+ raise TypeError("Can only add another NKCorrelation object")
+ if not (self._nbins == other._nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep):
+ raise ValueError("NKCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords)
+ self.raw_xi.ravel()[:] += other.raw_xi.ravel()[:]
+ self.meanr.ravel()[:] += other.meanr.ravel()[:]
+ self.meanlogr.ravel()[:] += other.meanlogr.ravel()[:]
+ self.weight.ravel()[:] += other.weight.ravel()[:]
+ self.npairs.ravel()[:] += other.npairs.ravel()[:]
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ np.sum([c.raw_xi for c in others], axis=0, out=self.raw_xi)
+ np.sum([c.meanr for c in others], axis=0, out=self.meanr)
+ np.sum([c.meanlogr for c in others], axis=0, out=self.meanlogr)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.npairs for c in others], axis=0, out=self.npairs)
+ self.xi = self.raw_xi
+ self.varxi = self.raw_varxi
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2, *, metric=None, num_threads=None, comm=None, low_mem=False,
+ initialize=True, finalize=True):
+ """Compute the correlation function.
+
+ Both arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the N field.
+ cat2 (Catalog): A catalog or list of catalogs for the K field.
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr2.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+ self._rk = None
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+
+ self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ vark = calculateVarK(cat2, low_mem=low_mem)
+ self.logger.info("vark = %f: sig_k = %f",vark,math.sqrt(vark))
+ self.finalize(vark)
+
+[docs] @depr_pos_kwargs
+ def calculateXi(self, *, rk=None):
+ r"""Calculate the correlation function possibly given another correlation function
+ that uses random points for the foreground objects.
+
+ - If rk is None, the simple correlation function :math:`\langle \kappa \rangle` is
+ returned.
+ - If rk is not None, then a compensated calculation is done:
+ :math:`\langle \kappa \rangle = (DK - RK)`, where DK represents the mean kappa
+ around the lenses and RK represents the mean kappa around random points.
+
+ After calling this function, the attributes ``xi``, ``varxi`` and ``cov`` will correspond
+ to the compensated values (if rk is provided). The raw, uncompensated values are
+ available as ``rawxi`` and ``raw_varxi``.
+
+ Parameters:
+ rk (NKCorrelation): The cross-correlation using random locations as the lenses (RK),
+ if desired. (default: None)
+
+ Returns:
+ Tuple containing
+
+ - xi = array of :math:`\xi(r)`
+ - varxi = array of variance estimates of :math:`\xi(r)`
+ """
+ if rk is not None:
+ self.xi = self.raw_xi - rk.xi
+ self._rk = rk
+
+ if rk.npatch1 not in (1,self.npatch1) or rk.npatch2 != self.npatch2:
+ raise RuntimeError("RK must be run with the same patches as DK")
+
+ if len(self.results) > 0:
+ # If there are any rk patch pairs that aren't in results (e.g. due to different
+ # edge effects among the various pairs in consideration), then we need to add
+ # some dummy results to make sure all the right pairs are computed when we make
+ # the vectors for the covariance matrix.
+ template = next(iter(self.results.values())) # Just need something to copy.
+ for ij in rk.results:
+ if ij in self.results: continue
+ new_cij = template.copy()
+ new_cij.xi.ravel()[:] = 0
+ new_cij.weight.ravel()[:] = 0
+ self.results[ij] = new_cij
+
+ self.cov = self.estimate_cov(self.var_method)
+ self.varxi.ravel()[:] = self.cov.diagonal()
+ else:
+ self.varxi = self.raw_varxi + rk.varxi
+ else:
+ self.xi = self.raw_xi
+ self.varxi = self.raw_varxi
+
+ return self.xi, self.varxi
+
+ def _calculate_xi_from_pairs(self, pairs):
+ self._sum([self.results[ij] for ij in pairs])
+ self._finalize()
+ if self._rk is not None:
+ # If rk has npatch1 = 1, adjust pairs appropriately
+ if self._rk.npatch1 == 1:
+ pairs = [(0,ij[1]) for ij in pairs if ij[0] == ij[1]]
+ # Make sure all ij are in the rk results (some might be missing, which is ok)
+ pairs = [ij for ij in pairs if self._rk._ok[ij[0],ij[1]]]
+ self._rk._calculate_xi_from_pairs(pairs)
+ self.xi -= self._rk.xi
+
+[docs] def write(self, file_name, * ,rk=None, file_type=None, precision=None,
+ write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ - If rk is None, the simple correlation function :math:`\langle \kappa \rangle(R)` is
+ used.
+ - If rk is not None, then a compensated calculation is done:
+ :math:`\langle \kappa \rangle = (DK - RK)`, where DK represents the mean kappa
+ around the lenses and RK represents the mean kappa around random points.
+
+ The output file will include the following columns:
+
+ ========== =========================================================
+ Column Description
+ ========== =========================================================
+ r_nom The nominal center of the bin in r
+ meanr The mean value :math:`\langle r\rangle` of pairs that
+ fell into each bin
+ meanlogr The mean value :math:`\langle \log(r)\rangle` of pairs
+ that fell into each bin
+ kappa The mean value :math:`\langle \kappa\rangle(r)`
+ sigma The sqrt of the variance estimate of
+ :math:`\langle \kappa\rangle`
+ weight The total weight contributing to each bin
+ npairs The total number of pairs in each bin
+ ========== =========================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ rk (NKCorrelation): The cross-correlation using random locations as the lenses (RK),
+ if desired. (default: None)
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing NK correlations to %s',file_name)
+ self.calculateXi(rk=rk)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results)
+
+ @property
+ def _write_col_names(self):
+ return ['r_nom','meanr','meanlogr','kappa','sigma','weight','npairs']
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.meanr, self.meanlogr,
+ self.xi, np.sqrt(self.varxi), self.weight, self.npairs ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `NKCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading NK correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ self.meanr = data['meanR'].reshape(s)
+ self.meanlogr = data['meanlogR'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meanr = data['meanr'].reshape(s)
+ self.meanlogr = data['meanlogr'].reshape(s)
+ self.xi = data['kappa'].reshape(s)
+ self.varxi = data['sigma'].reshape(s)**2
+ self.weight = data['weight'].reshape(s)
+ self.npairs = data['npairs'].reshape(s)
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.raw_xi = self.xi
+ self.raw_varxi = self.varxi
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: nncorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .binnedcorr2 import BinnedCorr2
+from .util import double_ptr as dp
+from .util import make_writer, make_reader, lazy_property
+from .util import depr_pos_kwargs
+
+
+[docs]class NNCorrelation(BinnedCorr2):
+ r"""This class handles the calculation and storage of a 2-point count-count correlation
+ function. i.e. the regular density correlation function.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ nbins: The number of bins in logr
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+
+ In addition, the following attributes are numpy arrays of length (nbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ meanr: The (weighted) mean value of r for the pairs in each bin.
+ If there are no pairs in a bin, then exp(logr) will be used instead.
+ meanlogr: The mean value of log(r) for the pairs in each bin.
+ If there are no pairs in a bin, then logr will be used instead.
+ weight: The total weight in each bin.
+ npairs: The number of pairs going into each bin (including pairs where one or
+ both objects have w=0).
+ tot: The total number of pairs processed, which is used to normalize
+ the randoms if they have a different number of pairs.
+
+ If `calculateXi` has been called, then the following will also be available:
+
+ Attributes:
+ xi: The correlation function, :math:`\xi(r)`
+ varxi: An estimate of the variance of :math:`\xi`
+ cov: An estimate of the full covariance matrix.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_auto` and/or
+ `process_cross`, then the units will not be applied to ``meanr`` or ``meanlogr`` until
+ the `finalize` function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> nn = treecorr.NNCorrelation(config)
+ >>> nn.process(cat) # For auto-correlation.
+ >>> nn.process(cat1,cat2) # For cross-correlation.
+ >>> rr.process... # Likewise for random-random correlations
+ >>> dr.process... # If desired, also do data-random correlations
+ >>> rd.process... # For cross-correlations, also do the reverse.
+ >>> nn.write(file_name,rr=rr,dr=dr,rd=rd) # Write out to a file.
+ >>> xi,varxi = nn.calculateXi(rr=rr,dr=dr,rd=rd) # Or get correlation function directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr2`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr2` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `NNCorrelation`. See class doc for details.
+ """
+ BinnedCorr2.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 1 # NData
+ self._ro._d2 = 1 # NData
+ self.meanr = np.zeros_like(self.rnom, dtype=float)
+ self.meanlogr = np.zeros_like(self.rnom, dtype=float)
+ self.weight = np.zeros_like(self.rnom, dtype=float)
+ self.npairs = np.zeros_like(self.rnom, dtype=float)
+ self.tot = 0.
+ self._rr_weight = None # Marker that calculateXi hasn't been called yet.
+ self._rr = None
+ self._dr = None
+ self._rd = None
+ self._write_rr = None
+ self._write_dr = None
+ self._write_rd = None
+ self.logger.debug('Finished building NNCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr2(
+ self._d1, self._d2, self._bintype,
+ self._min_sep,self._max_sep,self._nbins,self._bin_size,self.b,
+ self.min_rpar, self.max_rpar, self.xperiod, self.yperiod, self.zperiod,
+ dp(None), dp(None), dp(None), dp(None),
+ dp(self.meanr),dp(self.meanlogr),dp(self.weight),dp(self.npairs))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr2(self.corr, self._d1, self._d2, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `NNCorrelation` instances are equal"""
+ return (isinstance(other, NNCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.min_rpar == other.min_rpar and
+ self.max_rpar == other.max_rpar and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ self.tot == other.tot and
+ np.array_equal(self.meanr, other.meanr) and
+ np.array_equal(self.meanlogr, other.meanlogr) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.npairs, other.npairs))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = NNCorrelation.__new__(NNCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ if self._rd is not None:
+ ret._rd = self._rd.copy()
+ if self._dr is not None:
+ ret._dr = self._dr.copy()
+ if self._rr is not None:
+ ret._rr = self._rr.copy()
+ return ret
+
+ @lazy_property
+ def _zero_array(self):
+ # An array of all zeros with the same shape as self.weight (and other data arrays)
+ z = np.zeros_like(self.weight)
+ z.flags.writeable=False # Just to make sure we get an error if we try to change it.
+ return z
+
+ def _zero_copy(self, tot):
+ # A minimal "copy" with zero for the weight array, and the given value for tot.
+ ret = NNCorrelation.__new__(NNCorrelation)
+ ret._ro = self._ro
+ ret.coords = self.coords
+ ret.metric = self.metric
+ ret.config = self.config
+ ret.meanr = self._zero_array
+ ret.meanlogr = self._zero_array
+ ret.weight = self._zero_array
+ ret.npairs = self._zero_array
+ ret.tot = tot
+ ret._corr = None
+ ret._rr = ret._dr = ret._rd = None
+ ret._write_rr = ret._write_dr = ret._write_rd = None
+ # This override is really the main advantage of using this:
+ setattr(ret, '_nonzero', False)
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_auto(self, cat, *, metric=None, num_threads=None):
+ """Process a single catalog, accumulating the auto-correlation.
+
+ This accumulates the auto-correlation for the given catalog. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meanr, meanlogr.
+
+ Parameters:
+ cat (Catalog): The catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat.name == '':
+ self.logger.info('Starting process NN auto-correlations')
+ else:
+ self.logger.info('Starting process NN auto-correlations for cat %s.', cat.name)
+
+ self._set_metric(metric, cat.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ field = cat.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method, brute=bool(self.brute),
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',field.nTopLevelNodes)
+ _lib.ProcessAuto2(self.corr, field.data, self.output_dots,
+ field._d, self._coords, self._bintype, self._metric)
+ self.tot += 0.5 * cat.sumw**2
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meanr, meanlogr.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NN cross-correlations')
+ else:
+ self.logger.info('Starting process NN cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross2(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+ self.tot += cat1.sumw*cat2.sumw
+
+[docs] @depr_pos_kwargs
+ def process_pairwise(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process a single pair of catalogs, accumulating the cross-correlation, only using
+ the corresponding pairs of objects in each catalog.
+
+ This accumulates the sums into the bins, but does not finalize the calculation.
+ After calling this function as often as desired, the `finalize` command will
+ finish the calculation.
+
+ .. warning::
+
+ .. deprecated:: 4.1
+
+ This function is deprecated and slated to be removed.
+ If you have a need for it, please open an issue to describe your use case.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ import warnings
+ warnings.warn("The process_pairwise function is slated to be removed in a future version. "+
+ "If you are actually using this function usefully, please "+
+ "open an issue to describe your use case.", FutureWarning)
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NN pairwise-correlations')
+ else:
+ self.logger.info('Starting process NN pairwise-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+
+ f1 = cat1.getNSimpleField()
+ f2 = cat2.getNSimpleField()
+
+ _lib.ProcessPair(self.corr, f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords, self._bintype, self._metric)
+ self.tot += (cat1.sumw+cat2.sumw)/2.
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.meanr[mask1] /= self.weight[mask1]
+ self.meanlogr[mask1] /= self.weight[mask1]
+
+ # Update the units of meanr, meanlogr
+ self._apply_units(mask1)
+
+ # Use meanr, meanlogr when available, but set to nominal when no pairs in bin.
+ self.meanr[mask2] = self.rnom[mask2]
+ self.meanlogr[mask2] = self.logr[mask2]
+
+[docs] def finalize(self):
+ """Finalize the calculation of the correlation function.
+
+ The `process_auto` and `process_cross` commands accumulate values in each bin,
+ so they can be called multiple times if appropriate. Afterwards, this command
+ finishes the calculation of meanr, meanlogr by dividing by the total weight.
+ """
+ self._finalize()
+
+ @lazy_property
+ def _nonzero(self):
+ # The lazy version when we can be sure the object isn't going to accumulate any more.
+ return self.nonzero
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.meanr.ravel()[:] = 0.
+ self.meanlogr.ravel()[:] = 0.
+ self.weight.ravel()[:] = 0.
+ self.npairs.ravel()[:] = 0.
+ self.tot = 0.
+
+[docs] def __iadd__(self, other):
+ """Add a second `NNCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `NNCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, NNCorrelation):
+ raise TypeError("Can only add another NNCorrelation object")
+ if not (self._nbins == other._nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep):
+ raise ValueError("NNCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords)
+ self.meanr.ravel()[:] += other.meanr.ravel()[:]
+ self.meanlogr.ravel()[:] += other.meanlogr.ravel()[:]
+ self.weight.ravel()[:] += other.weight.ravel()[:]
+ self.npairs.ravel()[:] += other.npairs.ravel()[:]
+ self.tot += other.tot
+ return self
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ tot = np.sum([c.tot for c in others])
+ # Empty ones were only needed for tot. Remove them now.
+ others = [c for c in others if c._nonzero]
+ if len(others) == 0:
+ self._clear()
+ else:
+ np.sum([c.meanr for c in others], axis=0, out=self.meanr)
+ np.sum([c.meanlogr for c in others], axis=0, out=self.meanlogr)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.npairs for c in others], axis=0, out=self.npairs)
+ self.tot = tot
+
+ def _add_tot(self, i, j, c1, c2):
+ # When storing results from a patch-based run, tot needs to be accumulated even if
+ # the total weight being accumulated comes out to be zero.
+ # This only applies to NNCorrelation. For the other ones, this is a no op.
+ tot = c1.sumw * c2.sumw
+ self.tot += tot
+ # We also have to keep all pairs in the results dict, otherwise the tot calculation
+ # gets messed up. We need to accumulate the tot value of all pairs, even if
+ # the resulting weight is zero. But use a minimal copy with just the necessary fields
+ # to save some time.
+ self.results[(i,j)] = self._zero_copy(tot)
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2=None, *, metric=None, num_threads=None, comm=None, low_mem=False,
+ initialize=True, finalize=True):
+ """Compute the correlation function.
+
+ - If only 1 argument is given, then compute an auto-correlation function.
+ - If 2 arguments are given, then compute a cross-correlation function.
+
+ Both arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first N field.
+ cat2 (Catalog): A catalog or list of catalogs for the second N field, if any.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr2.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list):
+ cat1 = cat1.get_patches(low_mem=low_mem)
+ if cat2 is not None and not isinstance(cat2,list):
+ cat2 = cat2.get_patches(low_mem=low_mem)
+
+ if cat2 is None or len(cat2) == 0:
+ self._process_all_auto(cat1, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ self.finalize()
+
+ def _mean_weight(self):
+ mean_np = np.mean(self.npairs)
+ return 1 if mean_np == 0 else np.mean(self.weight)/mean_np
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ This raises a RuntimeError if calculateXi has not been run yet.
+ """
+ if self._rr is None:
+ raise RuntimeError("You need to call calculateXi before calling estimate_cov.")
+ return self.xi.ravel()
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ This is the weight array corresponding to `getStat`. In this case, it is the denominator
+ RR from the calculation done by calculateXi().
+ """
+ if self._rr_weight is not None:
+ return self._rr_weight.ravel()
+ else:
+ return self.tot
+
+[docs] @depr_pos_kwargs
+ def calculateXi(self, *, rr, dr=None, rd=None):
+ r"""Calculate the correlation function given another correlation function of random
+ points using the same mask, and possibly cross correlations of the data and random.
+
+ The rr value is the `NNCorrelation` function for random points.
+ For a signal that involves a cross correlations, there should be two random
+ cross-correlations: data-random and random-data, given as dr and rd.
+
+ - If dr is None, the simple correlation function :math:`\xi = (DD/RR - 1)` is used.
+ - if dr is given and rd is None, then :math:`\xi = (DD - 2DR + RR)/RR` is used.
+ - If dr and rd are both given, then :math:`\xi = (DD - DR - RD + RR)/RR` is used.
+
+ where DD is the data NN correlation function, which is the current object.
+
+ .. note::
+
+ The default method for estimating the variance is 'shot', which only includes the
+ shot noise propagated into the final correlation. This does not include sample
+ variance, so it is always an underestimate of the actual variance. To get better
+ estimates, you need to set ``var_method`` to something else and use patches in the
+ input catalog(s). cf. `Covariance Estimates`.
+
+ After calling this method, you can use the `BinnedCorr2.estimate_cov` method or use this
+ correlation object in the `estimate_multi_cov` function. Also, the calculated xi and
+ varxi returned from this function will be available as attributes.
+
+ Parameters:
+ rr (NNCorrelation): The auto-correlation of the random field (RR)
+ dr (NNCorrelation): The cross-correlation of the data with randoms (DR), if
+ desired, in which case the Landy-Szalay estimator will be
+ calculated. (default: None)
+ rd (NNCorrelation): The cross-correlation of the randoms with data (RD), if
+ desired. (default: None, which means use rd=dr)
+
+ Returns:
+ Tuple containing:
+
+ - xi = array of :math:`\xi(r)`
+ - varxi = an estimate of the variance of :math:`\xi(r)`
+ """
+ # Each random weight value needs to be rescaled by the ratio of total possible pairs.
+ if rr.tot == 0:
+ raise ValueError("rr has tot=0.")
+
+ # rrf is the factor to scale rr weights to get something commensurate to the dd density.
+ rrf = self.tot / rr.tot
+
+ # Likewise for the other two potential randoms:
+ if dr is not None:
+ if dr.tot == 0:
+ raise ValueError("dr has tot=0.")
+ drf = self.tot / dr.tot
+ if rd is not None:
+ if rd.tot == 0:
+ raise ValueError("rd has tot=0.")
+ rdf = self.tot / rd.tot
+
+ # Calculate xi based on which randoms are provided.
+ denom = rr.weight * rrf
+ if dr is None and rd is None:
+ self.xi = self.weight - denom
+ elif rd is not None and dr is None:
+ self.xi = self.weight - 2.*rd.weight * rdf + denom
+ elif dr is not None and rd is None:
+ self.xi = self.weight - 2.*dr.weight * drf + denom
+ else:
+ self.xi = self.weight - rd.weight * rdf - dr.weight * drf + denom
+
+ # Divide by RR in all cases.
+ if np.any(rr.weight == 0):
+ self.logger.warning("Warning: Some bins for the randoms had no pairs.")
+ denom[rr.weight==0] = 1. # guard against division by 0.
+ self.xi /= denom
+
+ # Set up necessary info for estimate_cov
+
+ # First the bits needed for shot noise covariance:
+ ddw = self._mean_weight()
+ rrw = rr._mean_weight()
+ if dr is not None:
+ drw = dr._mean_weight()
+ if rd is not None:
+ rdw = rd._mean_weight()
+
+ # Note: The use of varxi_factor for the shot noise varxi is semi-empirical.
+ # It gives the increase in the variance over the case where RR >> DD.
+ # I don't have a good derivation that this is the right factor to apply
+ # when the random catalog is not >> larger than the data.
+ # When I tried to derive this from first principles, I get the below formula,
+ # but without the **2. So I'm not sure why this factor needs to be squared.
+ # It seems at least plausible that I missed something in the derivation that
+ # leads to this getting squared, but I can't really justify it.
+ # But it's also possible that this is wrong...
+ # Anyway, it seems to give good results compared to the empirical variance.
+ # cf. test_nn.py:test_varxi
+ if dr is None and rd is None:
+ varxi_factor = 1 + rrf*rrw/ddw
+ elif rd is not None and dr is None:
+ varxi_factor = 1 + 2*rdf*rdw/ddw + rrf*rrw/ddw
+ elif dr is not None and rd is None:
+ varxi_factor = 1 + 2*drf*drw/ddw + rrf*rrw/ddw
+ else:
+ varxi_factor = 1 + drf*drw/ddw + rdf*rdw/ddw + rrf*rrw/ddw
+ self._var_num = ddw * varxi_factor**2
+ self._rr_weight = rr.weight * rrf
+
+ # Now set up the bits needed for patch-based covariance
+ self._rr = rr
+ self._dr = dr
+ self._rd = rd
+
+ if len(self.results) > 0:
+ # Check that rr,dr,rd use the same patches as dd
+ if rr.npatch1 != 1 and rr.npatch2 != 1:
+ if rr.npatch1 != self.npatch1 or rr.npatch2 != self.npatch2:
+ raise RuntimeError("If using patches, RR must be run with the same patches "
+ "as DD")
+
+ if dr is not None and (len(dr.results) == 0 or dr.npatch1 != self.npatch1 or
+ dr.npatch2 not in (self.npatch2, 1)):
+ raise RuntimeError("DR must be run with the same patches as DD")
+ if rd is not None and (len(rd.results) == 0 or rd.npatch2 != self.npatch2 or
+ rd.npatch1 not in (self.npatch1, 1)):
+ raise RuntimeError("RD must be run with the same patches as DD")
+
+ # If there are any rr,rd,dr patch pairs that aren't in results (because dr is a cross
+ # correlation, and dd,rr may be auto-correlations, or because the d catalogs has some
+ # patches with no items), then we need to add some dummy results to make sure all the
+ # right pairs are computed when we make the vectors for the covariance matrix.
+ add_ij = set()
+ if rr.npatch1 != 1 and rr.npatch2 != 1:
+ for ij in rr.results:
+ if ij not in self.results:
+ add_ij.add(ij)
+
+ if dr is not None and dr.npatch2 != 1:
+ for ij in dr.results:
+ if ij not in self.results:
+ add_ij.add(ij)
+
+ if rd is not None and rd.npatch1 != 1:
+ for ij in rd.results:
+ if ij not in self.results:
+ add_ij.add(ij)
+
+ if len(add_ij) > 0:
+ for ij in add_ij:
+ self.results[ij] = self._zero_copy(0)
+ self.__dict__.pop('_ok',None) # If it was already made, it will need to be redone.
+
+ # Now that it's all set up, calculate the covariance and set varxi to the diagonal.
+ self.cov = self.estimate_cov(self.var_method)
+ self.varxi = self.cov.diagonal()
+ return self.xi, self.varxi
+
+ def _calculate_xi_from_pairs(self, pairs):
+ self._sum([self.results[ij] for ij in pairs])
+ self._finalize()
+ if self._rr is None:
+ return
+ dd = self.weight
+ if len(self._rr.results) > 0:
+ # This is the usual case. R has patches just like D.
+ # Calculate rr and rrf in the normal way based on the same pairs as used for DD.
+ pairs1 = [ij for ij in pairs if self._rr._ok[ij[0],ij[1]]]
+ self._rr._sum([self._rr.results[ij] for ij in pairs1])
+ dd_tot = self.tot
+ else:
+ # In this case, R was not run with patches.
+ # This is not necessarily much worse in practice it turns out.
+ # We just need to scale RR down by the relative area.
+ # The approximation we'll use is that tot in the auto-correlations is
+ # proportional to area**2.
+ # So the sum of tot**0.5 when i==j gives an estimate of the fraction of the total area.
+ area_frac = np.sum([self.results[ij].tot**0.5 for ij in pairs if ij[0] == ij[1]])
+ area_frac /= np.sum([cij.tot**0.5 for ij,cij in self.results.items() if ij[0] == ij[1]])
+ # First figure out the original total for all DD that had the same footprint as RR.
+ dd_tot = np.sum([self.results[ij].tot for ij in self.results])
+ # The rrf we want will be a factor of area_frac smaller than the original
+ # dd_tot/rr_tot. We can effect this by multiplying the full dd_tot by area_frac
+ # and use that value normally below. (Also for drf and rdf.)
+ dd_tot *= area_frac
+
+ rr = self._rr.weight
+ rrf = dd_tot / self._rr.tot
+
+ if self._dr is not None:
+ if self._dr.npatch2 == 1:
+ # If r doesn't have patches, then convert all (i,i) pairs to (i,0).
+ pairs2 = [(ij[0],0) for ij in pairs if ij[0] == ij[1]]
+ else:
+ pairs2 = [ij for ij in pairs if self._dr._ok[ij[0],ij[1]]]
+ self._dr._sum([self._dr.results[ij] for ij in pairs2])
+ dr = self._dr.weight
+ drf = dd_tot / self._dr.tot
+ if self._rd is not None:
+ if self._rd.npatch1 == 1:
+ # If r doesn't have patches, then convert all (i,i) pairs to (0,i).
+ pairs3 = [(0,ij[1]) for ij in pairs if ij[0] == ij[1]]
+ else:
+ pairs3 = [ij for ij in pairs if self._rd._ok[ij[0],ij[1]]]
+ self._rd._sum([self._rd.results[ij] for ij in pairs3])
+ rd = self._rd.weight
+ rdf = dd_tot / self._rd.tot
+ denom = rr * rrf
+ if self._dr is None and self._rd is None:
+ xi = dd - denom
+ elif self._rd is not None and self._dr is None:
+ xi = dd - 2.*rd * rdf + denom
+ elif self._dr is not None and self._rd is None:
+ xi = dd - 2.*dr * drf + denom
+ else:
+ xi = dd - rd * rdf - dr * drf + denom
+ denom[denom == 0] = 1 # Guard against division by zero.
+ self.xi = xi / denom
+ self._rr_weight = denom
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, rr=None, dr=None, rd=None, file_type=None, precision=None,
+ write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ rr is the `NNCorrelation` function for random points.
+ If dr is None, the simple correlation function :math:`\xi = (DD - RR)/RR` is used.
+ if dr is given and rd is None, then :math:`\xi = (DD - 2DR + RR)/RR` is used.
+ If dr and rd are both given, then :math:`\xi = (DD - DR - RD + RR)/RR` is used.
+
+ Normally, at least rr should be provided, but if this is also None, then only the
+ basic accumulated number of pairs are output (along with the separation columns).
+
+ The output file will include the following columns:
+
+ ========== ==========================================================
+ Column Description
+ ========== ==========================================================
+ r_nom The nominal center of the bin in r
+ meanr The mean value :math:`\langle r\rangle` of pairs that fell
+ into each bin
+ meanlogr The mean value :math:`\langle \log(r)\rangle` of pairs that
+ fell into each bin
+ xi The estimator :math:`\xi` (if rr is given, or calculateXi
+ has been called)
+ sigma_xi The sqrt of the variance estimate of xi (if rr is given
+ or calculateXi has been called)
+ DD The total weight of pairs in each bin.
+ RR The total weight of RR pairs in each bin (if rr is given)
+ DR The total weight of DR pairs in each bin (if dr is given)
+ RD The total weight of RD pairs in each bin (if rd is given)
+ npairs The total number of pairs in each bin
+ ========== ==========================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ rr (NNCorrelation): The auto-correlation of the random field (RR)
+ dr (NNCorrelation): The cross-correlation of the data with randoms (DR), if
+ desired. (default: None)
+ rd (NNCorrelation): The cross-correlation of the randoms with data (RD), if
+ desired. (default: None, which means use rd=dr)
+ file_type (str): The type of file to write ('ASCII' or 'FITS').
+ (default: determine the type automatically from the extension
+ of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config
+ dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing NN correlations to %s',file_name)
+ # Temporary attributes, so the helper functions can access them.
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ self._write_rr = rr
+ self._write_dr = dr
+ self._write_rd = rd
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results, zero_tot=True)
+ self._write_rr = None
+ self._write_dr = None
+ self._write_rd = None
+
+ @property
+ def _write_col_names(self):
+ col_names = [ 'r_nom','meanr','meanlogr' ]
+ rr = self._write_rr
+ dr = self._write_dr
+ rd = self._write_rd
+ if rr is None:
+ if hasattr(self, 'xi'):
+ col_names += [ 'xi','sigma_xi' ]
+ col_names += [ 'DD', 'npairs' ]
+ else:
+ col_names += [ 'xi','sigma_xi','DD','RR' ]
+ if dr is not None and rd is not None:
+ col_names += ['DR','RD']
+ elif dr is not None or rd is not None:
+ col_names += ['DR']
+ col_names += [ 'npairs' ]
+ return col_names
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.meanr, self.meanlogr ]
+ rr = self._write_rr
+ dr = self._write_dr
+ rd = self._write_rd
+ if rr is None:
+ if hasattr(self, 'xi'):
+ data += [ self.xi, np.sqrt(self.varxi) ]
+ data += [ self.weight, self.npairs ]
+ if dr is not None:
+ raise TypeError("rr must be provided if dr is not None")
+ if rd is not None:
+ raise TypeError("rr must be provided if rd is not None")
+ else:
+ xi, varxi = self.calculateXi(rr=rr, dr=dr, rd=rd)
+ data += [ xi, np.sqrt(varxi),
+ self.weight, rr.weight * (self.tot/rr.tot) ]
+ if dr is not None and rd is not None:
+ data += [ dr.weight * (self.tot/dr.tot), rd.weight * (self.tot/rd.tot) ]
+ elif dr is not None or rd is not None:
+ if dr is None: dr = rd
+ data += [ dr.weight * (self.tot/dr.tot) ]
+ data += [ self.npairs ]
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'tot' : self.tot, 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `NNCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading NN correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ self.meanr = data['meanR'].reshape(s)
+ self.meanlogr = data['meanlogR'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meanr = data['meanr'].reshape(s)
+ self.meanlogr = data['meanlogr'].reshape(s)
+ self.weight = data['DD'].reshape(s)
+ self.npairs = data['npairs'].reshape(s)
+ self.tot = params['tot']
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ if 'xi' in data.dtype.names:
+ self.xi = data['xi'].reshape(s)
+ self.varxi = data['sigma_xi'].reshape(s)**2
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+
+[docs] @depr_pos_kwargs
+ def calculateNapSq(self, *, rr, R=None, dr=None, rd=None, m2_uform=None):
+ r"""Calculate the corrollary to the aperture mass statistics for counts.
+
+ .. math::
+
+ \langle N_{ap}^2 \rangle(R) &= \int_{0}^{rmax} \frac{r dr}{2R^2}
+ \left [ T_+\left(\frac{r}{R}\right) \xi(r) \right] \\
+
+ The ``m2_uform`` parameter sets which definition of the aperture mass to use.
+ The default is to use 'Crittenden'.
+
+ If ``m2_uform`` is 'Crittenden':
+
+ .. math::
+
+ U(r) &= \frac{1}{2\pi} (1-r^2) \exp(-r^2/2) \\
+ T_+(s) &= \frac{s^4 - 16s^2 + 32}{128} \exp(-s^2/4) \\
+ rmax &= \infty
+
+ cf. Crittenden, et al (2002): ApJ, 568, 20
+
+ If ``m2_uform`` is 'Schneider':
+
+ .. math::
+
+ U(r) &= \frac{9}{\pi} (1-r^2) (1/3-r^2) \\
+ T_+(s) &= \frac{12}{5\pi} (2-15s^2) \arccos(s/2) \\
+ &\qquad + \frac{1}{100\pi} s \sqrt{4-s^2} (120 + 2320s^2 - 754s^4 + 132s^6 - 9s^8) \\
+ rmax &= 2R
+
+ cf. Schneider, et al (2002): A&A, 389, 729
+
+ This is used by `NGCorrelation.writeNorm`. See that function and also
+ `GGCorrelation.calculateMapSq` for more details.
+
+ Parameters:
+ rr (NNCorrelation): The auto-correlation of the random field (RR)
+ R (array): The R values at which to calculate the aperture mass statistics.
+ (default: None, which means use self.rnom)
+ dr (NNCorrelation): The cross-correlation of the data with randoms (DR), if
+ desired. (default: None)
+ rd (NNCorrelation): The cross-correlation of the randoms with data (RD), if
+ desired. (default: None, which means use rd=dr)
+ m2_uform (str): Which form to use for the aperture mass. (default: 'Crittenden';
+ this value can also be given in the constructor in the config dict.)
+
+ Returns:
+ Tuple containing
+
+ - nsq = array of :math:`\langle N_{ap}^2 \rangle(R)`
+ - varnsq = array of variance estimates of this value
+ """
+ if m2_uform is None:
+ m2_uform = self.config.get('m2_uform', 'Crittenden')
+ if m2_uform not in ['Crittenden', 'Schneider']:
+ raise ValueError("Invalid m2_uform")
+ if R is None:
+ R = self.rnom
+
+ # Make s a matrix, so we can eventually do the integral by doing a matrix product.
+ s = np.outer(1./R, self.meanr)
+ ssq = s*s
+ if m2_uform == 'Crittenden':
+ exp_factor = np.exp(-ssq/4.)
+ Tp = (32. + ssq*(-16. + ssq)) / 128. * exp_factor
+ else:
+ Tp = np.zeros_like(s)
+ sa = s[s<2.]
+ ssqa = ssq[s<2.]
+ Tp[s<2.] = 12./(5.*np.pi) * (2.-15.*ssqa) * np.arccos(sa/2.)
+ Tp[s<2.] += 1./(100.*np.pi) * sa * np.sqrt(4.-ssqa) * (
+ 120. + ssqa*(2320. + ssqa*(-754. + ssqa*(132. - 9.*ssqa))))
+ Tp *= ssq
+
+ xi, varxi = self.calculateXi(rr=rr, dr=dr, rd=rd)
+
+ # Now do the integral by taking the matrix products.
+ # Note that dlogr = bin_size
+ Tpxi = Tp.dot(xi)
+ nsq = Tpxi * self.bin_size
+ varnsq = (Tp**2).dot(varxi) * self.bin_size**2
+
+ return nsq, varnsq
+
+# Copyright (c) 2003-2019 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+.. module:: nnncorrelation
+"""
+
+import numpy as np
+
+from . import _lib, _ffi
+from .binnedcorr3 import BinnedCorr3
+from .util import double_ptr as dp
+from .util import make_writer, make_reader, lazy_property
+from .util import depr_pos_kwargs
+
+
+[docs]class NNNCorrelation(BinnedCorr3):
+ """This class handles the calculation and storage of a 2-point count-count correlation
+ function. i.e. the regular density correlation function.
+
+ See the doc string of `BinnedCorr3` for a description of how the triangles are binned.
+
+ Ojects of this class holds the following attributes:
+
+ Attributes:
+ logr: The nominal center of the bin in log(r) (the natural logarithm of r).
+ nbins: The number of bins in logr where r = d2
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+ nubins: The number of bins in u where u = d3/d2
+ ubin_size: The size of the bins in u
+ min_u: The minimum u being considered
+ max_u: The maximum u being considered
+ nvbins: The number of bins in v where v = +-(d1-d2)/d3
+ vbin_size: The size of the bins in v
+ min_v: The minimum v being considered
+ max_v: The maximum v being considered
+ logr1d: The nominal centers of the nbins bins in log(r).
+ u1d: The nominal centers of the nubins bins in u.
+ v1d: The nominal centers of the nvbins bins in v.
+
+ In addition, the following attributes are numpy arrays whose shape is (nbins, nubins, nvbins):
+
+ Attributes:
+ logr: The nominal center of the bin in log(r).
+ rnom: The nominal center of the bin converted to regular distance.
+ i.e. r = exp(logr).
+ u: The nominal center of the bin in u.
+ v: The nominal center of the bin in v.
+ meand1: The (weighted) mean value of d1 for the triangles in each bin.
+ meanlogd1: The mean value of log(d1) for the triangles in each bin.
+ meand2: The (weighted) mean value of d2 (aka r) for the triangles in each bin.
+ meanlogd2: The mean value of log(d2) for the triangles in each bin.
+ meand2: The (weighted) mean value of d3 for the triangles in each bin.
+ meanlogd2: The mean value of log(d3) for the triangles in each bin.
+ meanu: The mean value of u for the triangles in each bin.
+ meanv: The mean value of v for the triangles in each bin.
+ weight: The total weight in each bin.
+ ntri: The number of triangles going into each bin (including those where one or
+ more objects have w=0).
+ tot: The total number of triangles processed, which is used to normalize
+ the randoms if they have a different number of triangles.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_auto` and/or
+ `process_cross`, then the units will not be applied to ``meanr`` or ``meanlogr`` until
+ the `finalize` function is called.
+
+ The typical usage pattern is as follows:
+
+ >>> nnn = treecorr.NNNCorrelation(config)
+ >>> nnn.process(cat) # For auto-correlation.
+ >>> rrr.process(rand) # Likewise for random-random correlations
+ >>> drr.process(cat,rand) # If desired, also do data-random correlations
+ >>> rdd.process(rand,cat) # Also with two data and one random
+ >>> nnn.write(file_name,rrr=rrr,drr=drr,...) # Write out to a file.
+ >>> zeta,varzeta = nnn.calculateZeta(rrr=rrr,drr=drr,rdd=rdd) # Or get zeta directly.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr3`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr3` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `NNNCorrelation`. See class doc for details.
+ """
+ BinnedCorr3.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 1 # NData
+ self._ro._d2 = 1 # NData
+ self._ro._d3 = 1 # NData
+ shape = self.logr.shape
+ self.meand1 = np.zeros(shape, dtype=float)
+ self.meanlogd1 = np.zeros(shape, dtype=float)
+ self.meand2 = np.zeros(shape, dtype=float)
+ self.meanlogd2 = np.zeros(shape, dtype=float)
+ self.meand3 = np.zeros(shape, dtype=float)
+ self.meanlogd3 = np.zeros(shape, dtype=float)
+ self.meanu = np.zeros(shape, dtype=float)
+ self.meanv = np.zeros(shape, dtype=float)
+ self.weight = np.zeros(shape, dtype=float)
+ self.ntri = np.zeros(shape, dtype=float)
+ self.tot = 0.
+ self._rrr_weight = None
+ self._rrr = None
+ self._drr = None
+ self._rdd = None
+ self._write_rrr = None
+ self._write_drr = None
+ self._write_rdd = None
+ self.logger.debug('Finished building NNNCorr')
+
+ @property
+ def corr(self):
+ if self._corr is None:
+ self._corr = _lib.BuildCorr3(
+ self._d1, self._d2, self._d3, self._bintype,
+ self._min_sep,self._max_sep,self.nbins,self._bin_size,self.b,
+ self.min_u,self.max_u,self.nubins,self.ubin_size,self.bu,
+ self.min_v,self.max_v,self.nvbins,self.vbin_size,self.bv,
+ self.xperiod, self.yperiod, self.zperiod,
+ dp(None), dp(None), dp(None), dp(None),
+ dp(None), dp(None), dp(None), dp(None),
+ dp(self.meand1), dp(self.meanlogd1), dp(self.meand2), dp(self.meanlogd2),
+ dp(self.meand3), dp(self.meanlogd3), dp(self.meanu), dp(self.meanv),
+ dp(self.weight), dp(self.ntri))
+ return self._corr
+
+ def __del__(self):
+ # Using memory allocated from the C layer means we have to explicitly deallocate it
+ # rather than being able to rely on the Python memory manager.
+ if self._corr is not None:
+ if not _ffi._lock.locked(): # pragma: no branch
+ _lib.DestroyCorr3(self.corr, self._d1, self._d2, self._d3, self._bintype)
+
+[docs] def __eq__(self, other):
+ """Return whether two `NNNCorrelation` instances are equal"""
+ return (isinstance(other, NNNCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nubins == other.nubins and
+ self.ubin_size == other.ubin_size and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v and
+ self.nvbins == other.nvbins and
+ self.vbin_size == other.vbin_size and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ self.tot == other.tot and
+ np.array_equal(self.meand1, other.meand1) and
+ np.array_equal(self.meanlogd1, other.meanlogd1) and
+ np.array_equal(self.meand2, other.meand2) and
+ np.array_equal(self.meanlogd2, other.meanlogd2) and
+ np.array_equal(self.meand3, other.meand3) and
+ np.array_equal(self.meanlogd3, other.meanlogd3) and
+ np.array_equal(self.meanu, other.meanu) and
+ np.array_equal(self.meanv, other.meanv) and
+ np.array_equal(self.weight, other.weight) and
+ np.array_equal(self.ntri, other.ntri))
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = NNNCorrelation.__new__(NNNCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, np.ndarray):
+ # Only items that might change need to by deep copied.
+ ret.__dict__[key] = item.copy()
+ else:
+ # For everything else, shallow copy is fine.
+ # In particular don't deep copy config or logger
+ # Most of the rest are scalars, which copy fine this way.
+ # And the read-only things are all in _ro.
+ # The results dict is trickier. We rely on it being copied in places, but we
+ # never add more to it after the copy, so shallow copy is fine.
+ ret.__dict__[key] = item
+ ret._corr = None # We'll want to make a new one of these if we need it.
+ if self._drr is not None:
+ ret._drr = self._drr.copy()
+ if self._rdd is not None:
+ ret._rdd = self._rdd.copy()
+ if self._rrr is not None:
+ ret._rrr = self._rrr.copy()
+ return ret
+
+ @lazy_property
+ def _zero_array(self):
+ # An array of all zeros with the same shape as self.weight (and other data arrays)
+ z = np.zeros_like(self.weight)
+ z.flags.writeable=False # Just to make sure we get an error if we try to change it.
+ return z
+
+ def _zero_copy(self, tot):
+ # A minimal "copy" with zero for the weight array, and the given value for tot.
+ ret = NNNCorrelation.__new__(NNNCorrelation)
+ ret._ro = self._ro
+ ret.coords = self.coords
+ ret.metric = self.metric
+ ret.config = self.config
+ ret.meand1 = self._zero_array
+ ret.meanlogd1 = self._zero_array
+ ret.meand2 = self._zero_array
+ ret.meanlogd2 = self._zero_array
+ ret.meand3 = self._zero_array
+ ret.meanlogd3 = self._zero_array
+ ret.meanu = self._zero_array
+ ret.meanv = self._zero_array
+ ret.weight = self._zero_array
+ ret.ntri = self._zero_array
+ ret.tot = tot
+ ret._corr = None
+ ret._rrr = ret._drr = ret._rdd = None
+ ret._write_rrr = ret._write_drr = ret._write_rdd = None
+ # This override is really the main advantage of using this:
+ setattr(ret, '_nonzero', False)
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_auto(self, cat, *, metric=None, num_threads=None):
+ """Process a single catalog, accumulating the auto-correlation.
+
+ This accumulates the auto-correlation for the given catalog. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ Parameters:
+ cat (Catalog): The catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat.name == '':
+ self.logger.info('Starting process NNN auto-correlations')
+ else:
+ self.logger.info('Starting process NNN auto-correlations for cat %s.', cat.name)
+
+ self._set_metric(metric, cat.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ field = cat.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method, brute=bool(self.brute),
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',field.nTopLevelNodes)
+ _lib.ProcessAuto3(self.corr, field.data, self.output_dots,
+ field._d, self._coords, self._bintype, self._metric)
+ self.tot += (1./6.) * cat.sumw**3
+
+[docs] @depr_pos_kwargs
+ def process_cross12(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process two catalogs, accumulating the 3pt cross-correlation, where one of the
+ points in each triangle come from the first catalog, and two come from the second.
+
+ This accumulates the cross-correlation for the given catalogs as part of a larger
+ auto-correlation calculation. E.g. when splitting up a large catalog into patches,
+ this is appropriate to use for the cross correlation between different patches
+ as part of the complete auto-correlation of the full catalog.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process. (1 point in each triangle will come
+ from this catalog.)
+ cat2 (Catalog): The second catalog to process. (2 points in each triangle will come
+ from this catalog.)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NNN (1-2) cross-correlations')
+ else:
+ self.logger.info('Starting process NNN (1-2) cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 3 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross12(self.corr, self.corr, self.corr,
+ f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords,
+ self._bintype, self._metric)
+ self.tot += cat1.sumw * cat2.sumw**2 / 2.
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, cat3, *, metric=None, num_threads=None):
+ """Process a set of three catalogs, accumulating the 3pt cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs as part of a larger
+ auto-correlation calculation. E.g. when splitting up a large catalog into patches,
+ this is appropriate to use for the cross correlation between different patches
+ as part of the complete auto-correlation of the full catalog.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ cat3 (Catalog): The third catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '' and cat3.name == '':
+ self.logger.info('Starting process NNN cross-correlations')
+ else:
+ self.logger.info('Starting process NNN cross-correlations for cats %s, %s, %s.',
+ cat1.name, cat2.name, cat3.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords, cat3.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f3 = cat3.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 3,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 6 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross3(self.corr, self.corr, self.corr,
+ self.corr, self.corr, self.corr,
+ f1.data, f2.data, f3.data, self.output_dots,
+ f1._d, f2._d, f3._d, self._coords, self._bintype, self._metric)
+ self.tot += cat1.sumw * cat2.sumw * cat3.sumw
+
+ def _finalize(self):
+ mask1 = self.weight != 0
+ mask2 = self.weight == 0
+
+ self.meand1[mask1] /= self.weight[mask1]
+ self.meanlogd1[mask1] /= self.weight[mask1]
+ self.meand2[mask1] /= self.weight[mask1]
+ self.meanlogd2[mask1] /= self.weight[mask1]
+ self.meand3[mask1] /= self.weight[mask1]
+ self.meanlogd3[mask1] /= self.weight[mask1]
+ self.meanu[mask1] /= self.weight[mask1]
+ self.meanv[mask1] /= self.weight[mask1]
+
+ # Update the units
+ self._apply_units(mask1)
+
+ # Use meanlogr when available, but set to nominal when no triangles in bin.
+ self.meand2[mask2] = self.rnom[mask2]
+ self.meanlogd2[mask2] = self.logr[mask2]
+ self.meanu[mask2] = self.u[mask2]
+ self.meanv[mask2] = self.v[mask2]
+ self.meand3[mask2] = self.u[mask2] * self.meand2[mask2]
+ self.meanlogd3[mask2] = np.log(self.meand3[mask2])
+ self.meand1[mask2] = self.v[mask2] * self.meand3[mask2] + self.meand2[mask2]
+ self.meanlogd1[mask2] = np.log(self.meand1[mask2])
+
+[docs] def finalize(self):
+ """Finalize the calculation of meand1, meanlogd1, etc.
+
+ The `process_auto` and `process_cross` commands accumulate values in each bin,
+ so they can be called multiple times if appropriate. Afterwards, this command
+ finishes the calculation of meanlogr, meanu, meanv by dividing by the total weight.
+ """
+ self._finalize()
+
+ @lazy_property
+ def _nonzero(self):
+ # The lazy version when we can be sure the object isn't going to accumulate any more.
+ return self.nonzero
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ self.meand1[:,:,:] = 0.
+ self.meanlogd1[:,:,:] = 0.
+ self.meand2[:,:,:] = 0.
+ self.meanlogd2[:,:,:] = 0.
+ self.meand3[:,:,:] = 0.
+ self.meanlogd3[:,:,:] = 0.
+ self.meanu[:,:,:] = 0.
+ self.meanv[:,:,:] = 0.
+ self.weight[:,:,:] = 0.
+ self.ntri[:,:,:] = 0.
+ self.tot = 0.
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ tot = np.sum([c.tot for c in others])
+ # Empty ones were only needed for tot. Remove them now.
+ others = [c for c in others if c._nonzero]
+ if len(others) == 0:
+ self._clear()
+ else:
+ np.sum([c.meand1 for c in others], axis=0, out=self.meand1)
+ np.sum([c.meanlogd1 for c in others], axis=0, out=self.meanlogd1)
+ np.sum([c.meand2 for c in others], axis=0, out=self.meand2)
+ np.sum([c.meanlogd2 for c in others], axis=0, out=self.meanlogd2)
+ np.sum([c.meand3 for c in others], axis=0, out=self.meand3)
+ np.sum([c.meanlogd3 for c in others], axis=0, out=self.meanlogd3)
+ np.sum([c.meanu for c in others], axis=0, out=self.meanu)
+ np.sum([c.meanv for c in others], axis=0, out=self.meanv)
+ np.sum([c.weight for c in others], axis=0, out=self.weight)
+ np.sum([c.ntri for c in others], axis=0, out=self.ntri)
+ self.tot = tot
+
+ def _add_tot(self, i, j, k, c1, c2, c3):
+ # When storing results from a patch-based run, tot needs to be accumulated even if
+ # the total weight being accumulated comes out to be zero.
+ # This only applies to NNNCorrelation. For the other ones, this is a no op.
+ tot = c1.sumw * c2.sumw * c3.sumw
+ if c2 is c3:
+ # Account for 1/2 factor in cross12 cases.
+ tot /= 2.
+ self.tot += tot
+ # We also have to keep all pairs in the results dict, otherwise the tot calculation
+ # gets messed up. We need to accumulate the tot value of all pairs, even if
+ # the resulting weight is zero.
+ self.results[(i,j,k)] = self._zero_copy(tot)
+
+[docs] def __iadd__(self, other):
+ """Add a second `NNNCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `NNNCorrelation` objects should not have had `finalize`
+ called yet. Then, after adding them together, you should call `finalize` on the sum.
+ """
+ if not isinstance(other, NNNCorrelation):
+ raise TypeError("Can only add another NNNCorrelation object")
+ if not (self.nbins == other.nbins and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.nubins == other.nubins and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nvbins == other.nvbins and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v):
+ raise ValueError("NNNCorrelation to be added is not compatible with this one.")
+
+ self._set_metric(other.metric, other.coords, other.coords, other.coords)
+ self.tot += other.tot
+
+ # If other is empty, then we're done now.
+ if not other.nonzero:
+ return self
+
+ self.meand1[:] += other.meand1[:]
+ self.meanlogd1[:] += other.meanlogd1[:]
+ self.meand2[:] += other.meand2[:]
+ self.meanlogd2[:] += other.meanlogd2[:]
+ self.meand3[:] += other.meand3[:]
+ self.meanlogd3[:] += other.meanlogd3[:]
+ self.meanu[:] += other.meanu[:]
+ self.meanv[:] += other.meanv[:]
+ self.weight[:] += other.weight[:]
+ self.ntri[:] += other.ntri[:]
+ return self
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2=None, cat3=None, *, metric=None, num_threads=None,
+ comm=None, low_mem=False, initialize=True, finalize=True):
+ """Accumulate the 3pt correlation of the points in the given Catalog(s).
+
+ - If only 1 argument is given, then compute an auto-correlation function.
+ - If 2 arguments are given, then compute a cross-correlation function with the
+ first catalog taking one corner of the triangles, and the second taking two corners.
+ - If 3 arguments are given, then compute a three-way cross-correlation.
+
+ All arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ .. note::
+
+ For a correlation of multiple catalogs, it typically matters which corner of the
+ triangle comes from which catalog, which is not kept track of by this function.
+ The final accumulation will have d1 > d2 > d3 regardless of which input catalog
+ appears at each corner. The class which keeps track of which catalog appears
+ in each position in the triangle is `NNNCrossCorrelation`.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first N field.
+ cat2 (Catalog): A catalog or list of catalogs for the second N field.
+ (default: None)
+ cat3 (Catalog): A catalog or list of catalogs for the third N field.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr3.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ if initialize:
+ self.clear()
+
+ if not isinstance(cat1,list): cat1 = cat1.get_patches()
+ if cat2 is not None and not isinstance(cat2,list): cat2 = cat2.get_patches()
+ if cat3 is not None and not isinstance(cat3,list): cat3 = cat3.get_patches()
+
+ if cat2 is None:
+ if cat3 is not None:
+ raise ValueError("For two catalog case, use cat1,cat2, not cat1,cat3")
+ self._process_all_auto(cat1, metric, num_threads)
+ elif cat3 is None:
+ self._process_all_cross12(cat1, cat2, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, cat3, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ self.finalize()
+
+ def _mean_weight(self):
+ mean_np = np.mean(self.ntri)
+ return 1 if mean_np == 0 else np.mean(self.weight)/mean_np
+
+[docs] def getStat(self):
+ """The standard statistic for the current correlation object as a 1-d array.
+
+ This raises a RuntimeError if calculateZeta has not been run yet.
+ """
+ if self._rrr_weight is None:
+ raise RuntimeError("You need to call calculateZeta before calling estimate_cov.")
+ return self.zeta.ravel()
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ This is the weight array corresponding to `getStat`. In this case, it is the denominator
+ RRR from the calculation done by calculateZeta().
+ """
+ if self._rrr_weight is not None:
+ return self._rrr_weight.ravel()
+ else:
+ return self.tot
+
+[docs] @depr_pos_kwargs
+ def calculateZeta(self, *, rrr, drr=None, rdd=None):
+ r"""Calculate the 3pt function given another 3pt function of random
+ points using the same mask, and possibly cross correlations of the data and random.
+
+ There are two possible formulae that are currently supported.
+
+ 1. The simplest formula to use is :math:`\zeta^\prime = (DDD-RRR)/RRR`.
+ In this case, only rrr needs to be given, the `NNNCorrelation` of a random field.
+ However, note that in this case, the return value is not normally called :math:`\zeta`.
+ Rather, this is an estimator of
+
+ .. math::
+
+ \zeta^\prime(d_1,d_2,d_3) = \zeta(d_1,d_2,d_3) + \xi(d_1) + \xi(d_2) + \xi(d_3)
+
+ where :math:`\xi` is the two-point correlation function for each leg of the triangle.
+ You would typically want to calculate that separately and subtract off the
+ two-point contributions.
+
+ 2. For auto-correlations, a better formula is :math:`\zeta = (DDD-RDD+DRR-RRR)/RRR`.
+ In this case, RDD is the number of triangles where 1 point comes from the randoms
+ and 2 points are from the data. Similarly, DRR has 1 point from the data and 2 from
+ the randoms. These are what are calculated from calling::
+
+ >>> drr.process(data_cat, rand_cat)
+ >>> rdd.process(rand_cat, data_cat)
+
+ .. note::
+
+ One might thing the formula should be :math:`\zeta = (DDD-3RDD+3DRR-RRR)/RRR`
+ by analogy with the 2pt Landy-Szalay formula. However, the way these are
+ calculated, the object we are calling RDD already includes triangles where R
+ is in each of the 3 locations. So it is really more like RDD + DRD + DDR.
+ These are not computed separately. Rather the single computation of ``rdd``
+ described above accumulates all three permutations together. So that one
+ object includes everything for the second term. Likewise ``drr`` has all the
+ permutations that are relevant for the third term.
+
+ - If only rrr is provided, the first formula will be used.
+ - If all of rrr, drr, rdd are provided then the second will be used.
+
+ Parameters:
+ rrr (NNNCorrelation): The auto-correlation of the random field (RRR)
+ drr (NNNCorrelation): DRR if desired. (default: None)
+ rdd (NNNCorrelation): RDD if desired. (default: None)
+
+ Returns:
+ Tuple containing
+
+ - zeta = array of :math:`\zeta(d_1,d_2,d_3)`
+ - varzeta = array of variance estimates of :math:`\zeta(d_1,d_2,d_3)`
+ """
+ # Each random ntri value needs to be rescaled by the ratio of total possible tri.
+ if rrr.tot == 0:
+ raise ValueError("rrr has tot=0.")
+
+ if (rdd is not None) != (drr is not None):
+ raise TypeError("Must provide both rdd and drr (or neither).")
+
+ # rrrf is the factor to scale rrr weights to get something commensurate to the ddd density.
+ rrrf = self.tot / rrr.tot
+
+ # Likewise for the other two potential randoms:
+ if drr is not None:
+ if drr.tot == 0:
+ raise ValueError("drr has tot=0.")
+ drrf = self.tot / drr.tot
+ if rdd is not None:
+ if rdd.tot == 0:
+ raise ValueError("rdd has tot=0.")
+ rddf = self.tot / rdd.tot
+
+ # Calculate zeta based on which randoms are provided.
+ denom = rrr.weight * rrrf
+ if rdd is None:
+ self.zeta = self.weight - denom
+ else:
+ self.zeta = self.weight - rdd.weight * rddf + drr.weight * drrf - denom
+
+ # Divide by DRR in all cases.
+ if np.any(rrr.weight == 0):
+ self.logger.warning("Warning: Some bins for the randoms had no triangles.")
+ denom[rrr.weight==0] = 1. # guard against division by 0.
+ self.zeta /= denom
+
+ # Set up necessary info for estimate_cov
+
+ # First the bits needed for shot noise covariance:
+ dddw = self._mean_weight()
+ rrrw = rrr._mean_weight()
+ if drr is not None:
+ drrw = drr._mean_weight()
+ if rdd is not None:
+ rddw = rdd._mean_weight()
+
+ # Note: The use of varzeta_factor for the shot noise varzeta is even less justified
+ # than in the NN varxi case. This is merely motivated by analogy with the
+ # 2pt version.
+ if rdd is None:
+ varzeta_factor = 1 + rrrf*rrrw/dddw
+ else:
+ varzeta_factor = 1 + drrf*drrw/dddw + rddf*rddw/dddw + rrrf*rrrw/dddw
+ self._var_num = dddw * varzeta_factor**2 # Should this be **3? Hmm...
+ self._rrr_weight = rrr.weight * rrrf
+
+ # Now set up the bits needed for patch-based covariance
+ self._rrr = rrr
+ self._drr = drr
+ self._rdd = rdd
+
+ if len(self.results) > 0:
+ # Check that all use the same patches as ddd
+ if rrr.npatch1 != 1:
+ if rrr.npatch1 != self.npatch1:
+ raise RuntimeError("If using patches, RRR must be run with the same patches "
+ "as DDD")
+ if drr is not None and (len(drr.results) == 0 or drr.npatch1 != self.npatch1
+ or drr.npatch2 not in (self.npatch2, 1)):
+ raise RuntimeError("DRR must be run with the same patches as DDD")
+ if rdd is not None and (len(rdd.results) == 0 or rdd.npatch2 != self.npatch2
+ or rdd.npatch1 not in (self.npatch1, 1)):
+ raise RuntimeError("RDD must be run with the same patches as DDD")
+
+ # If there are any rrr,drr,rdd patch sets that aren't in results, then we need to add
+ # some dummy results to make sure all the right ijk "pair"s are computed when we make
+ # the vectors for the covariance matrix.
+ add_ijk = set()
+ if rrr.npatch1 != 1:
+ for ijk in rrr.results:
+ if ijk not in self.results:
+ add_ijk.add(ijk)
+
+ if drr is not None and drr.npatch2 != 1:
+ for ijk in drr.results:
+ if ijk not in self.results:
+ add_ijk.add(ijk)
+
+ if rdd is not None and rdd.npatch1 != 1:
+ for ijk in rdd.results:
+ if ijk not in self.results:
+ add_ijk.add(ijk)
+
+ if len(add_ijk) > 0:
+ for ijk in add_ijk:
+ self.results[ijk] = self._zero_copy(0)
+ self.__dict__.pop('_ok',None) # If it was already made, it will need to be redone.
+
+ # Now that it's all set up, calculate the covariance and set varzeta to the diagonal.
+ self.cov = self.estimate_cov(self.var_method)
+ self.varzeta = self.cov.diagonal().reshape(self.zeta.shape)
+ return self.zeta, self.varzeta
+
+ def _calculate_xi_from_pairs(self, pairs):
+ # Note: we keep the notation ij and pairs here, even though they are really ijk and
+ # triples.
+ self._sum([self.results[ij] for ij in pairs])
+ self._finalize()
+ if self._rrr is None:
+ return
+ ddd = self.weight
+ if len(self._rrr.results) > 0:
+ # This is the usual case. R has patches just like D.
+ # Calculate rrr and rrrf in the normal way based on the same pairs as used for DDD.
+ pairs1 = [ij for ij in pairs if self._rrr._ok[ij[0],ij[1],ij[2]]]
+ self._rrr._sum([self._rrr.results[ij] for ij in pairs1])
+ ddd_tot = self.tot
+ else:
+ # In this case, R was not run with patches.
+ # We need to scale RRR down by the relative area.
+ # The approximation we'll use is that tot in the auto-correlations is
+ # proportional to area**3.
+ # The sum of tot**(1/3) when i=j=k gives an estimate of the fraction of the total area.
+ area_frac = np.sum([self.results[ij].tot**(1./3.) for ij in pairs
+ if ij[0] == ij[1] == ij[2]])
+ area_frac /= np.sum([cij.tot**(1./3.) for ij,cij in self.results.items()
+ if ij[0] == ij[1] == ij[2]])
+ # First figure out the original total for all DDD that had the same footprint as RRR.
+ ddd_tot = np.sum([self.results[ij].tot for ij in self.results])
+ # The rrrf we want will be a factor of area_frac smaller than the original
+ # ddd_tot/rrr_tot. We can effect this by multiplying the full ddd_tot by area_frac
+ # and use that value normally below. (Also for drrf and rddf.)
+ ddd_tot *= area_frac
+
+ rrr = self._rrr.weight
+ rrrf = ddd_tot / self._rrr.tot
+
+ if self._drr is not None:
+ if self._drr.npatch2 == 1:
+ # If r doesn't have patches, then convert all (i,i,i) pairs to (i,0,0).
+ pairs2 = [(ij[0],0,0) for ij in pairs if ij[0] == ij[1] == ij[2]]
+ else:
+ pairs2 = [ij for ij in pairs if self._drr._ok[ij[0],ij[1],ij[2]]]
+ self._drr._sum([self._drr.results[ij] for ij in pairs2])
+ drr = self._drr.weight
+ drrf = ddd_tot / self._drr.tot
+ if self._rdd is not None:
+ if self._rdd.npatch1 == 1:
+ # If r doesn't have patches, then convert all (i,i,j) pairs to (0,i,j)
+ # and all (i,j,i to (0,j,i).
+ pairs3 = [(0,ij[1],ij[2]) for ij in pairs if ij[0] == ij[1] or ij[0] == ij[2]]
+ else:
+ pairs3 = [ij for ij in pairs if self._rdd._ok[ij[0],ij[1],ij[2]]]
+ self._rdd._sum([self._rdd.results[ij] for ij in pairs3])
+ rdd = self._rdd.weight
+ rddf = ddd_tot / self._rdd.tot
+ denom = rrr * rrrf
+ if self._drr is None:
+ zeta = ddd - denom
+ else:
+ zeta = ddd - rdd * rddf + drr * drrf - denom
+ denom[denom == 0] = 1 # Guard against division by zero.
+ self.zeta = zeta / denom
+ self._rrr_weight = denom
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, rrr=None, drr=None, rdd=None, file_type=None, precision=None,
+ write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ Normally, at least rrr should be provided, but if this is None, then only the
+ basic accumulated number of triangles are output (along with the columns parametrizing
+ the size and shape of the triangles).
+
+ If at least rrr is given, then it will output an estimate of the final 3pt correlation
+ function, :math:`\zeta`. There are two possible formulae that are currently supported.
+
+ 1. The simplest formula to use is :math:`\zeta^\prime = (DDD-RRR)/RRR`.
+ In this case, only rrr needs to be given, the `NNNCorrelation` of a random field.
+ However, note that in this case, the return value is not what is normally called
+ :math:`\zeta`. Rather, this is an estimator of
+
+ .. math::
+ \zeta^\prime(d_1,d_2,d_3) = \zeta(d_1,d_2,d_3) + \xi(d_1) + \xi(d_2) + \xi(d_3)
+
+ where :math:`\xi` is the two-point correlation function for each leg of the triangle.
+ You would typically want to calculate that separately and subtract off the
+ two-point contributions.
+
+ 2. For auto-correlations, a better formula is :math:`\zeta = (DDD-RDD+DRR-RRR)/RRR`.
+ In this case, RDD is the number of triangles where 1 point comes from the randoms
+ and 2 points are from the data. Similarly, DRR has 1 point from the data and 2 from
+ the randoms.
+ For this case, all combinations rrr, drr, and rdd must be provided.
+
+ The output file will include the following columns:
+
+ ========== ================================================================
+ Column Description
+ ========== ================================================================
+ r_nom The nominal center of the bin in r = d2 where d1 > d2 > d3
+ u_nom The nominal center of the bin in u = d3/d2
+ v_nom The nominal center of the bin in v = +-(d1-d2)/d3
+ meand1 The mean value :math:`\langle d1\rangle` of triangles that fell
+ into each bin
+ meanlogd1 The mean value :math:`\langle \log(d1)\rangle` of triangles that
+ fell into each bin
+ meand2 The mean value :math:`\langle d2\rangle` of triangles that fell
+ into each bin
+ meanlogd2 The mean value :math:`\langle \log(d2)\rangle` of triangles that
+ fell into each bin
+ meand3 The mean value :math:`\langle d3\rangle` of triangles that fell
+ into each bin
+ meanlogd3 The mean value :math:`\langle \log(d3)\rangle` of triangles that
+ fell into each bin
+ meanu The mean value :math:`\langle u\rangle` of triangles that fell
+ into each bin
+ meanv The mean value :math:`\langle v\rangle` of triangles that fell
+ into each bin
+ zeta The estimator :math:`\zeta(r,u,v)` (if rrr is given)
+ sigma_zeta The sqrt of the variance estimate of :math:`\zeta`
+ (if rrr is given)
+ DDD The total weight of DDD triangles in each bin
+ RRR The total weight of RRR triangles in each bin (if rrr is given)
+ DRR The total weight of DRR triangles in each bin (if drr is given)
+ RDD The total weight of RDD triangles in each bin (if rdd is given)
+ ntri The number of triangles contributing to each bin
+ ========== ================================================================
+
+ If ``sep_units`` was given at construction, then the distances will all be in these units.
+ Otherwise, they will be in either the same units as x,y,z (for flat or 3d coordinates) or
+ radians (for spherical coordinates).
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ rrr (NNNCorrelation): The auto-correlation of the random field (RRR)
+ drr (NNNCorrelation): DRR if desired. (default: None)
+ rdd (NNNCorrelation): RDD if desired. (default: None)
+ file_type (str): The type of file to write ('ASCII' or 'FITS').
+ (default: determine the type automatically from the extension
+ of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config
+ dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing NNN correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ self._write_rrr = rrr
+ self._write_drr = drr
+ self._write_rdd = rdd
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ self._write(writer, name, write_patch_results, zero_tot=True)
+ self._write_rrr = None
+ self._write_drr = None
+ self._write_rdd = None
+
+ @property
+ def _write_col_names(self):
+ rrr = self._write_rrr
+ drr = self._write_drr
+ rdd = self._write_rdd
+ col_names = [ 'r_nom', 'u_nom', 'v_nom', 'meand1', 'meanlogd1', 'meand2', 'meanlogd2',
+ 'meand3', 'meanlogd3', 'meanu', 'meanv' ]
+ if rrr is None:
+ col_names += [ 'DDD', 'ntri' ]
+ else:
+ col_names += [ 'zeta','sigma_zeta','DDD','RRR' ]
+ if drr is not None:
+ col_names += ['DRR','RDD']
+ col_names += [ 'ntri' ]
+ return col_names
+
+ @property
+ def _write_data(self):
+ data = [ self.rnom, self.u, self.v,
+ self.meand1, self.meanlogd1, self.meand2, self.meanlogd2,
+ self.meand3, self.meanlogd3, self.meanu, self.meanv ]
+ rrr = self._write_rrr
+ drr = self._write_drr
+ rdd = self._write_rdd
+ if rrr is None:
+ if drr is not None or rdd is not None:
+ raise TypeError("rrr must be provided if other combinations are not None")
+ data += [ self.weight, self.ntri ]
+ else:
+ # This will check for other invalid combinations of rrr, drr, etc.
+ zeta, varzeta = self.calculateZeta(rrr=rrr, drr=drr, rdd=rdd)
+
+ data += [ zeta, np.sqrt(varzeta),
+ self.weight, rrr.weight * (self.tot/rrr.tot) ]
+
+ if drr is not None:
+ data += [ drr.weight * (self.tot/drr.tot), rdd.weight * (self.tot/rdd.tot) ]
+ data += [ self.ntri ]
+
+ data = [ col.flatten() for col in data ]
+ return data
+
+ @property
+ def _write_params(self):
+ return { 'tot' : self.tot, 'coords' : self.coords, 'metric' : self.metric,
+ 'sep_units' : self.sep_units, 'bin_type' : self.bin_type }
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `NNNCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading NNN correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ self._read(reader)
+
+ def _read_from_data(self, data, params):
+ s = self.logr.shape
+ if 'R_nom' in data.dtype.names: # pragma: no cover
+ self._ro.rnom = data['R_nom'].reshape(s)
+ else:
+ self._ro.rnom = data['r_nom'].reshape(s)
+ self.meand1 = data['meand1'].reshape(s)
+ self.meanlogd1 = data['meanlogd1'].reshape(s)
+ self.meand2 = data['meand2'].reshape(s)
+ self.meanlogd2 = data['meanlogd2'].reshape(s)
+ self.meand3 = data['meand3'].reshape(s)
+ self.meanlogd3 = data['meanlogd3'].reshape(s)
+ self.meanu = data['meanu'].reshape(s)
+ self.meanv = data['meanv'].reshape(s)
+ self.weight = data['DDD'].reshape(s)
+ self.ntri = data['ntri'].reshape(s)
+ if 'zeta' in data.dtype.names:
+ self.zeta = data['zeta'].reshape(s)
+ self.varzeta = data['sigma_zeta'].reshape(s)**2
+ self.tot = params['tot']
+ self.coords = params['coords'].strip()
+ self.metric = params['metric'].strip()
+ self._ro.sep_units = params['sep_units'].strip()
+ self._ro.bin_type = params['bin_type'].strip()
+ self.npatch1 = params.get('npatch1', 1)
+ self.npatch2 = params.get('npatch2', 1)
+ self.npatch3 = params.get('npatch3', 1)
+
+
+[docs]class NNNCrossCorrelation(BinnedCorr3):
+ r"""This class handles the calculation a 3-point count-count-count cross-correlation
+ function.
+
+ For 3-point cross correlations, it matters which of the two or three fields falls on
+ each corner of the triangle. E.g. is field 1 on the corner opposite d1 (the longest
+ size of the triangle) or is it field 2 (or 3) there? This is in contrast to the 2-point
+ correlation where the symmetry of the situation means that it doesn't matter which point
+ is identified with each field. This makes it significantly more complicated to keep track
+ of all the relevant information for a 3-point cross correlation function.
+
+ The `NNNCorrelation` class holds a single :math:`\zeta` functions describing all
+ possible triangles, parameterized according to their relative side lengths ordered as
+ d1 > d2 > d3.
+
+ For a cross-correlation of two fields: N1 - N1 - N2 (i.e. the N1 field is at two of the
+ corners and N2 is at one corner), then we need three these :math:`\zeta` functions
+ to capture all of the triangles, since the N2 points may be opposite d1 or d2 or d3.
+ For a cross-correlation of three fields: N1 - N2 - N3, we need six sets, to account for
+ all of the possible permutations relative to the triangle sides.
+
+ Therefore, this class holds 6 instances of `NNNCorrelation`, which in turn hold the
+ information about triangles in each of the relevant configurations. We name these:
+
+ Attributes:
+ n1n2n3: Triangles where N1 is opposite d1, N2 is opposite d2, N3 is opposite d3.
+ n1n3n2: Triangles where N1 is opposite d1, N3 is opposite d2, N2 is opposite d3.
+ n2n1n3: Triangles where N2 is opposite d1, N1 is opposite d2, N3 is opposite d3.
+ n2n3n1: Triangles where N2 is opposite d1, N3 is opposite d2, N1 is opposite d3.
+ n3n1n2: Triangles where N3 is opposite d1, N1 is opposite d2, N2 is opposite d3.
+ n3n2n1: Triangles where N3 is opposite d1, N2 is opposite d2, N1 is opposite d3.
+
+ If for instance N2 and N3 are the same field, then e.g. n1n2n3 and n1n3n2 will have
+ the same values.
+
+ Ojects of this class also hold the following attributes, which are identical in each of
+ the above NNNCorrelation instances.
+
+ Attributes:
+ nbins: The number of bins in logr where r = d2
+ bin_size: The size of the bins in logr
+ min_sep: The minimum separation being considered
+ max_sep: The maximum separation being considered
+ nubins: The number of bins in u where u = d3/d2
+ ubin_size: The size of the bins in u
+ min_u: The minimum u being considered
+ max_u: The maximum u being considered
+ nvbins: The number of bins in v where v = +-(d1-d2)/d3
+ vbin_size: The size of the bins in v
+ min_v: The minimum v being considered
+ max_v: The maximum v being considered
+ logr1d: The nominal centers of the nbins bins in log(r).
+ u1d: The nominal centers of the nubins bins in u.
+ v1d: The nominal centers of the nvbins bins in v.
+
+ If ``sep_units`` are given (either in the config dict or as a named kwarg) then the distances
+ will all be in these units.
+
+ .. note::
+
+ If you separate out the steps of the `process` command and use `process_cross` directly,
+ then the units will not be applied to ``meanr`` or ``meanlogr`` until the `finalize`
+ function is called.
+
+ Parameters:
+ config (dict): A configuration dict that can be used to pass in kwargs if desired.
+ This dict is allowed to have addition entries besides those listed
+ in `BinnedCorr3`, which are ignored here. (default: None)
+ logger: If desired, a logger object for logging. (default: None, in which case
+ one will be built according to the config dict's verbose level.)
+
+ Keyword Arguments:
+ **kwargs: See the documentation for `BinnedCorr3` for the list of allowed keyword
+ arguments, which may be passed either directly or in the config dict.
+ """
+[docs] @depr_pos_kwargs
+ def __init__(self, config=None, *, logger=None, **kwargs):
+ """Initialize `NNNCrossCorrelation`. See class doc for details.
+ """
+ BinnedCorr3.__init__(self, config, logger=logger, **kwargs)
+
+ self._ro._d1 = 1 # NData
+ self._ro._d2 = 1 # NData
+ self._ro._d3 = 1 # NData
+
+ self.n1n2n3 = NNNCorrelation(config, logger=logger, **kwargs)
+ self.n1n3n2 = NNNCorrelation(config, logger=logger, **kwargs)
+ self.n2n1n3 = NNNCorrelation(config, logger=logger, **kwargs)
+ self.n2n3n1 = NNNCorrelation(config, logger=logger, **kwargs)
+ self.n3n1n2 = NNNCorrelation(config, logger=logger, **kwargs)
+ self.n3n2n1 = NNNCorrelation(config, logger=logger, **kwargs)
+ self._all = [self.n1n2n3, self.n1n3n2, self.n2n1n3, self.n2n3n1, self.n3n1n2, self.n3n2n1]
+
+ self.tot = 0.
+ self.logger.debug('Finished building NNNCrossCorr')
+
+[docs] def __eq__(self, other):
+ """Return whether two `NNNCrossCorrelation` instances are equal"""
+ return (isinstance(other, NNNCrossCorrelation) and
+ self.nbins == other.nbins and
+ self.bin_size == other.bin_size and
+ self.min_sep == other.min_sep and
+ self.max_sep == other.max_sep and
+ self.sep_units == other.sep_units and
+ self.min_u == other.min_u and
+ self.max_u == other.max_u and
+ self.nubins == other.nubins and
+ self.ubin_size == other.ubin_size and
+ self.min_v == other.min_v and
+ self.max_v == other.max_v and
+ self.nvbins == other.nvbins and
+ self.vbin_size == other.vbin_size and
+ self.coords == other.coords and
+ self.bin_type == other.bin_type and
+ self.bin_slop == other.bin_slop and
+ self.xperiod == other.xperiod and
+ self.yperiod == other.yperiod and
+ self.zperiod == other.zperiod and
+ self.n1n2n3 == other.n1n2n3 and
+ self.n1n3n2 == other.n1n3n2 and
+ self.n2n1n3 == other.n2n1n3 and
+ self.n2n3n1 == other.n2n3n1 and
+ self.n3n1n2 == other.n3n1n2 and
+ self.n3n2n1 == other.n3n2n1)
+
+[docs] def copy(self):
+ """Make a copy"""
+ ret = NNNCrossCorrelation.__new__(NNNCrossCorrelation)
+ for key, item in self.__dict__.items():
+ if isinstance(item, NNNCorrelation):
+ ret.__dict__[key] = item.copy()
+ else:
+ ret.__dict__[key] = item
+ # This needs to be the new list:
+ ret._all = [ret.n1n2n3, ret.n1n3n2, ret.n2n1n3, ret.n2n3n1, ret.n3n1n2, ret.n3n2n1]
+ return ret
+
+ def _zero_copy(self, tot):
+ # A minimal "copy" with zero for the weight array, and the given value for tot.
+ ret = NNNCrossCorrelation.__new__(NNNCrossCorrelation)
+ ret._ro = self._ro
+ ret.n1n2n3 = self.n1n2n3._zero_copy(tot)
+ ret.n1n3n2 = self.n1n3n2._zero_copy(tot)
+ ret.n2n1n3 = self.n2n1n3._zero_copy(tot)
+ ret.n2n3n1 = self.n2n3n1._zero_copy(tot)
+ ret.n3n1n2 = self.n3n1n2._zero_copy(tot)
+ ret.n3n2n1 = self.n3n2n1._zero_copy(tot)
+ ret._all = [ret.n1n2n3, ret.n1n3n2, ret.n2n1n3, ret.n2n3n1, ret.n3n1n2, ret.n3n2n1]
+ ret.tot = tot
+ setattr(ret, '_nonzero', False)
+ return ret
+
+
+
+[docs] @depr_pos_kwargs
+ def process_cross12(self, cat1, cat2, *, metric=None, num_threads=None):
+ """Process two catalogs, accumulating the 3pt cross-correlation, where one of the
+ points in each triangle come from the first catalog, and two come from the second.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ .. note::
+
+ This only adds to the attributes n1n2n3, n2n1n3, n2n3n1, not the ones where
+ 3 comes before 2. When running this via the regular `process` method, it will
+ combine them at the end to make sure n1n2n3 == n1n3n2, etc. for a complete
+ calculation of the 1-2 cross-correlation.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process. (1 point in each triangle will come
+ from this catalog.)
+ cat2 (Catalog): The second catalog to process. (2 points in each triangle will come
+ from this catalog.)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '':
+ self.logger.info('Starting process NNN (1-2) cross-correlations')
+ else:
+ self.logger.info('Starting process NNN (1-2) cross-correlations for cats %s, %s.',
+ cat1.name, cat2.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords)
+ for nnn in self._all:
+ nnn._set_metric(self.metric, self.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ # Note: all 3 correlation objects are the same. Thus, all triangles will be placed
+ # into self.corr, whichever way the three catalogs are permuted for each triangle.
+ _lib.ProcessCross12(self.n1n2n3.corr, self.n2n1n3.corr, self.n2n3n1.corr,
+ f1.data, f2.data, self.output_dots,
+ f1._d, f2._d, self._coords,
+ self._bintype, self._metric)
+ tot = cat1.sumw * cat2.sumw**2 / 2.
+ self.n1n2n3.tot += tot
+ self.n2n1n3.tot += tot
+ self.n2n3n1.tot += tot
+ self.tot += tot
+
+[docs] @depr_pos_kwargs
+ def process_cross(self, cat1, cat2, cat3, *, metric=None, num_threads=None):
+ """Process a set of three catalogs, accumulating the 3pt cross-correlation.
+
+ This accumulates the cross-correlation for the given catalogs. After
+ calling this function as often as desired, the `finalize` command will
+ finish the calculation of meand1, meanlogd1, etc.
+
+ Parameters:
+ cat1 (Catalog): The first catalog to process
+ cat2 (Catalog): The second catalog to process
+ cat3 (Catalog): The third catalog to process
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ """
+ if cat1.name == '' and cat2.name == '' and cat3.name == '':
+ self.logger.info('Starting process NNN cross-correlations')
+ else:
+ self.logger.info('Starting process NNN cross-correlations for cats %s, %s, %s.',
+ cat1.name, cat2.name, cat3.name)
+
+ self._set_metric(metric, cat1.coords, cat2.coords, cat3.coords)
+ for nnn in self._all:
+ nnn._set_metric(self.metric, self.coords)
+ self._set_num_threads(num_threads)
+ min_size, max_size = self._get_minmax_size()
+
+ f1 = cat1.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 1,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f2 = cat2.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 2,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+ f3 = cat3.getNField(min_size=min_size, max_size=max_size,
+ split_method=self.split_method,
+ brute=self.brute is True or self.brute == 3,
+ min_top=self.min_top, max_top=self.max_top,
+ coords=self.coords)
+
+ self.logger.info('Starting %d jobs.',f1.nTopLevelNodes)
+ _lib.ProcessCross3(self.n1n2n3.corr, self.n1n3n2.corr,
+ self.n2n1n3.corr, self.n2n3n1.corr,
+ self.n3n1n2.corr, self.n3n2n1.corr,
+ f1.data, f2.data, f3.data, self.output_dots,
+ f1._d, f2._d, f3._d, self._coords, self._bintype, self._metric)
+ tot = cat1.sumw * cat2.sumw * cat3.sumw
+ for nnn in self._all:
+ nnn.tot += tot
+ self.tot += tot
+
+ def _finalize(self):
+ for nnn in self._all:
+ nnn._finalize()
+
+[docs] def finalize(self):
+ """Finalize the calculation of the correlation function.
+
+ The `process_cross` command accumulate values in each bin, so they can be called
+ multiple times if appropriate. Afterwards, this command finishes the calculation
+ by dividing by the total weight.
+ """
+ for nnn in self._all:
+ nnn.finalize()
+
+ @property
+ def nonzero(self):
+ """Return if there are any values accumulated yet. (i.e. ntri > 0)
+ """
+ return any(nnn.nonzero for nnn in self._all)
+
+ @lazy_property
+ def _nonzero(self):
+ # The lazy version when we can be sure the object isn't going to accumulate any more.
+ return self.nonzero
+
+ def _clear(self):
+ """Clear the data vectors
+ """
+ for nnn in self._all:
+ nnn._clear()
+ self.tot = 0
+
+ def _sum(self, others):
+ # Equivalent to the operation of:
+ # self._clear()
+ # for other in others:
+ # self += other
+ # but no sanity checks and use numpy.sum for faster calculation.
+ self.tot = np.sum([c.tot for c in others])
+ # Empty ones were only needed for tot. Remove them now.
+ others = [c for c in others if c._nonzero]
+ other_all = zip(*[c._all for c in others]) # Transpose list of lists
+ for nnn,o_nnn in zip(self._all, other_all):
+ nnn._sum(o_nnn)
+
+ def _add_tot(self, i, j, k, c1, c2, c3):
+ tot = c1.sumw * c2.sumw * c3.sumw
+ self.tot += tot
+ for c in self._all:
+ c.tot += tot
+ self.results[(i,j,k)] = self._zero_copy(tot)
+
+[docs] def __iadd__(self, other):
+ """Add a second `NNNCrossCorrelation`'s data to this one.
+
+ .. note::
+
+ For this to make sense, both `NNNCrossCorrelation` objects should not have had
+ `finalize` called yet. Then, after adding them together, you should call `finalize`
+ on the sum.
+ """
+ if not isinstance(other, NNNCrossCorrelation):
+ raise TypeError("Can only add another NNNCrossCorrelation object")
+ self.n1n2n3 += other.n1n2n3
+ self.n1n3n2 += other.n1n3n2
+ self.n2n1n3 += other.n2n1n3
+ self.n2n3n1 += other.n2n3n1
+ self.n3n1n2 += other.n3n1n2
+ self.n3n2n1 += other.n3n2n1
+ self.tot += other.tot
+ return self
+
+[docs] def getWeight(self):
+ """The weight array for the current correlation object as a 1-d array.
+
+ For NNNCrossCorrelation, this is always just 1. We don't currently have any ability
+ to automatically handle a random catalog for NNNCrossCorrelations, so we don't know
+ what the correct weight would be for a given patch or set of patches. This value
+ is only used by the sample method of covariance estimation, so this limitation means
+ that sample covariances may be expected to be less accurate than normal when used with
+ NNNCorrelations.
+ """
+ return 1.
+
+[docs] @depr_pos_kwargs
+ def process(self, cat1, cat2, cat3=None, *, metric=None, num_threads=None,
+ comm=None, low_mem=False, initialize=True, finalize=True):
+ """Accumulate the cross-correlation of the points in the given Catalogs: cat1, cat2, cat3.
+
+ - If 2 arguments are given, then compute a cross-correlation function with the
+ first catalog taking one corner of the triangles, and the second taking two corners.
+ - If 3 arguments are given, then compute a three-way cross-correlation function.
+
+ All arguments may be lists, in which case all items in the list are used
+ for that element of the correlation.
+
+ Parameters:
+ cat1 (Catalog): A catalog or list of catalogs for the first N field.
+ cat2 (Catalog): A catalog or list of catalogs for the second N field.
+ cat3 (Catalog): A catalog or list of catalogs for the third N field.
+ (default: None)
+ metric (str): Which metric to use. See `Metrics` for details.
+ (default: 'Euclidean'; this value can also be given in the
+ constructor in the config dict.)
+ num_threads (int): How many OpenMP threads to use during the calculation.
+ (default: use the number of cpu cores; this value can also be given
+ in the constructor in the config dict.)
+ comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
+ processes. If used, the rank=0 process will have the final
+ computation. This only works if using patches. (default: None)
+ low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
+ This only works if using patches. (default: False)
+ initialize (bool): Whether to begin the calculation with a call to
+ `BinnedCorr3.clear`. (default: True)
+ finalize (bool): Whether to complete the calculation with a call to `finalize`.
+ (default: True)
+ """
+ import math
+ if initialize:
+ self.clear()
+ self._process12 = False
+
+ if not isinstance(cat1,list): cat1 = cat1.get_patches()
+ if not isinstance(cat2,list): cat2 = cat2.get_patches()
+ if cat3 is not None and not isinstance(cat3,list): cat3 = cat3.get_patches()
+
+ if cat3 is None:
+ self._process12 = True
+ self._process_all_cross12(cat1, cat2, metric, num_threads, comm, low_mem)
+ else:
+ self._process_all_cross(cat1, cat2, cat3, metric, num_threads, comm, low_mem)
+
+ if finalize:
+ if self._process12:
+ # Then some of the processing involved a cross12 calculation.
+ # This means that spots 2 and 3 should not be distinguished.
+ # Combine the relevant arrays.
+ self.n1n2n3 += self.n1n3n2
+ self.n2n1n3 += self.n3n1n2
+ self.n2n3n1 += self.n3n2n1
+ # Copy back by doing clear and +=.
+ self.n1n3n2.clear()
+ self.n3n1n2.clear()
+ self.n3n2n1.clear()
+ self.n1n3n2 += self.n1n2n3
+ self.n3n1n2 += self.n2n1n3
+ self.n3n2n1 += self.n2n3n1
+
+ self.finalize()
+
+[docs] @depr_pos_kwargs
+ def write(self, file_name, *, file_type=None, precision=None, write_patch_results=False):
+ r"""Write the correlation function to the file, file_name.
+
+ Parameters:
+ file_name (str): The name of the file to write to.
+ file_type (str): The type of file to write ('ASCII' or 'FITS'). (default: determine
+ the type automatically from the extension of file_name.)
+ precision (int): For ASCII output catalogs, the desired precision. (default: 4;
+ this value can also be given in the constructor in the config dict.)
+ write_patch_results (bool): Whether to write the patch-based results as well.
+ (default: False)
+ """
+ self.logger.info('Writing NNN cross-correlations to %s',file_name)
+ precision = self.config.get('precision', 4) if precision is None else precision
+ name = 'main' if write_patch_results else None
+ with make_writer(file_name, precision, file_type, self.logger) as writer:
+ names = [ 'n1n2n3', 'n1n3n2', 'n2n1n3', 'n2n3n1', 'n3n1n2', 'n3n2n1' ]
+ for name, corr in zip(names, self._all):
+ corr._write(writer, name, write_patch_results, zero_tot=True)
+
+[docs] @depr_pos_kwargs
+ def read(self, file_name, *, file_type=None):
+ """Read in values from a file.
+
+ This should be a file that was written by TreeCorr, preferably a FITS file, so there
+ is no loss of information.
+
+ .. warning::
+
+ The `NNNCrossCorrelation` object should be constructed with the same configuration
+ parameters as the one being read. e.g. the same min_sep, max_sep, etc. This is not
+ checked by the read function.
+
+ Parameters:
+ file_name (str): The name of the file to read in.
+ file_type (str): The type of file ('ASCII' or 'FITS'). (default: determine the type
+ automatically from the extension of file_name.)
+ """
+ self.logger.info('Reading NNN cross-correlations from %s',file_name)
+ with make_reader(file_name, file_type, self.logger) as reader:
+ names = [ 'n1n2n3', 'n1n3n2', 'n2n1n3', 'n2n3n1', 'n3n1n2', 'n3n2n1' ]
+ for name, corr in zip(names, self._all):
+ corr._read(reader, name)
+
+# Copyright (c) 2003-2020 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+"""
+This is a set of helper classes that read data for treecorr.Catalog objects.
+They have a bit of a mixed bag of functions, not intended to be complete,
+just to cover the needs of that class.
+
+HDF and FITS files differ in that the former can have different length columns
+in the same data group, unlike fits HDUs, which have columns of all the same length.
+Where possible we check for that here, and a user would have to be fairly determined to
+trigger this.
+
+The other difference is that fitsio reads a structured array, whereas h5py will
+read a dictionary of arrays. This does not cause problems here, since both are
+indexed by string, but may prevent usage elsewhere. If so we could convert them to
+both provide dicts.
+"""
+import numpy as np
+
+[docs]class AsciiReader(object):
+ """Reader interface for ASCII files using numpy.
+ """
+ can_slice = True
+ default_ext = None
+
+ def __init__(self, file_name, delimiter=None, comment_marker='#', logger=None):
+ """
+ Parameters:
+ file_name (str): The file name
+ delimiter (str): What delimiter to use between values. (default: None,
+ which means any whitespace)
+ comment_marker (str): What token indicates a comment line. (default: '#')
+ """
+ self.file_name = file_name
+ self.delimiter = delimiter
+ self.comment_marker = comment_marker
+ self.nrows = None
+ self._file = None
+
+ @property
+ def file(self):
+ if self._file is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._file
+
+ def __contains__(self, ext):
+ """Check if ext is None.
+
+ ASCII files don't have extensions, so the only ext allowed is None.
+
+ Parameters:
+ ext (str): The extension to check
+
+ Returns:
+ Whether ext is None
+ """
+ # None is the only valid "extension" for ASCII files
+ return ext is None
+
+[docs] def check_valid_ext(self, ext):
+ """Check if an extension is valid for reading, and raise ValueError if not.
+
+ None is the only valid extension for ASCII files.
+
+ Parameters:
+ ext (str): The extension to check
+ """
+ if ext is not None:
+ raise ValueError("Invalid ext={} for file {}".format(ext,self.file_name))
+
+
+[docs] def read(self, cols, s=slice(None), ext=None):
+ """Read a slice of a column or list of columns from a specified extension.
+
+ Parameters:
+ cols (str/list): The name(s) of column(s) to read
+ s (slice/array): A slice object or selection of integers to read (default: all)
+ ext (str): The extension (ignored)
+
+ Returns:
+ The data as a dict or single numpy array as appropriate
+ """
+ self.file # Check that we are in a with context, so other things are set up correctly.
+
+ # Figure out how many rows to skip at the start
+ if isinstance(s, slice) and s.start is not None:
+ skiprows = self.comment_rows + s.start
+ else:
+ skiprows = self.comment_rows
+
+ # And how many to read (if we know)
+ # Note: genfromtxt can't handle a skip, so defer that to later.
+ # Also if s is an array, that won't work here either.
+ if isinstance(s, slice) and s.start is not None and s.stop is not None:
+ nrows = s.stop - s.start
+ else:
+ nrows = None
+
+ # And which columns to read
+ geti = lambda col: self.col_names.index(col) if col in self.col_names else int(col)-1
+ if np.isscalar(cols):
+ icols = [geti(cols)]
+ else:
+ icols = [geti(col) for col in cols]
+
+ # Actually read the data
+ data = np.genfromtxt(self.file, comments=self.comment_marker,
+ delimiter=self.delimiter, usecols=icols,
+ skip_header=skiprows, max_rows=nrows)
+ self.file.seek(0)
+
+ # If only one column, then the shape comes in as one-d. Reshape it:
+ if len(icols) == 1:
+ data = data.reshape(len(data),1)
+
+ # If only one row, then the shape comes in as one-d. Reshape it:
+ if len(data.shape) == 1:
+ data = data.reshape(1,len(data))
+
+ # Select the rows we want if start/end wasn't sufficient.
+ if isinstance(s, slice):
+ data = data[::s.step,:]
+ else:
+ data = data[s,:]
+
+ # Return is slightly different if we have multiple columns or not.
+ if np.isscalar(cols):
+ return data[:,0]
+ else:
+ return {col : data[:,i] for i,col in enumerate(cols)}
+
+[docs] def read_params(self, ext=None):
+ """Read the params in the given extension, if any.
+
+ Parameters:
+ ext (str): The extension (ignored -- Ascii always reads the next group)
+
+ Returns:
+ params
+ """
+ header = next(self.file)
+ params = {}
+ if header[1] == '#':
+ assert header[0] == '#'
+ tokens = header[2:].split()
+ if tokens[0][0] != '{':
+ # Then before the dict, we have group_name
+ name1 = tokens[0]
+ if name1 != ext and ext is not None:
+ raise OSError("Mismatch in group names. Expected %s, found %s"%(ext, name1))
+ header = next(self.file)
+ if header[1] == '#':
+ assert header[0] == '#'
+ params = eval(header[2:].strip())
+ header = next(self.file)
+ # In case these changed since the first group. (Which happens for nn.results.)
+ self.col_names = header[1:].split()
+ self.ncols = len(self.col_names)
+ return params
+
+[docs] def read_data(self, ext=None, max_rows=None):
+ """Read all data in the file, and the parameters in the header, if any.
+
+ Parameters:
+ ext (str): The extension (ignored -- Ascii always reads the next group)
+ max_rows (int): The max number of rows to read. (default: None)
+
+ Returns:
+ data
+ """
+ data = np.genfromtxt(self.file, names=self.col_names, max_rows=max_rows)
+ return data
+
+[docs] def row_count(self, col=None, ext=None):
+ """Count the number of rows in the file.
+
+ Parameters:
+ col (str): The column to use (ignored)
+ ext (str): The extension (ignored)
+
+ Returns:
+ The number of rows
+ """
+ if self.nrows is not None:
+ return self.nrows
+
+ # cf. https://stackoverflow.com/a/850962/1332281
+ # On my system with python 3.7, bufcount was the fastest among these solutions.
+ # I also found 256K was the optimal buf size for my system. Probably YMMV, but
+ # micro-optimizing this is not so important. It's never used by treecorr.
+ # Only the FitsReader ever calls row_count other than from the unit tests.
+ lines = 0
+ buf_size = 256 * 1024
+ buf = self.file.read(buf_size)
+ while buf:
+ lines += buf.count('\n')
+ buf = self.file.read(buf_size)
+ self.file.seek(0) # Go back to the beginning
+ self.nrows = lines - self.comment_rows
+ return self.nrows
+
+[docs] def names(self, ext=None):
+ """Return a list of the names of all the columns in an extension
+
+ Parameters:
+ ext (str): The extension (ignored)
+
+ Returns:
+ A list of string column names
+ """
+ self.file # Check that we are in a with context, so other things are set up correctly.
+
+ # Include both int values as strings and any real names we know about.
+ return [str(i+1) for i in range(self.ncols)] + list(self.col_names)
+
+ def __enter__(self):
+ # See how many comment rows there are at the start
+ self.comment_rows = 0
+ with open(self.file_name, 'r') as fid:
+ for line in fid: # pragma: no branch
+ if line.startswith(self.comment_marker): self.comment_rows += 1
+ else: break
+
+ # Do a trivial read of 1 row, just to get basic info about columns
+ self.ncols = None
+ if self.comment_rows >= 1:
+ try:
+ data = np.genfromtxt(self.file_name, comments=self.comment_marker,
+ delimiter=self.delimiter, names=True,
+ skip_header=self.comment_rows-1, max_rows=1)
+ self.col_names = data.dtype.names
+ self.ncols = len(self.col_names)
+ except Exception:
+ pass
+ if self.ncols is None:
+ data = np.genfromtxt(self.file_name, comments=self.comment_marker,
+ delimiter=self.delimiter, max_rows=1)
+ self.col_names = []
+ if len(data.shape) != 1: # pragma: no cover
+ raise OSError('Unable to parse the input catalog as a numpy array')
+ self.ncols = data.shape[0]
+
+ self._file = open(self.file_name, 'r')
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ self._file.close()
+ self._file = None
+
+
+[docs]class PandasReader(AsciiReader):
+ """Reader interface for ASCII files using pandas.
+ """
+ def __init__(self, file_name, delimiter=None, comment_marker='#', logger=None):
+ """
+ Parameters:
+ file_name (str): The file name
+ delimiter (str): What delimiter to use between values. (default: None,
+ which means any whitespace)
+ comment_marker (str): What token indicates a comment line. (default: '#')
+ """
+ try:
+ import pandas
+ except ImportError:
+ if logger:
+ logger.error("Unable to import pandas. Cannot read %s"%file_name)
+ raise
+
+ AsciiReader.__init__(self, file_name, delimiter, comment_marker)
+ # This is how pandas handles whitespace
+ self.sep = r'\s+' if self.delimiter is None else self.delimiter
+
+
+[docs] def read(self, cols, s=slice(None), ext=None):
+ """Read a slice of a column or list of columns from a specified extension.
+
+ Parameters:
+ cols (str/list): The name(s) of column(s) to read
+ s (slice/array): A slice object or selection of integers to read (default: all)
+ ext (str): The extension (ignored)
+
+ Returns:
+ The data as a dict or single numpy array as appropriate
+ """
+ import pandas
+ self.file # Check that we are in a with context, so other things are set up correctly.
+
+ # Figure out how many rows to skip at the start
+ if isinstance(s, slice) and s.start is not None:
+ skiprows = self.comment_rows + s.start
+ else:
+ skiprows = self.comment_rows
+
+ # And how many to read (if we know)
+ # Note: genfromtxt can't handle a skip, so defer that to later.
+ # Also if s is an array, that won't work here either.
+ if isinstance(s, slice) and s.start is not None and s.stop is not None:
+ nrows = s.stop - s.start
+ else:
+ nrows = None
+
+ # Pandas has the ability to skip according to a function, so we can accommodate
+ # arbitrary s (either slice or array of indices):
+ if isinstance(s, slice) and s.step is not None:
+ start = skiprows
+ skiprows = lambda x: x < start or (x-start) % s.step != 0
+ if nrows is not None:
+ nrows = (nrows-1) // s.step + 1
+
+ if not isinstance(s, slice):
+ # Then s is a numpy array of indices
+ start = skiprows
+ ss = set(s) # for efficiency
+ skiprows = lambda x: x-start not in ss
+
+ # And which columns to read
+ geti = lambda col: self.col_names.index(col) if col in self.col_names else int(col)-1
+ if np.isscalar(cols):
+ icols = [geti(cols)]
+ else:
+ icols = [geti(col) for col in cols]
+
+ # Actually read the data
+ df = pandas.read_csv(self.file, comment=self.comment_marker,
+ sep=self.sep, usecols=icols, header=None,
+ skiprows=skiprows, nrows=nrows)
+ self.file.seek(0)
+
+ # Return is slightly different if we have multiple columns or not.
+ if np.isscalar(cols):
+ return df.iloc[:,0].to_numpy()
+ else:
+ return {col : df.loc[:,icols[i]].to_numpy() for i,col in enumerate(cols)}
+
+[docs]class ParquetReader():
+ """Reader interface for Parquet files using pandas.
+ """
+ can_slice = True
+ default_ext = None
+
+ def __init__(self, file_name, delimiter=None, comment_marker='#', logger=None):
+ """
+ Parameters:
+ file_name (str): The file name
+ delimiter (str): What delimiter to use between values. (default: None,
+ which means any whitespace)
+ comment_marker (str): What token indicates a comment line. (default: '#')
+ """
+ try:
+ import pandas
+ except ImportError:
+ if logger:
+ logger.error("Unable to import pandas. Cannot read %s"%file_name)
+ raise
+
+ self.file_name = file_name
+ self._df = None
+
+ @property
+ def df(self):
+ if self._df is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._df
+
+ def __contains__(self, ext):
+ """Check if ext is None.
+
+ Parquet files don't have extensions, so the only ext allowed is None.
+
+ Parameters:
+ ext (str): The extension to check
+
+ Returns:
+ Whether ext is None
+ """
+ # None is the only valid "extension" for Parquet files
+ return ext is None
+
+[docs] def check_valid_ext(self, ext):
+ """Check if an extension is valid for reading, and raise ValueError if not.
+
+ None is the only valid extension for ASCII files.
+
+ Parameters:
+ ext (str): The extension to check
+ """
+ if ext is not None:
+ raise ValueError("Invalid ext={} for file {}".format(ext,self.file_name))
+
+[docs] def read(self, cols, s=slice(None), ext=None):
+ """Read a slice of a column or list of columns from a specified extension.
+
+ Parameters:
+ cols (str/list): The name(s) of column(s) to read
+ s (slice/array): A slice object or selection of integers to read (default: all)
+ ext (str): The extension (ignored)
+
+ Returns:
+ The data as a recarray or simple numpy array as appropriate
+ """
+ if np.isscalar(cols):
+ return self.df[cols][s].to_numpy()
+ else:
+ return self.df[cols][s].to_records()
+
+[docs] def row_count(self, col=None, ext=None):
+ """Count the number of rows in the named extension and column
+
+ Unlike in FitsReader, col is required.
+
+ Parameters:
+ col (str): The column to use (ignored)
+ ext (str): The extension (ignored)
+
+ Returns:
+ The number of rows
+ """
+ return len(self.df)
+
+[docs] def names(self, ext=None):
+ """Return a list of the names of all the columns in an extension
+
+ Parameters:
+ ext (str): The extension to search for columns (ignored)
+
+ Returns:
+ A list of string column names
+ """
+ return self.df.columns
+
+ def __enter__(self):
+ import pandas
+ self._df = pandas.read_parquet(self.file_name)
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ # free the memory in the dataframe at end of "with" statement
+ self._df = None
+
+
+[docs]class FitsReader(object):
+ """Reader interface for FITS files.
+ Uses fitsio to read columns, etc.
+ """
+ default_ext = 1
+
+ def __init__(self, file_name, logger=None):
+ """
+ Parameters:
+ file_name (str): The file name
+ """
+ try:
+ import fitsio
+ except ImportError:
+ if logger:
+ logger.error("Unable to import fitsio. Cannot read %s"%file_name)
+ raise
+
+ self._file = None # Only works inside a with block.
+
+ # record file name to know what to open when entering
+ self.file_name = file_name
+
+ # There is a bug in earlier fitsio versions that prevents slicing
+ self.can_slice = fitsio.__version__ > '1.0.6'
+
+ @property
+ def file(self):
+ if self._file is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._file
+
+ def __contains__(self, ext):
+ """Check if there is an extension with the given name or index in the file.
+
+ This may be either a name or an integer.
+
+ Parameters:
+ ext (str/int): The extension to check for (default: 1)
+
+ Returns:
+ Whether the extension exists
+ """
+ ext = self._update_ext(ext)
+ return ext in self.file
+
+[docs] def check_valid_ext(self, ext):
+ """Check if an extension is valid for reading, and raise ValueError if not.
+
+ The ext must both exist and be a table (not an image)
+
+ Parameters:
+ ext (str/int): The extension to check
+ """
+ import fitsio
+
+ ext = self._update_ext(ext)
+ if ext not in self:
+ raise ValueError("Invalid ext={} for file {} (does not exist)".format(
+ ext, self.file_name))
+
+ if not isinstance(self.file[ext], fitsio.hdu.TableHDU):
+ raise ValueError("Invalid ext={} for file {} (Not a TableHDU)".format(
+ ext, self.file_name))
+
+ def _update_ext(self, ext):
+ # FITS extensions can be indexed by number or
+ # string. Try converting to an integer if the current
+ # value is not found. If not let the error be caught later.
+ if ext is None:
+ ext = 1
+ if ext not in self.file:
+ try:
+ ext = int(ext)
+ except ValueError:
+ pass
+ return ext
+
+[docs] def read(self, cols, s=slice(None), ext=None):
+ """Read a slice of a column or list of columns from a specified extension
+
+ Parameters:
+ cols (str/list): The name(s) of column(s) to read
+ s (slice/array): A slice object or selection of integers to read (default: all)
+ ext (str/int)): The FITS extension to use (default: 1)
+
+ Returns:
+ The data as a recarray or simple numpy array as appropriate
+ """
+ ext = self._update_ext(ext)
+ return self.file[ext][cols][s]
+
+[docs] def read_params(self, ext=None):
+ """Read the params in the given extension, if any.
+
+ Parameters:
+ ext (str/int): The FITS extension to use (default: 1)
+
+ Returns:
+ params
+ """
+ ext = self._update_ext(ext)
+ params = self.file[ext].read_header()
+ return params
+
+[docs] def read_data(self, ext=None, max_rows=None):
+ """Read all data in the file, and the parameters in the header, if any.
+
+ Parameters:
+ ext (str/int): The FITS extension to use (default: 1)
+ max_rows (int): The max number of rows to read. (ignored)
+
+ Returns:
+ data
+ """
+ ext = self._update_ext(ext)
+ data = self.file[ext].read()
+ return data
+
+[docs] def row_count(self, col=None, ext=None):
+ """Count the number of rows in the named extension
+
+ For compatibility with the HDF interface, which can have columns
+ of different lengths, we allow a second argument, col, but it is
+ ignored here.
+
+ Parameters:
+ col (str): The column to use (ignored)
+ ext (str/int): The FITS extension to use (default: 1)
+
+ Returns:
+ The number of rows
+ """
+ ext = self._update_ext(ext)
+ return self.file[ext].get_nrows()
+
+[docs] def names(self, ext=None):
+ """Return a list of the names of all the columns in an extension
+
+ Parameters:
+ ext (str/int): The extension to search for columns (default: 1)
+
+ Returns:
+ A list of string column names
+ """
+ ext = self._update_ext(ext)
+ return self.file[ext].get_colnames()
+
+ def __enter__(self):
+ import fitsio
+ self._file = fitsio.FITS(self.file_name, 'r')
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ # Context manager closer - we just close the file at the end,
+ # regardless of the error
+ self.file.close()
+ self._file = None
+
+
+[docs]class HdfReader(object):
+ """Reader interface for HDF5 files.
+ Uses h5py to read columns, etc.
+ """
+ # h5py can always accept slices as indices
+ can_slice = True
+ default_ext = '/'
+
+ def __init__(self, file_name, logger=None):
+ """
+ Parameters:
+ file_name (str): The file name
+ """
+ try:
+ import h5py
+ except ImportError:
+ if logger:
+ logger.error("Unable to import h5py. Cannot read %s"%file_name)
+ raise
+
+ self._file = None # Only works inside a with block.
+ self.file_name = file_name
+
+ @property
+ def file(self):
+ if self._file is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._file
+
+ def __contains__(self, ext):
+ """Check if there is an extension with the given name in the file.
+
+ Parameters:
+ ext (str): The extension to check for
+
+ Returns:
+ Whether the extension exists
+ """
+ return ext in self.file
+
+ def _group(self, ext):
+ if ext is None:
+ ext = '/'
+ try:
+ return self.file[ext]
+ except KeyError:
+ raise OSError("Group name %s not found in HDF5 file."%(ext))
+
+[docs] def check_valid_ext(self, ext):
+ """Check if an extension is valid for reading, and raise ValueError if not.
+
+ The ext must exist - there is no other requirement for HDF files.
+
+ Parameters:
+ ext (str): The extension to check
+ """
+ if ext not in self:
+ raise ValueError("Invalid ext={} for file {} (does not exist)".format(
+ ext,self.file_name))
+
+[docs] def read(self, cols, s=slice(None), ext=None):
+ """Read a slice of a column or list of columns from a specified extension.
+
+ Slices should always be used when reading HDF files - using a sequence of
+ integers is painfully slow.
+
+ Parameters:
+ cols (str/list): The name(s) of column(s) to read
+ s (slice/array): A slice object or selection of integers to read (default: all)
+ ext (str): The HDF (sub-)group to use (default: '/')
+
+ Returns:
+ The data as a dict or single numpy array as appropriate
+ """
+ g = self._group(ext)
+ if np.isscalar(cols):
+ return g[cols][s]
+ else:
+ return {col : g[col][s] for col in cols}
+
+[docs] def read_params(self, ext=None):
+ """Read the params in the given extension, if any.
+
+ Parameters:
+ ext (str): The HDF (sub-)group to use (default: '/')
+
+ Returns:
+ params
+ """
+ g = self._group(ext)
+ params = dict(g.attrs)
+
+ return params
+
+[docs] def read_data(self, ext=None, max_rows=None):
+ """Read all data in the file, and the parameters in the attributes, if any.
+
+ Parameters:
+ ext (str): The HDF (sub-)group to use (default: '/')
+ max_rows (int): The max number of rows to read. (ignored)
+
+ Returns:
+ data
+ """
+ g = self._group(ext)
+
+ # This does not actually load the column
+ col_vals = list(g.values())
+ col_names = list(g.keys())
+
+ ncol = len(col_names)
+ num_rows = col_vals[0].size
+ dtype=[(name, col.dtype) for (name, col) in zip(col_names, col_vals)]
+ data = np.empty(num_rows, dtype=dtype)
+
+ # Now we actually read everything
+ for (name, col) in zip(col_names, col_vals):
+ data[name] = col[:]
+
+ return data
+
+[docs] def row_count(self, col, ext=None):
+ """Count the number of rows in the named extension and column
+
+ Unlike in FitsReader, col is required.
+
+ Parameters:
+ col (str): The column to use
+ ext (str): The HDF group name to use (default: '/')
+
+ Returns:
+ The number of rows
+ """
+ return self._group(ext)[col].size
+
+[docs] def names(self, ext=None):
+ """Return a list of the names of all the columns in an extension
+
+ Parameters:
+ ext (str): The extension to search for columns (default: '/')
+
+ Returns:
+ A list of string column names
+ """
+ return list(self._group(ext).keys())
+
+ def __enter__(self):
+ import h5py
+ self._file = h5py.File(self.file_name, 'r')
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ # closes file at end of "with" statement
+ self._file.close()
+ self._file = None
+
+# Copyright (c) 2003-2020 by Mike Jarvis
+#
+# TreeCorr is free software: redistribution and use in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions, and the disclaimer given in the accompanying LICENSE
+# file.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions, and the disclaimer given in the documentation
+# and/or other materials provided with the distribution.
+
+import os
+import numpy as np
+
+def ensure_dir(target):
+ d = os.path.dirname(target)
+ if d != '':
+ if not os.path.exists(d):
+ os.makedirs(d)
+
+[docs]class AsciiWriter(object):
+ """Write data to an ASCII (text) file.
+ """
+ def __init__(self, file_name, *, precision=4, logger=None):
+ """
+ Parameters:
+ file_name: The file name
+ precision: The number of digits of precision to output.
+ logger: If desired, a logger object for logging. (default: None)
+ """
+ self.file_name = file_name
+ self.logger = logger
+ self.set_precision(precision)
+ self._file = None
+ ensure_dir(file_name)
+
+ def set_precision(self, precision):
+ self.precision = precision
+ self.width = precision+8
+ self.fmt = '%%%d.%de'%(self.width, self.precision)
+
+ @property
+ def file(self):
+ if self._file is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._file
+
+[docs] def write(self, col_names, columns, *, params=None, ext=None):
+ """Write some columns to an output ASCII file with the given column names.
+
+ Parameters:
+ col_names: A list of columns names for the given columns. These will be written
+ in a header comment line at the top of the output file.
+ columns: A list of numpy arrays with the data to write.
+ params: A dict of extra parameters to write at the top of the output file.
+ ext: Optional ext name for these data. (default: None)
+ """
+ ncol = len(col_names)
+ data = np.empty( (len(columns[0]), ncol) )
+ for i,col in enumerate(columns):
+ data[:,i] = col
+
+ # Note: The first one is 1 shorter to allow space for the initial #.
+ header = ("#" + "{:^%d}"%(self.width-1) +
+ " {:^%d}"%(self.width) * (ncol-1) + "\n").format(*col_names)
+
+ if ext is not None:
+ s = '## %s\n'%ext
+ self.file.write(s.encode())
+ if params is not None:
+ s = '## %r\n'%(params)
+ self.file.write(s.encode())
+ self.file.write(header.encode())
+ np.savetxt(self.file, data, fmt=self.fmt)
+
+ def __enter__(self):
+ self._file = open(self.file_name, 'wb')
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ self._file.close()
+ self._file = None
+
+
+[docs]class FitsWriter(object):
+ """Writer interface for FITS files.
+ """
+ def __init__(self, file_name, *, logger=None):
+ """
+ Parameters:
+ file_name: The file name
+ logger: If desired, a logger object for logging. (default: None)
+ """
+ try:
+ import fitsio # noqa: F401
+ except ImportError:
+ if logger:
+ logger.error("Unable to import fitsio. Cannot write to %s"%file_name)
+ raise
+ self.file_name = file_name
+ self.logger = logger
+ self._file = None
+ ensure_dir(file_name)
+
+ def set_precision(self, precision):
+ pass
+
+ @property
+ def file(self):
+ if self._file is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._file
+
+[docs] def write(self, col_names, columns, *, params=None, ext=None):
+ """Write some columns to an output ASCII file with the given column names.
+
+ If name is not None, then it is used as the name of the extension for these data.
+
+ Parameters:
+ col_names: A list of columns names for the given columns. These will be written
+ in a header comment line at the top of the output file.
+ columns: A list of numpy arrays with the data to write.
+ params: A dict of extra parameters to write at the top of the output file.
+ ext: Optional ext name for these data. (default: None)
+ """
+ data = np.empty(len(columns[0]), dtype=[ (c,'f8') for c in col_names ])
+ for (c, col) in zip(col_names, columns):
+ data[c] = col
+ self.file.write(data, header=params, extname=ext)
+
+ def __enter__(self):
+ import fitsio
+ self._file = fitsio.FITS(self.file_name, 'rw', clobber=True)
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ self.file.close()
+ self._file = None
+
+
+[docs]class HdfWriter(object):
+ """Writer interface for HDF5 files.
+ Uses h5py to read columns, etc.
+ """
+ def __init__(self, file_name, *, logger=None):
+ """
+ Parameters:
+ file_name: The file name
+ logger: If desired, a logger object for logging. (default: None)
+ """
+ try:
+ import h5py # noqa: F401
+ except ImportError:
+ if logger:
+ logger.error("Unable to import h5py. Cannot write to %s"%file_name)
+ raise
+ self.file_name = file_name
+ self.logger = logger
+ self._file = None
+ ensure_dir(file_name)
+
+ def set_precision(self, precision):
+ pass
+
+ @property
+ def file(self):
+ if self._file is None:
+ raise RuntimeError('Illegal operation when not in a "with" context')
+ return self._file
+
+[docs] def write(self, col_names, columns, *, params=None, ext=None):
+ """Write some columns to an output ASCII file with the given column names.
+
+ If name is not None, then it is used as the name of the extension for these data.
+
+ Parameters:
+ col_names: A list of columns names for the given columns. These will be written
+ in a header comment line at the top of the output file.
+ columns: A list of numpy arrays with the data to write.
+ params: A dict of extra parameters to write at the top of the output file.
+ ext: Optional group name for these data. (default: None)
+ """
+ if ext is not None:
+ hdf = self.file.create_group(ext)
+ else:
+ hdf = self.file
+ if params is not None:
+ hdf.attrs.update(params)
+ for (name, col) in zip(col_names, columns):
+ hdf.create_dataset(name, data=col)
+
+ def __enter__(self):
+ import h5py
+ self._file = h5py.File(self.file_name, 'w')
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ # closes file at end of "with" statement
+ self._file.close()
+ self._file = None
+