From e179843f9d7f0fa83c12d89019074c8fda744228 Mon Sep 17 00:00:00 2001 From: Martin Wolf Date: Thu, 31 Aug 2023 19:34:03 +0200 Subject: [PATCH 1/4] Reorder eventdata --- skyllh/core/pdf.py | 25 ++++++++++++++----------- skyllh/core/utils/multidimgridpdf.py | 4 ++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/skyllh/core/pdf.py b/skyllh/core/pdf.py index 05242c9fd8..fd30dcb3d3 100644 --- a/skyllh/core/pdf.py +++ b/skyllh/core/pdf.py @@ -1308,7 +1308,7 @@ def get_pd_with_eventdata( parameter names and values of the models. By definition, this PDF does not depend on any parameters. eventdata : instance of numpy ndarray - The (N_values,V)-shaped numpy ndarray holding the V data attributes + The (V,N_values)-shaped numpy ndarray holding the V data attributes for each of the N_values events needed for the evaluation of the PDF. evt_mask : instance of numpy ndarray | None @@ -1341,18 +1341,18 @@ def get_pd_with_eventdata( if isinstance(self._pdf, RegularGridInterpolator): with TaskTimer(tl, 'Get pd from RegularGridInterpolator.'): if evt_mask is None: - pd = self._pdf(eventdata) + pd = self._pdf(eventdata.T) else: - pd = self._pdf(eventdata[evt_mask]) + pd = self._pdf(eventdata.T[evt_mask]) else: with TaskTimer(tl, 'Get pd from photospline fit.'): - V = eventdata.shape[1] + V = eventdata.shape[0] if evt_mask is None: pd = self._pdf.evaluate_simple( - [eventdata[:, i] for i in range(0, V)]) + [eventdata[i] for i in range(0, V)]) else: pd = self._pdf.evaluate_simple( - [eventdata[:, i][evt_mask] for i in range(0, V)]) + [eventdata[i][evt_mask] for i in range(0, V)]) with TaskTimer(tl, 'Normalize MultiDimGridPDF with norm factor.'): norm = self._norm_factor_func( @@ -1375,7 +1375,8 @@ def get_pd_with_eventdata( @staticmethod def create_eventdata_for_sigpdf( tdm, - axes): + axes, + ): """Creates the (N_values,V)-shaped eventdata ndarray necessary for evaluating the signal PDF. @@ -1402,14 +1403,15 @@ def create_eventdata_for_sigpdf( TypeError( f'Unable to determine the type of the data field {name}!') - eventdata = np.array(eventdata_fields).T + eventdata = np.array(eventdata_fields) return eventdata @staticmethod def create_eventdata_for_bkgpdf( tdm, - axes): + axes, + ): """Creates the (N_values,V)-shaped eventdata ndarray necessary for evaluating the background PDF. @@ -1425,7 +1427,7 @@ def create_eventdata_for_bkgpdf( for axis in axes: eventdata_fields.append(tdm.get_data(axis.name)) - eventdata = np.array(eventdata_fields).T + eventdata = np.array(eventdata_fields) return eventdata @@ -1433,7 +1435,8 @@ def get_pd( self, tdm, params_recarray=None, - tl=None): + tl=None, + ): """Calculates the probability density for the given trial events given the specified local parameters. diff --git a/skyllh/core/utils/multidimgridpdf.py b/skyllh/core/utils/multidimgridpdf.py index b6022c26bd..41e67d3bbd 100644 --- a/skyllh/core/utils/multidimgridpdf.py +++ b/skyllh/core/utils/multidimgridpdf.py @@ -40,9 +40,9 @@ def kde_pdf_sig_spatial_norm_factor_func( log10_psi_idx = pdf._axes.get_index_by_name(log10_psi_name) if evt_mask is None: - psi = 10**eventdata[:, log10_psi_idx] + psi = 10**eventdata[log10_psi_idx] else: - psi = 10**eventdata[:, log10_psi_idx][evt_mask] + psi = 10**eventdata[log10_psi_idx][evt_mask] norm = 1. / (2 * np.pi * np.log(10) * psi * np.sin(psi)) From 438a6bd4e07c2dea55baf13e2d23d85ca356d1dc Mon Sep 17 00:00:00 2001 From: Martin Wolf Date: Thu, 31 Aug 2023 20:27:05 +0200 Subject: [PATCH 2/4] fix doc string --- skyllh/core/signalpdf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/skyllh/core/signalpdf.py b/skyllh/core/signalpdf.py index 1920ad8087..13f713bc20 100644 --- a/skyllh/core/signalpdf.py +++ b/skyllh/core/signalpdf.py @@ -655,7 +655,7 @@ def _get_eventdata(self, tdm, tl=None): Returns ------- eventdata : instance of numpy ndarray - The (N_values,V)-shaped eventdata ndarray. + The (V,N_values)-shaped eventdata ndarray. """ if (self._cache_tdm_trial_data_state_id is None) or\ (self._cache_tdm_trial_data_state_id != tdm.trial_data_state_id): @@ -711,7 +711,7 @@ def _evaluate_pdfs( tdm : instance of TrialDataManager The instance of TrialDataManager holding the trial event data. eventdata : instance of numpy ndarray - The (N_values,V)-shaped numpy ndarray holding the event data for + The (V,N_values)-shaped numpy ndarray holding the event data for the PDF evaluation. gridparams_recarray : instance of numpy structured ndarray The numpy structured ndarray of length N_sources with the @@ -976,7 +976,7 @@ def _get_eventdata(self, tdm, tl=None): Returns ------- eventdata : instance of numpy ndarray - The (N_values,V)-shaped eventdata ndarray. + The (V,N_values)-shaped eventdata ndarray. """ if (self._cache_tdm_trial_data_state_id is None) or\ (self._cache_tdm_trial_data_state_id != tdm.trial_data_state_id): From acf082b9e547494c730b4b1558380523f8f2beb0 Mon Sep 17 00:00:00 2001 From: Martin Wolf Date: Thu, 31 Aug 2023 20:28:17 +0200 Subject: [PATCH 3/4] Change eventdata structure --- skyllh/core/interpolate.py | 10 +++++----- skyllh/core/pdf.py | 10 +++++----- skyllh/i3/pdfratio.py | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/skyllh/core/interpolate.py b/skyllh/core/interpolate.py index fe2a391ec6..4e4b3fedf0 100644 --- a/skyllh/core/interpolate.py +++ b/skyllh/core/interpolate.py @@ -46,7 +46,7 @@ def __init__(self, func, param_grid_set, **kwargs): tdm : instance of TrialDataManager The TrialDataManager instance holding the trial event data. eventdata : instance of numpy ndarray - A two-dimensional (N_events,V)-shaped numpy ndarray holding + A two-dimensional (V,N_events)-shaped numpy ndarray holding the event data, where N_events is the number of trial events, and V the dimensionality of the event data. gridparams_recarray : instance of numpy record ndarray @@ -118,7 +118,7 @@ def __call__(self, tdm, eventdata, params_recarray): tdm : instance of TrialDataManager The TrialDataManager instance holding the trial data. eventdata : numpy ndarray - The 2D (N_events,V)-shaped numpy ndarray holding the event data, + The 2D (V,N_events)-shaped numpy ndarray holding the event data, where N_events is the number of trial events, and V the dimensionality of the event data. params_recarray : instance of numpy record ndarray @@ -184,7 +184,7 @@ def __call__(self, tdm, eventdata, params_recarray): tdm : instance of TrialDataManager The TrialDataManager instance holding the trial data. eventdata : instance of numpy.ndarray - The (N_events,V)-shaped numpy ndarray holding the event data, + The (V,N_events)-shaped numpy ndarray holding the event data, where N_events is the number of events, and V the dimensionality of the event data. params_recarray : instance of numpy.ndarray @@ -331,7 +331,7 @@ def __call__(self, tdm, eventdata, params_recarray): tdm : instance of TrialDataManager The TrialDataManager instance holding the trial data. eventdata : instance of numpy ndarray - The (N_events,V)-shaped numpy ndarray holding the event data, + The (V,N_events)-shaped numpy ndarray holding the event data, where N_events is the number of events, and V the dimensionality of the event data. params_recarray : numpy record ndarray @@ -530,7 +530,7 @@ def __call__(self, tdm, eventdata, params_recarray): tdm : instance of TrialDataManager The TrialDataManager instance holding the trial data. eventdata : instance of numpy ndarray - The (N_events,V)-shaped numpy ndarray holding the event data, + The (V,N_events)-shaped numpy ndarray holding the event data, where N_events is the number of events, and V the dimensionality of the event data. params_recarray : numpy record ndarray diff --git a/skyllh/core/pdf.py b/skyllh/core/pdf.py index fd30dcb3d3..7aecb292f8 100644 --- a/skyllh/core/pdf.py +++ b/skyllh/core/pdf.py @@ -1017,7 +1017,7 @@ def __init__( TrialDataManager holding the event data for which to calculate the PDF values, ``params_recarray`` is a numpy structured ndarray holding the local parameter names and values, ``eventdata`` is - is a (N_values,V)-shaped numpy ndarray holding the event data + is a (V,N_values)-shaped numpy ndarray holding the event data necessary for this PDF, and ``evt_mask`` is an optional (N_values,)-shaped numpy ndarray holding the mask for the events, i.e. rows in ``eventdata``, which should be considered. If ``None``, @@ -1129,7 +1129,7 @@ def norm_factor_func(self): where ``pdf`` is this PDF instance, ``tdm`` is an instance of TrialDataManager holding the events for which to calculate the PDF values, ``params_recarray`` is a numpy structured ndarray holding the - local parameter names and values, ``eventdata`` is a (N_values,V)-shaped + local parameter names and values, ``eventdata`` is a (V,N_values)-shaped numpy ndarray holding the event data necessary for this PDF, and ``evt_mask`` is an optional (N_values,)-shaped numpy ndarray holding the mask for the events, i.e. rows in ``eventdata``, which should be @@ -1146,7 +1146,7 @@ def norm_factor_func(self, func): # event. def func(pdf, tdm, params_recarray, eventdata, evt_mask=None): if evt_mask is None: - n_values = eventdata.shape[0] + n_values = eventdata.shape[1] else: n_values = np.count_nonzero(evt_mask) return np.ones((n_values,), dtype=np.float64) @@ -1377,7 +1377,7 @@ def create_eventdata_for_sigpdf( tdm, axes, ): - """Creates the (N_values,V)-shaped eventdata ndarray necessary for + """Creates the (V,N_values)-shaped eventdata ndarray necessary for evaluating the signal PDF. Parameters @@ -1412,7 +1412,7 @@ def create_eventdata_for_bkgpdf( tdm, axes, ): - """Creates the (N_values,V)-shaped eventdata ndarray necessary for + """Creates the (V,N_values)-shaped eventdata ndarray necessary for evaluating the background PDF. Parameters diff --git a/skyllh/i3/pdfratio.py b/skyllh/i3/pdfratio.py index f86024e272..547fcc4d1d 100644 --- a/skyllh/i3/pdfratio.py +++ b/skyllh/i3/pdfratio.py @@ -301,7 +301,7 @@ def _evaluate_splines( The TrialDataManager instance holding the trial data and the event mapping to the sources via the ``src_evt_idx`` property. eventdata : instance of numpy ndarray - The (N_events,V)-shaped numpy ndarray holding the event data, where + The (V,N_events)-shaped numpy ndarray holding the event data, where N_events is the number of events, and V the dimensionality of the event data. gridparams_recarray : instance of numpy structured ndarray @@ -328,7 +328,7 @@ def _evaluate_splines( # We got a single parameter set. We will use it for all sources. spline = self._get_spline_for_param_values(gridparams_recarray[0]) - eventdata = np.take(eventdata, evt_idxs, axis=0) + eventdata = np.take(eventdata, evt_idxs, axis=1) values = spline(eventdata) return values @@ -341,9 +341,9 @@ def _evaluate_splines( # Select the eventdata that belongs to the current source. m = src_idxs == sidx - src_eventdata = np.take(eventdata, evt_idxs[m], axis=0) + src_eventdata = np.take(eventdata, evt_idxs[m], axis=1) - n = src_eventdata.shape[0] + n = src_eventdata.shape[1] sl = slice(v_start, v_start+n) values[sl] = spline(src_eventdata) @@ -402,7 +402,7 @@ def _calculate_ratio_and_grads( """ # Create a 2D event data array holding only the needed event data fields # for the PDF ratio spline evaluation. - eventdata = np.vstack([tdm[fn] for fn in self._data_field_names]).T + eventdata = np.vstack([tdm[fn] for fn in self._data_field_names]) (ratio, grads) = self._interpolmethod( tdm=tdm, From 4189c0e4d8eb16a127623e348a8c43a78437153d Mon Sep 17 00:00:00 2001 From: Martin Wolf Date: Thu, 31 Aug 2023 20:39:08 +0200 Subject: [PATCH 4/4] Adapt to new eventdata structure --- tests/core/test_interpolate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/core/test_interpolate.py b/tests/core/test_interpolate.py index d1f26a7bd2..730befec52 100644 --- a/tests/core/test_interpolate.py +++ b/tests/core/test_interpolate.py @@ -41,7 +41,7 @@ def line(m, p, b): p = gridparams_recarray['p'] - n_selected_events = eventdata.shape[0] + n_selected_events = eventdata.shape[1] values = np.repeat(line(m=2, p=p, b=1), n_selected_events) @@ -69,7 +69,7 @@ def product(p1, p2): p1 = gridparams_recarray['p1'] p2 = gridparams_recarray['p2'] - n_selected_events = eventdata.shape[0] + n_selected_events = eventdata.shape[1] values = np.repeat(product(p1, p2), n_selected_events) @@ -144,7 +144,7 @@ def setUp(self): self.tdm = create_tdm(n_sources=3, n_selected_events=2) self.eventdata = np.zeros( - (self.tdm.n_selected_events, 1), dtype=np.float64) + (1, self.tdm.n_selected_events), dtype=np.float64) def test__call__with_different_source_values(self): """Test for when the interpolation parameters have different values for @@ -249,7 +249,7 @@ def setUp(self): self.tdm = create_tdm(n_sources=3, n_selected_events=2) self.eventdata = np.zeros( - (self.tdm.n_selected_events, 1), dtype=np.float64) + (1, self.tdm.n_selected_events), dtype=np.float64) def test__call__with_different_source_values(self): """Test for when the interpolation parameter has different values for @@ -341,7 +341,7 @@ def setUp(self): self.tdm = create_tdm(n_sources=3, n_selected_events=2) self.eventdata = np.zeros( - (self.tdm.n_selected_events, 1), dtype=np.float64) + (1, self.tdm.n_selected_events), dtype=np.float64) def test__call__with_different_source_values(self): """Test for when the interpolation parameter has different values for