From 4e0d212627966419520fef9e0a76eeb9ebab89ce Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Fri, 24 May 2024 12:46:14 -0400 Subject: [PATCH 1/9] Update jump.py --- src/stcal/jump/jump.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index f6e0e2586..0665a6c29 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -591,6 +591,12 @@ def flag_large_events( prev_sat = np.bitwise_and(prev_gdq, sat_flag) not_prev_sat = np.logical_not(prev_sat) new_sat = current_sat * not_prev_sat + if group < ngrps - 1: + next_gdq = gdq[integration, group + 1, :, :] + next_sat = np.bitwise_and(next_gdq, sat_flag) + not_current_sat = np.logical_not(current_sat) + next_new_sat = next_sat * not_current_sat + next_sat_ellipses = find_ellipses(next_new_sat, sat_flag, min_sat_area) sat_ellipses = find_ellipses(new_sat, sat_flag, min_sat_area) # find the ellipse parameters for jump regions jump_ellipses = find_ellipses(gdq[integration, group, :, :], jump_flag, min_jump_area) @@ -603,6 +609,7 @@ def flag_large_events( group, jump_ellipses, sat_ellipses, + next_sat_ellipses, low_threshold, high_threshold, min_sat_radius_extend, @@ -807,6 +814,7 @@ def make_snowballs( group, jump_ellipses, sat_ellipses, + next_sat_ellipses, low_threshold, high_threshold, min_sat_radius, @@ -822,29 +830,20 @@ def make_snowballs( snowballs = [] num_groups = gdq.shape[1] for jump in jump_ellipses: - # center of jump should be saturated - jump_center = jump[0] - if ( - # if center of the jump ellipse is not saturated in this group and is saturated in - # the next group add the jump ellipse to the snowball list - group < (num_groups - 1) - and gdq[integration, group + 1, round(jump_center[1]), round(jump_center[0])] == sat_flag - and gdq[integration, group, round(jump_center[1]), round(jump_center[0])] != sat_flag - ) or ( + if near_edge(jump, low_threshold, high_threshold): # if the jump ellipse is near the edge, do not require saturation in the # center of the jump ellipse - near_edge(jump, low_threshold, high_threshold) - ): snowballs.append(jump) else: for sat in sat_ellipses: - # center of saturation is within the enclosing jump rectangle - if ( - point_inside_ellipse(sat[0], jump) - and gdq[integration, group, round(jump_center[1]), round(jump_center[0])] == sat_flag - and jump not in snowballs - ): - snowballs.append(jump) + if ((point_inside_ellipse(sat[0], jump) + and jump not in snowballs)): + snowballs.append(jump) + if group < num_groups - 1: + # Is there saturation inside the jump in the next group? + for next_sat in next_sat_ellipses: + if ((point_inside_ellipse(next_sat[0], jump)) and jump not in snowballs): + snowballs.append(jump) # extend the saturated ellipses that are larger than the min_sat_radius gdq[integration, :, :, :], persist_jumps[integration, :, :] = extend_saturation( gdq[integration, :, :, :], @@ -864,8 +863,9 @@ def make_snowballs( def point_inside_ellipse(point, ellipse): delta_center = np.sqrt((point[0] - ellipse[0][0]) ** 2 + (point[1] - ellipse[0][1]) ** 2) minor_axis = min(ellipse[1][0], ellipse[1][1]) + major_axis = max(ellipse[1][0], ellipse[1][1]) - return delta_center < minor_axis + return delta_center < major_axis def near_edge(jump, low_threshold, high_threshold): From 3136e476049714965ad3ca4869cca367ce737fc1 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Fri, 24 May 2024 14:58:58 -0400 Subject: [PATCH 2/9] fixes --- src/stcal/jump/jump.py | 3 ++- src/stcal/ramp_fitting/ols_fit.py | 2 +- tests/test_jump.py | 13 ++++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index 0665a6c29..d1af3b8fa 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -12,7 +12,7 @@ from . import constants from . import twopoint_difference as twopt - +from astropy.io import fits log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -465,6 +465,7 @@ def detect_jumps( # This is the flag that controls the flagging of snowballs. if expand_large_events: + fits.writeto("ingdq.fits", gdq) gdq, total_snowballs = flag_large_events( gdq, jump_flag, diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index f440ba6d8..2cfd37c81 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -9,7 +9,7 @@ import numpy as np -from .slope_fitter import ols_slope_fitter # c extension +#from .slope_fitter import ols_slope_fitter # c extension from . import ramp_fit_class, utils diff --git a/tests/test_jump.py b/tests/test_jump.py index b4a688e82..170377597 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -542,13 +542,16 @@ def test_inside_ellipes5(): result = point_inside_ellipse(point, ellipse) assert result -@pytest.mark.skip(" used for local testing") +#@pytest.mark.skip(" used for local testing") def test_flag_persist_groups(): # gdq = fits.getdata("persistgdq.fits") - gdq = np.zeros(shape=(2,2,2,2)) + hdul = fits.open("../../../notebooks/garden_snowball_00_dark_current.fits") + hdul.info() + gdq = hdul['GROUPDQ'].data +# gdq = np.zeros(shape=(2,2,2,2)) print(gdq.shape[0]) - gdq = gdq[:, 0:10, :, :] - total_snowballs = flag_large_events( +# gdq = gdq[:, 0:10, :, :] + gdqout, total_snowballs = flag_large_events( gdq, DQFLAGS["JUMP_DET"], DQFLAGS["SATURATED"], @@ -561,7 +564,7 @@ def test_flag_persist_groups(): sat_expand=1.1, mask_persist_grps_next_int=True, persist_grps_flagged=0) - + fits.writeto("gdqout.fits",gdqout,overwrite=True) def test_calc_num_slices(): n_rows = 20 max_available_cores = 10 From ab343bc5d8d2a7a85f45b1fecf437e936fdaef12 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 28 May 2024 11:42:49 -0400 Subject: [PATCH 3/9] updates --- src/stcal/jump/jump.py | 2 +- tests/test_jump.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index d1af3b8fa..50444a7e3 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -465,7 +465,7 @@ def detect_jumps( # This is the flag that controls the flagging of snowballs. if expand_large_events: - fits.writeto("ingdq.fits", gdq) +# fits.writeto("ingdq.fits", gdq) gdq, total_snowballs = flag_large_events( gdq, jump_flag, diff --git a/tests/test_jump.py b/tests/test_jump.py index 170377597..46f6005ec 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -545,11 +545,9 @@ def test_inside_ellipes5(): #@pytest.mark.skip(" used for local testing") def test_flag_persist_groups(): # gdq = fits.getdata("persistgdq.fits") - hdul = fits.open("../../../notebooks/garden_snowball_00_dark_current.fits") - hdul.info() - gdq = hdul['GROUPDQ'].data -# gdq = np.zeros(shape=(2,2,2,2)) - print(gdq.shape[0]) + gdq = fits.getdata("../../../notebooks/ingdq.fits") +# gdq = hdul['GROUPDQ'].data + print(gdq.shape) # gdq = gdq[:, 0:10, :, :] gdqout, total_snowballs = flag_large_events( gdq, @@ -564,7 +562,7 @@ def test_flag_persist_groups(): sat_expand=1.1, mask_persist_grps_next_int=True, persist_grps_flagged=0) - fits.writeto("gdqout.fits",gdqout,overwrite=True) + fits.writeto("../../../notebooks/gdqout.fits", gdqout, overwrite=True) def test_calc_num_slices(): n_rows = 20 max_available_cores = 10 From d70db66cbc01ac5af3b53cb99d83ff080998f734 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 28 May 2024 14:37:13 -0400 Subject: [PATCH 4/9] Update jump.py --- src/stcal/jump/jump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index 50444a7e3..0f2c4659a 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -863,7 +863,7 @@ def make_snowballs( def point_inside_ellipse(point, ellipse): delta_center = np.sqrt((point[0] - ellipse[0][0]) ** 2 + (point[1] - ellipse[0][1]) ** 2) - minor_axis = min(ellipse[1][0], ellipse[1][1]) +# minor_axis = min(ellipse[1][0], ellipse[1][1]) major_axis = max(ellipse[1][0], ellipse[1][1]) return delta_center < major_axis From 6181819ba0ff3ec931568c007456f0079b24868f Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Mon, 3 Jun 2024 10:44:36 -0400 Subject: [PATCH 5/9] Update CHANGES.rst --- CHANGES.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b13b68860..b4973f750 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,11 @@ Changes to API Bug Fixes --------- - +jump +~~~~ +- Flag asymmetrical snowballs that are missed by the current code (JP-3638). The was changed to + not require that the center of the snowball jump ellipse contains a saturated + pixel. [#261] - 1.7.1 (2024-05-21) From e3349cb2024fc1d69647b67b9a11d8b31e65112d Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Mon, 3 Jun 2024 15:17:11 -0400 Subject: [PATCH 6/9] cleanup --- src/stcal/jump/jump.py | 6 ++---- tests/test_jump.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index 0f2c4659a..a102bd5cc 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -837,9 +837,8 @@ def make_snowballs( snowballs.append(jump) else: for sat in sat_ellipses: - if ((point_inside_ellipse(sat[0], jump) - and jump not in snowballs)): - snowballs.append(jump) + if ((point_inside_ellipse(sat[0], jump) and jump not in snowballs)): + snowballs.append(jump) if group < num_groups - 1: # Is there saturation inside the jump in the next group? for next_sat in next_sat_ellipses: @@ -863,7 +862,6 @@ def make_snowballs( def point_inside_ellipse(point, ellipse): delta_center = np.sqrt((point[0] - ellipse[0][0]) ** 2 + (point[1] - ellipse[0][1]) ** 2) -# minor_axis = min(ellipse[1][0], ellipse[1][1]) major_axis = max(ellipse[1][0], ellipse[1][1]) return delta_center < major_axis diff --git a/tests/test_jump.py b/tests/test_jump.py index 46f6005ec..ab7a9dfc5 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -542,7 +542,7 @@ def test_inside_ellipes5(): result = point_inside_ellipse(point, ellipse) assert result -#@pytest.mark.skip(" used for local testing") +@pytest.mark.skip(" used for local testing") def test_flag_persist_groups(): # gdq = fits.getdata("persistgdq.fits") gdq = fits.getdata("../../../notebooks/ingdq.fits") From 8a44a3e3bc3fd883a7f098359c4481f9ef50d92e Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Mon, 3 Jun 2024 16:24:14 -0400 Subject: [PATCH 7/9] Update test_jump.py --- tests/test_jump.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_jump.py b/tests/test_jump.py index ab7a9dfc5..8d6720484 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -526,15 +526,20 @@ def test_inside_ellipse5(): ellipse = ((0, 0), (1, 2), -10) point = (1, 0.6) result = point_inside_ellipse(point, ellipse) - assert not result + assert result def test_inside_ellipse4(): ellipse = ((0, 0), (1, 2), 0) point = (1, 0.5) result = point_inside_ellipse(point, ellipse) - assert not result + assert result +def test_inside_ellipse6(): + ellipse = ((0, 0), (1, 2), 0) + point = (3, 0.5) + result = point_inside_ellipse(point, ellipse) + assert not result def test_inside_ellipes5(): point = (1110.5, 870.5) From ebafdeaba7e88750f8df43467794239b0368e1c5 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 4 Jun 2024 10:58:48 -0400 Subject: [PATCH 8/9] address comments --- src/stcal/jump/jump.py | 3 +-- src/stcal/ramp_fitting/ols_fit.py | 2 +- tests/test_jump.py | 21 --------------------- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index a102bd5cc..4af4b3831 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -12,7 +12,7 @@ from . import constants from . import twopoint_difference as twopt -from astropy.io import fits + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -465,7 +465,6 @@ def detect_jumps( # This is the flag that controls the flagging of snowballs. if expand_large_events: -# fits.writeto("ingdq.fits", gdq) gdq, total_snowballs = flag_large_events( gdq, jump_flag, diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index 2cfd37c81..f440ba6d8 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -9,7 +9,7 @@ import numpy as np -#from .slope_fitter import ols_slope_fitter # c extension +from .slope_fitter import ols_slope_fitter # c extension from . import ramp_fit_class, utils diff --git a/tests/test_jump.py b/tests/test_jump.py index 8d6720484..e0bf2b62e 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -547,27 +547,6 @@ def test_inside_ellipes5(): result = point_inside_ellipse(point, ellipse) assert result -@pytest.mark.skip(" used for local testing") -def test_flag_persist_groups(): -# gdq = fits.getdata("persistgdq.fits") - gdq = fits.getdata("../../../notebooks/ingdq.fits") -# gdq = hdul['GROUPDQ'].data - print(gdq.shape) -# gdq = gdq[:, 0:10, :, :] - gdqout, total_snowballs = flag_large_events( - gdq, - DQFLAGS["JUMP_DET"], - DQFLAGS["SATURATED"], - min_sat_area=1, - min_jump_area=6, - expand_factor=1.9, - edge_size=0, - sat_required_snowball=True, - min_sat_radius_extend=2.5, - sat_expand=1.1, - mask_persist_grps_next_int=True, - persist_grps_flagged=0) - fits.writeto("../../../notebooks/gdqout.fits", gdqout, overwrite=True) def test_calc_num_slices(): n_rows = 20 max_available_cores = 10 From dd386f4e97293925c020edd3c2f46ea75624fa44 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 4 Jun 2024 14:52:47 -0400 Subject: [PATCH 9/9] Update CHANGES.rst --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 94313384e..2b0f5d162 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,8 +15,8 @@ Bug Fixes --------- jump ~~~~ -- Flag asymmetrical snowballs that are missed by the current code (JP-3638). The was changed to - not require that the center of the snowball jump ellipse contains a saturated +- Flag asymmetrical snowballs that are missed by the current code (JP-3638). This was changed to + not require that the center of the snowball jump ellipse is a saturated pixel. [#261] -