Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Estimate cleaning threshold with CatB pedestals if availables #1143

Merged
merged 10 commits into from
Nov 30, 2023
Merged

Estimate cleaning threshold with CatB pedestals if availables #1143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
f8d8e6d
add estimation of the cleaning threshold with CatB pedestals
FrancaCassol Jul 19, 2023
7cd69a9
correct calibration keys
FrancaCassol Jul 21, 2023
55f62d3
correct typo
FrancaCassol Jul 26, 2023
6b8c3b7
correct value to clip
FrancaCassol Jul 28, 2023
08901cc
optimized threshold calculation
FrancaCassol Jul 28, 2023
da57710
change Cat-B keys name
FrancaCassol Nov 3, 2023
dc8d047
Merge branch 'main' into cleaning_with_CatB
rlopezcoto Nov 30, 2023
a760115
lowercase c
rlopezcoto Nov 30, 2023
10f8456
lowercase c
rlopezcoto Nov 30, 2023
3512db4
lowercase c
rlopezcoto Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions lstchain/io/io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@
dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal"
dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration"
dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield"
dl1_mon_tel_CatB_ped_key = "/dl1/monitoring/telescope/catB/pedestal"
dl1_mon_tel_CatB_cal_key = "/dl1/monitoring/telescope/catB/calibration"
dl1_mon_tel_CatB_flat_key = "/dl1/monitoring/telescope/catB/flatfield"
dl1_mon_tel_catB_ped_key = "/dl1/monitoring/telescope/catB/pedestal"
dl1_mon_tel_catB_cal_key = "/dl1/monitoring/telescope/catB/calibration"
dl1_mon_tel_catB_flat_key = "/dl1/monitoring/telescope/catB/flatfield"
dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam"
dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam"
dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam"
Expand Down
29 changes: 22 additions & 7 deletions lstchain/scripts/lstchain_dl1ab.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@
from lstchain.io.io import (
dl1_images_lstcam_key,
dl1_params_lstcam_key,
dl1_mon_tel_CatB_cal_key,
dl1_mon_tel_CatB_ped_key,
dl1_mon_tel_CatB_flat_key,
global_metadata,
write_metadata,
dl1_mon_tel_catB_ped_key,
dl1_mon_tel_catB_flat_key,
dl1_mon_tel_catB_cal_key
)
from lstchain.io.lstcontainers import DL1ParametersContainer
from lstchain.reco.disp import disp
Expand Down Expand Up @@ -138,6 +138,7 @@

catB_calib_time = np.array(catB_calib["time_min"])
catB_dc_to_pe = np.array(catB_calib["dc_to_pe"])

catB_pedestal_per_sample = np.array(catB_calib["pedestal_per_sample"])

catB_time_correction = np.array(catB_calib["time_correction"])
Expand Down Expand Up @@ -184,11 +185,18 @@
pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
cleaning_params = get_cleaning_parameters(config, clean_method_name)
pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(f"Fraction of pixel cleaning thresholds above picture thr.:"
log.info(f"Fraction of Cat_A pixel cleaning thresholds above Cat_A picture thr.:"

Check warning on line 188 in lstchain/scripts/lstchain_dl1ab.py

View check run for this annotation

Codecov / codecov/patch

lstchain/scripts/lstchain_dl1ab.py#L188 

Added line #L188 was not covered by tests
f"{np.sum(pedestal_thresh > pic_th) / len(pedestal_thresh):.3f}")
picture_th = np.clip(pedestal_thresh, pic_th, None)
log.info(f"Tailcut clean with pedestal threshold config used:"
f"{config['tailcuts_clean_with_pedestal_threshold']}")

if args.catB_calibration_file is not None:
catB_pedestal_mean = np.array(catB_pedestal["charge_mean"])
catB_pedestal_std= np.array(catB_pedestal["charge_std"])
catB_threshold_clean_pe = catB_pedestal_mean + sigma * catB_pedestal_std

Check warning on line 197 in lstchain/scripts/lstchain_dl1ab.py

View check run for this annotation

Codecov / codecov/patch

lstchain/scripts/lstchain_dl1ab.py#L194-L197 

Added lines #L194 - L197 were not covered by tests
Comment on lines +194 to +197
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe these lines could be somehow integrated inside get_threshold_from_dl1_file, which I think does something similar.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that they could come in a function, but I think it would be better to have a different one because there will be cases in which you may want tio extract catB parameters instead of the regular ones, right?
In any case, I leave @FrancaCassol to decide if it is a good idea to put a pin on this for the future



else:
clean_method_name = 'tailcut'
cleaning_params = get_cleaning_parameters(config, clean_method_name)
Expand Down Expand Up @@ -344,6 +352,13 @@
image_table['image'][ii] = image
image_table['peak_time'][ii] = peak_time

# use CatB pedestals to estimate the picture threshold
# as defined in the config file
if args.pedestal_cleaning:
threshold_clean_pe = catB_threshold_clean_pe[calib_idx][selected_gain, pixel_index]
threshold_clean_pe[unusable_pixels] = pic_th
FrancaCassol marked this conversation as resolved.
Show resolved Hide resolved
picture_th = np.clip(threshold_clean_pe, pic_th, None)

Check warning on line 360 in lstchain/scripts/lstchain_dl1ab.py

View check run for this annotation

Codecov / codecov/patch

lstchain/scripts/lstchain_dl1ab.py#L357-L360 

Added lines #L357 - L360 were not covered by tests

if increase_nsb:
# Add noise in pixels, to adjust MC to data noise levels.
# TO BE DONE: in case of "pedestal cleaning" (not used now
Expand Down Expand Up @@ -454,9 +469,9 @@

# write a cat-B calibrations in DL1b
if catB_calib:
write_table(catB_calib, outfile, dl1_mon_tel_CatB_cal_key)
write_table(catB_pedestal, outfile, dl1_mon_tel_CatB_ped_key)
write_table(catB_flatfield, outfile, dl1_mon_tel_CatB_flat_key)
write_table(catB_calib, outfile, dl1_mon_tel_catB_cal_key)
write_table(catB_pedestal, outfile, dl1_mon_tel_catB_ped_key)
write_table(catB_flatfield, outfile, dl1_mon_tel_catB_flat_key)

Check warning on line 474 in lstchain/scripts/lstchain_dl1ab.py

View check run for this annotation

Codecov / codecov/patch

lstchain/scripts/lstchain_dl1ab.py#L472-L474 

Added lines #L472 - L474 were not covered by tests

write_metadata(metadata, args.output_file)

Expand Down
12 changes: 6 additions & 6 deletions lstchain/scripts/lstchain_merge_hdf5_files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@
dl1_params_tel_mon_ped_key,
dl1_params_tel_mon_cal_key,
dl1_params_tel_mon_flat_key,
dl1_mon_tel_CatB_cal_key,
dl1_mon_tel_CatB_ped_key,
dl1_mon_tel_CatB_flat_key
dl1_mon_tel_catB_cal_key,
dl1_mon_tel_catB_ped_key,
dl1_mon_tel_catB_flat_key
)

default_keys_to_copy = [dl1_params_tel_mon_ped_key,
dl1_params_tel_mon_cal_key,
dl1_params_tel_mon_flat_key,
dl1_mon_tel_CatB_cal_key,
dl1_mon_tel_CatB_ped_key,
dl1_mon_tel_CatB_flat_key
dl1_mon_tel_catB_cal_key,
dl1_mon_tel_catB_ped_key,
dl1_mon_tel_catB_flat_key
]

parser = argparse.ArgumentParser(description='Merge HDF5 files')
Expand Down
Loading