Skip to content

Commit

Permalink
Merge pull request #1192 from cta-observatory/correct_dl1_search_path
Browse files Browse the repository at this point in the history 
Correct the search of interleaved files in the official tree
  • Loading branch information
@moralejo
moralejo authored Mar 21, 2024
2 parents ef4ce6a + 404c8a2 commit 5b80be8
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 35 deletions.
11 changes: 2 additions & 9 deletions lstchain/onsite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,17 +241,10 @@ def find_DL1_subrun(run, sub_run, dl1_dir=DEFAULT_DL1_PATH):

return file_list[0]

def find_interleaved_subruns(run, r0_dir=DEFAULT_R0_PATH, dl1_dir=DEFAULT_DL1_PATH):
def find_interleaved_subruns(run, interleaved_dir):
'''
Find the given subrun of interleaved file in onsite tree (dir [dl1_dir]/interleaved)
Return the list of interleaved files for a given run
'''

# look in R0 to find the date
r0_list = find_r0_subrun(run,0,r0_dir)
date = r0_list.parent.name

# search the files
interleaved_dir = Path(f"{dl1_dir}/{date}/interleaved")

file_list = sorted(interleaved_dir.rglob(f'interleaved_LST-1.Run{run:05d}.*.h5'))

Expand Down
27 changes: 15 additions & 12 deletions lstchain/scripts/onsite/onsite_create_cat_B_calibration_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
CAT_B_PIXEL_DIR,
create_pro_symlink,
find_interleaved_subruns,
find_r0_subrun,
find_systematics_correction_file,
find_calibration_file,
find_filter_wheels,
Expand All @@ -40,18 +41,15 @@
required.add_argument('-r', '--run_number', help="Run number of interleaved data",
type=int, required=True)

version = lstchain.__version__

optional.add_argument('-c', '--catA_calibration_run',
help="Cat-A calibration run to be used. If None, it looks for the calibration run of the date of the interleaved data.",
type=int)
optional.add_argument('-v', '--prod_version', help="Version of the production",
default=f"v{version}")

optional.add_argument('-s', '--statistics', help="Number of events for the flat-field and pedestal statistics",
type=int, default=2500)
optional.add_argument('-b', '--base_dir', help="Root dir for the output directory tree", type=Path,
default=DEFAULT_BASE_PATH)
optional.add_argument('--dl1-dir', help="Root dir for the input DL1 tree. By default, <base_dir>/DL1 will be used",
optional.add_argument('--interleaved-dir', help="Root dir for the input interleaved files. By default, <base_dir>/DL1/date/version/interleaved will be used",
type=Path)
optional.add_argument('--r0-dir', help="Root dir for the input r0 tree. By default, <base_dir>/R0 will be used",
type=Path)
Expand Down Expand Up @@ -88,7 +86,7 @@ def main():
args, remaining_args = parser.parse_known_args()
run = args.run_number
n_subruns = args.n_subruns
prod_id = args.prod_version
prod_id = f"v{lstchain.__version__}"
stat_events = args.statistics

sys_date = args.sys_date
Expand All @@ -98,8 +96,6 @@ def main():
yes = args.yes
pro_symlink = not args.no_pro_symlink
r0_dir = args.r0_dir or args.base_dir / 'R0'
dl1_dir = args.dl1_dir or args.base_dir / 'DL1'


# looks for the filter values in the database if not given
if args.filters is None:
Expand All @@ -118,15 +114,22 @@ def main():

print(f"\n--> Config file {config_file}")

# verify input file
input_files = find_interleaved_subruns(run, r0_dir, dl1_dir)
input_path = input_files[0].parent
# look in R0 to find the date
r0_list = find_r0_subrun(run,0,r0_dir)
date = r0_list.parent.name

# find input path
ver = prod_id.rsplit(".")
input_path = args.interleaved_dir or args.base_dir / 'DL1'/ f"{date}/{ver[0]}.{ver[1]}/interleaved"

# verify input file
input_files = find_interleaved_subruns(run, input_path)

print(f"\n--> Found {len(input_files)} interleaved subruns in {input_path}")
if n_subruns < MAX_SUBRUNS:
print(f"--> Process {n_subruns} subruns")

# verify output dir
date = input_files[0].parents[1].name
calib_dir = args.base_dir / CAT_B_PIXEL_DIR
output_dir = calib_dir / "calibration" / date / prod_id
if not output_dir.exists():
Expand Down
29 changes: 15 additions & 14 deletions lstchain/scripts/onsite/onsite_create_cat_B_calibration_files_with_batch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
DEFAULT_BASE_PATH,
CAT_B_PIXEL_DIR,
find_interleaved_subruns,
find_r0_subrun,
DEFAULT_CONFIG_CAT_B_CALIB,
)

Expand All @@ -35,18 +36,15 @@
type=int, nargs="+")
optional.add_argument('-f', '--filters_list', help="Filter list (same order as run list)",
type=int, nargs="+")
version = lstchain.__version__
optional.add_argument('-v', '--prod_version',
help="Version of the production",
default=f"v{version}")

optional.add_argument('-s', '--statistics',
help="Number of events for the flat-field and pedestal statistics",
type=int,
default=2500)
optional.add_argument('-b', '--base_dir', help="Root dir for the output directory tree", type=Path,
default=DEFAULT_BASE_PATH)
optional.add_argument('--dl1-dir', help="Root dir for the input r tree. By default, <base_dir>/DL1 will be used",
type=Path)
optional.add_argument('--interleaved-dir', help="Root dir for the input interleaved files. By default, <base_dir>/DL1/date/version/interleaved will be used",
type=Path)
optional.add_argument('--r0-dir', help="Root dir for the input r0 tree. By default, <base_dir>/R0 will be used",
type=Path)

Expand Down Expand Up @@ -80,7 +78,7 @@ def main():

filters_list = args.filters_list

prod_id = args.prod_version
prod_id = f"v{lstchain.__version__}"
stat_events = args.statistics
base_dir = args.base_dir

Expand All @@ -90,8 +88,6 @@ def main():
yes = args.yes
queue = args.queue
r0_dir = args.r0_dir or args.base_dir / 'R0'
dl1_dir = args.dl1_dir or Path(args.base_dir) / 'DL1'

calib_dir = base_dir / CAT_B_PIXEL_DIR

if shutil.which('srun') is None:
Expand All @@ -117,16 +113,21 @@ def main():
if filters_list is not None:
filters = filters_list[i]

input_files = find_interleaved_subruns(run, r0_dir, dl1_dir)
# look in R0 to find the date
r0_list = find_r0_subrun(run,0,r0_dir)
date = r0_list.parent.name

# find input path
ver = prod_id.rsplit(".")
input_path = args.interleaved_dir or args.base_dir / 'DL1'/ f"{date}/{ver[0]}.{ver[1]}/interleaved"

input_files = find_interleaved_subruns(run, input_path)

date = input_files[0].parent.name
input_path = input_files[0].parent
print(f"--> Found {len(input_files)} interleaved subruns in {input_path}")
if n_subruns:
print(f"--> Process {n_subruns} subruns")

# verify output dir
date = input_files[0].parents[1].name
calib_dir = args.base_dir / CAT_B_PIXEL_DIR
output_dir = calib_dir / "calibration" / date / prod_id
if not output_dir.exists():
Expand Down Expand Up @@ -158,7 +159,7 @@ def main():
"onsite_create_cat_B_calibration_file",
f"-r {run}",
f"-v {prod_id}",
f"--dl1-dir {dl1_dir}",
f"--interleaved-dir {input_path}",
f"--r0-dir {r0_dir}",
f"-b {base_dir}",
f"-s {stat_events}",
Expand Down

0 comments on commit 5b80be8

Please sign in to comment.