From 4025a275326470e9561a222ebdaedea103ef2fca Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Thu, 23 Feb 2023 09:37:55 +0100 Subject: [PATCH 01/21] adding excpetion for when no drawer temperature is found in DB for camera_monitoring --- src/nectarchain/dqm/camera_monitoring.py | 206 ++++++++++--------- src/nectarchain/dqm/dqm_summary_processor.py | 28 +++ 2 files changed, 138 insertions(+), 96 deletions(-) diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index e82afeb0..835ebe71 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -38,16 +38,20 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): SqlFileName = SqlFilePath + "nectarcam_monitoring_db_" + SqlFileDate + ".sqlite" print("SqlFileName", SqlFileName) + con = sqlite3.connect(SqlFileName) + cursor = con.cursor() + try: + #print(cursor.fetchall()) + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + TempData=cursor.execute('''SELECT * FROM monitoring_drawer_temperatures''') + #print(TempData.description) + self.DrawerTemp = cursor.fetchall() + cursor.close() + + except sqlite3.Error as err: + print("Error Code: ", err) + print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") - con = sqlite3.connect(SqlFileName) - cursor = con.cursor() - # print(cursor.fetchall()) - cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - # TempData = cursor.execute('''SELECT * FROM monitoring_drawer_temperatures''') - # print(TempData.description) - - self.DrawerTemp = cursor.fetchall() - cursor.close() def ProcessEvent(self, evt): trigger_time = evt.trigger.time.value @@ -57,52 +61,55 @@ def ProcessEvent(self, evt): self.event_id.append(trigger_id) def FinishRun(self): - self.event_id = np.array(self.event_id) - self.event_times = np.array(self.event_times) - - self.run_start = self.event_times[self.event_id == np.min(self.event_id)] - 100 - self.run_end = np.max(self.event_times) + 100 - - self.DrawerTemp = np.array(self.DrawerTemp) - self.DrawerTimes = np.array(self.DrawerTemp[:, 3]) - - for i in range(len(self.DrawerTimes)): - self.DrawerTimes[i] = astropytime.Time( - self.DrawerTimes[i], format="iso" - ).unix - - self.DrawerTemp11 = self.DrawerTemp[:, 4][self.DrawerTimes > self.run_start] - self.DrawerTemp21 = self.DrawerTemp[:, 5][self.DrawerTimes > self.run_start] - self.DrawerNum1 = self.DrawerTemp[:, 2][self.DrawerTimes > self.run_start] - - self.DrawerTimes_new = self.DrawerTimes[self.DrawerTimes > self.run_start] - - self.DrawerTemp12 = self.DrawerTemp11[self.DrawerTimes_new < self.run_end] - self.DrawerTemp22 = self.DrawerTemp21[self.DrawerTimes_new < self.run_end] - self.DrawerNum2 = self.DrawerNum1[self.DrawerTimes_new < self.run_end] - - self.DrawerTemp1_mean = [] - self.DrawerTemp2_mean = [] - TotalDrawers = np.max(self.DrawerNum2) - - for i in range(TotalDrawers + 1): - for j in range(7): - self.DrawerTemp1_mean.append( - np.mean(self.DrawerTemp12[self.DrawerNum2 == i]) - ) - self.DrawerTemp2_mean.append( - np.mean(self.DrawerTemp22[self.DrawerNum2 == i]) - ) - self.DrawerTemp1_mean = np.array(self.DrawerTemp1_mean) - self.DrawerTemp2_mean = np.array(self.DrawerTemp2_mean) + try: + + self.event_id = np.array(self.event_id) + self.event_times = np.array(self.event_times) + + self.run_start = self.event_times[self.event_id == np.min(self.event_id)] - 100 + self.run_end = np.max(self.event_times) + 100 + + self.DrawerTemp = np.array(self.DrawerTemp) + self.DrawerTimes = np.array(self.DrawerTemp[:,3]) + + for i in range(len(self.DrawerTimes)): + self.DrawerTimes[i] = astropytime.Time(self.DrawerTimes[i], format = 'iso').unix + + + self.DrawerTemp11 = self.DrawerTemp[:,4][self.DrawerTimes > self.run_start] + self.DrawerTemp21 = self.DrawerTemp[:,5][self.DrawerTimes > self.run_start] + self.DrawerNum1 = self.DrawerTemp[:,2][self.DrawerTimes > self.run_start] + + self.DrawerTimes_new = self.DrawerTimes[self.DrawerTimes > self.run_start] + + self.DrawerTemp12 = self.DrawerTemp11[self.DrawerTimes_new < self.run_end] + self.DrawerTemp22 = self.DrawerTemp21[self.DrawerTimes_new < self.run_end] + self.DrawerNum2 = self.DrawerNum1[self.DrawerTimes_new < self.run_end] + + self.DrawerTemp1_mean = [] + self.DrawerTemp2_mean = [] + TotalDrawers = np.max(self.DrawerNum2) + + for i in range(TotalDrawers+1): + for j in range(7): + self.DrawerTemp1_mean.append(np.mean(self.DrawerTemp12[self.DrawerNum2 == i])) + self.DrawerTemp2_mean.append(np.mean(self.DrawerTemp22[self.DrawerNum2 == i])) + self.DrawerTemp1_mean = np.array(self.DrawerTemp1_mean) + self.DrawerTemp2_mean = np.array(self.DrawerTemp2_mean) + + self.DrawerTemp_mean = (self.DrawerTemp1_mean + self.DrawerTemp2_mean)/2 + except Exception as err: + print("Error Code: ", err) + print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") - self.DrawerTemp_mean = (self.DrawerTemp1_mean + self.DrawerTemp2_mean) / 2 def GetResults(self): self.CameraMonitoring_Results_Dict = {} - self.CameraMonitoring_Results_Dict[ - "CAMERA-TEMPERATURE-AVERAGE" - ] = self.DrawerTemp_mean + try: + self.CameraMonitoring_Results_Dict["CAMERA-TEMPERATURE-AVERAGE"] = self.DrawerTemp_mean + except Exception as err: + print("Error Code: ", err) + print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") return self.CameraMonitoring_Results_Dict @@ -110,50 +117,57 @@ def PlotResults(self, name, FigPath): self.ChargeInt_Figures_Dict = {} self.ChargeInt_Figures_Names_Dict = {} - fig, disp = plt.subplots() - disp = CameraDisplay(self.camera) - disp.image = self.DrawerTemp_mean - disp.cmap = plt.cm.coolwarm - disp.axes.text(1.8, -0.3, "Temperature", fontsize=12, rotation=90) - disp.add_colorbar() - plt.title("Camera temperature average") - full_name = name + "_CameraTemperature_Mean.png" - FullPath = FigPath + full_name - self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE"] = fig - self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE"] = FullPath - - plt.close() - - fig1, disp = plt.subplots() - disp = CameraDisplay(self.camera) - disp.image = self.DrawerTemp1_mean - disp.cmap = plt.cm.coolwarm - disp.axes.text(1.8, -0.3, "Temperature 1", fontsize=12, rotation=90) - disp.add_colorbar() - plt.title("Camera temperature average 1") - full_name = name + "_CameraTemperature_average1.png" - FullPath = FigPath + full_name - self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-1"] = fig1 - self.ChargeInt_Figures_Names_Dict[ - "CAMERA-TEMPERATURE-IMAGE-AVERAGE-1" - ] = FullPath - - plt.close() - - fig2, disp = plt.subplots() - disp = CameraDisplay(self.camera) - disp.image = self.DrawerTemp2_mean - disp.cmap = plt.cm.coolwarm - disp.axes.text(1.8, -0.3, "Temperature 2", fontsize=12, rotation=90) - disp.add_colorbar() - plt.title("Camera temperature average 2") - full_name = name + "_CameraTemperature_average2.png" - FullPath = FigPath + full_name - self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-2"] = fig2 - self.ChargeInt_Figures_Names_Dict[ - "CAMERA-TEMPERATURE-IMAGE-AVERAGE-2" - ] = FullPath - - plt.close() + try: + + fig, disp = plt.subplots() + disp = CameraDisplay(self.camera) + disp.image = self.DrawerTemp_mean + disp.cmap = plt.cm.coolwarm + disp.axes.text(1.8, -0.3, 'Temperature', fontsize=12,rotation=90) + disp.add_colorbar() + plt.title("Camera temperature average") + full_name = name + '_CameraTemperature_Mean.png' + FullPath = FigPath +full_name + self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE"] = fig + self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE"] = FullPath + + plt.close() + + + + fig1, disp = plt.subplots() + disp = CameraDisplay(self.camera) + disp.image = self.DrawerTemp1_mean + disp.cmap = plt.cm.coolwarm + disp.axes.text(1.8, -0.3, 'Temperature 1', fontsize=12,rotation=90) + disp.add_colorbar() + plt.title("Camera temperature average 1") + full_name = name + '_CameraTemperature_average1.png' + FullPath = FigPath +full_name + self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-1"] = fig1 + self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-1"] = FullPath + + plt.close() + + + + fig2, disp = plt.subplots() + disp = CameraDisplay(self.camera) + disp.image = self.DrawerTemp2_mean + disp.cmap = plt.cm.coolwarm + disp.axes.text(1.8, -0.3, 'Temperature 2', fontsize=12,rotation=90) + disp.add_colorbar() + plt.title("Camera temperature average 2") + full_name = name + '_CameraTemperature_average2.png' + FullPath = FigPath +full_name + self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-2"] = fig2 + self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-2"] = FullPath + + plt.close() + + + except Exception as err: + print("Error Code: ", err) + print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") return self.ChargeInt_Figures_Dict, self.ChargeInt_Figures_Names_Dict diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index e12ba2ee..1b7d34cf 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -1,5 +1,33 @@ import pickle +import os +import sys + + +#from ctapipe.io import event_source +import sys + +from matplotlib import pyplot as plt +import numpy as np +from scipy.stats import norm +from traitlets.config.loader import Config +from astropy.io import fits +from astropy.table import Table + +# ctapipe modules +from ctapipe import utils +from ctapipe.visualization import CameraDisplay +#from ctapipe.plotting.camera import CameraPlotter +from ctapipe.image.extractor import * +from ctapipe.io import EventSeeker +from ctapipe.instrument import CameraGeometry + +from ctapipe.io.hdf5tableio import HDF5TableWriter, HDF5TableReader + +from ctapipe.io import EventSource +import ctapipe.instrument.camera.readout + +from astropy import time as astropytime class dqm_summary: def __init__(self): From 23c12ef8de85e227085416e3d0c49b5a8a455474 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Thu, 23 Feb 2023 18:05:40 +0100 Subject: [PATCH 02/21] Outout now is saved in astropy tables in FITS format. Commented plotting. Does not store the mwf for each pixel now, only the average over events over the overall camera --- src/nectarchain/dqm/ReadFitsFile.py | 10 +++++ src/nectarchain/dqm/dqm_summary_processor.py | 36 ++++++++++++++++-- src/nectarchain/dqm/mean_waveforms.py | 40 ++++++++------------ src/nectarchain/dqm/start_calib.py | 8 ++-- 4 files changed, 61 insertions(+), 33 deletions(-) create mode 100644 src/nectarchain/dqm/ReadFitsFile.py diff --git a/src/nectarchain/dqm/ReadFitsFile.py b/src/nectarchain/dqm/ReadFitsFile.py new file mode 100644 index 00000000..8a506acd --- /dev/null +++ b/src/nectarchain/dqm/ReadFitsFile.py @@ -0,0 +1,10 @@ +from astropy.io import fits + +# Open the fits file +with fits.open('NectarCAM_Run2720_Results.fits') as hdulist: + # Get the first table by name + # Print the tables + hdulist.info() + + for i in range(1,4): + print(hdulist[i].columns.names) \ No newline at end of file diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index 1b7d34cf..bc9da6a7 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -58,8 +58,36 @@ def PlotResults( ): print("Processor 5") - def WriteAllResults(self, path, DICT): - PickleName = path + "_Results.pickle" - with open(PickleName, "wb") as handle: - pickle.dump(DICT, handle, protocol=pickle.HIGHEST_PROTOCOL) + + def WriteAllResults(self,path, DICT): + data2 = Table() + data1 = Table() + data = Table() + hdulist = fits.HDUList() + for i, j in DICT.items(): + if i == "Results_TriggerStatistics": + for n2, m2 in j.items(): + data2[n2] = m2 + hdu2 = fits.BinTableHDU(data2) + hdu2.name = "Trigger" + + + elif ((i == "Results_MeanWaveForms_HighGain") or (i == "Results_MeanWaveForms_LowGain")): + for n1, m1 in j.items(): + data1[n1] = m1 + hdu1 = fits.BinTableHDU(data1) + hdu1.name = "MWF" + + + else: + for n, m in j.items(): + data[n] = m + hdu = fits.BinTableHDU(data) + hdu.name = "Camera" + + hdulist.append(hdu2) + hdulist.append(hdu1) + hdulist.append(hdu) + FileName = path + '_Results.fits' + hdulist.writeto(FileName, overwrite=True) return None diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py index 02cc4ac4..ad660b3a 100644 --- a/src/nectarchain/dqm/mean_waveforms.py +++ b/src/nectarchain/dqm/mean_waveforms.py @@ -67,34 +67,24 @@ def GetResults(self): # INITIATE DICT self.MeanWaveForms_Results_Dict = {} - # ASSIGN RESUTLS TO DICT - if self.k == 0: - self.MeanWaveForms_Results_Dict[ - "WF-PHY-AVERAGE-HIGH-GAIN" - ] = self.Mwf_average - self.MeanWaveForms_Results_Dict[ - "WF-PHY-AVERAGE-CHAN-HIGH-GAIN" - ] = self.Mwf_Mean_overChan + #ASSIGN RESUTLS TO DICT + if (self.k==0): + #self.MeanWaveForms_Results_Dict["WF-PHY-AVERAGE-HIGH-GAIN"] = self.Mwf_average + self.MeanWaveForms_Results_Dict["WF-PHY-AVERAGE-CHAN-HIGH-GAIN"] = self.Mwf_Mean_overChan if self.counter_ped > 0: - self.MeanWaveForms_Results_Dict[ - "WF-PED-AVERAGE-HIGH-GAIN" - ] = self.Mwf_ped_average - self.MeanWaveForms_Results_Dict[ - "WF-AVERAGE-PED-CHAN-HIGH-GAIN" - ] = self.Mwf_ped_Mean_overChan + #self.MeanWaveForms_Results_Dict["WF-PED-AVERAGE-HIGH-GAIN"] = self.Mwf_ped_average + self.MeanWaveForms_Results_Dict["WF-AVERAGE-PED-CHAN-HIGH-GAIN"] = self.Mwf_ped_Mean_overChan - if self.k == 1: - self.MeanWaveForms_Results_Dict["WF-AVERAGE-LOW-GAIN"] = self.Mwf_average - self.MeanWaveForms_Results_Dict[ - "WF-AVERAGE-CHAN-LOW-GAIN" - ] = self.Mwf_Mean_overChan + + + if (self.k ==1): + #self.MeanWaveForms_Results_Dict["WF-AVERAGE-LOW-GAIN"] = self.Mwf_average + self.MeanWaveForms_Results_Dict["WF-AVERAGE-CHAN-LOW-GAIN"] = self.Mwf_Mean_overChan if self.counter_ped > 0: - self.MeanWaveForms_Results_Dict[ - "WF-PHY-PED-AVERAGE-LOW-GAIN" - ] = self.Mwf_ped_average - self.MeanWaveForms_Results_Dict[ - "WF-PHY-AVERAGE-PED-CHAN-LOW-GAIN" - ] = self.Mwf_ped_Mean_overChan + #self.MeanWaveForms_Results_Dict["WF-PHY-PED-AVERAGE-LOW-GAIN"] = self.Mwf_ped_average + self.MeanWaveForms_Results_Dict["WF-PHY-AVERAGE-PED-CHAN-LOW-GAIN"] = self.Mwf_ped_Mean_overChan + + return self.MeanWaveForms_Results_Dict diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index a740ae8e..f0be9038 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -133,11 +133,10 @@ def CreateFigFolder(name, type): NESTED_DICT[NESTED_DICT_KEYS[dict_num]] = p.GetResults() dict_num += 1 -# in order to allow to change the name easily -name = name -# if we want to write all results in 1 pickle file we do this. -p.WriteAllResults(ResPath, NESTED_DICT) +name = name #in order to allow to change the name easily +p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 pickle file we do this. +''' for p in processors: processor_figure_dict, processor_figure_name_dict = p.PlotResults(name, FigPath) @@ -150,6 +149,7 @@ def CreateFigFolder(name, type): plt.clf() plt.cla() plt.close() +''' end = time.time() print("Processing time:", end - start) From 2e606a3ad25c046b8213c55c36b33f65f3a267a3 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Fri, 24 Feb 2023 13:05:02 +0100 Subject: [PATCH 03/21] Added the plotting as an option in the arguments --- src/nectarchain/dqm/ReadFitsFile.py | 14 +++++- src/nectarchain/dqm/start_calib.py | 68 +++++++++++++++++++---------- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/nectarchain/dqm/ReadFitsFile.py b/src/nectarchain/dqm/ReadFitsFile.py index 8a506acd..f806167f 100644 --- a/src/nectarchain/dqm/ReadFitsFile.py +++ b/src/nectarchain/dqm/ReadFitsFile.py @@ -1,10 +1,20 @@ from astropy.io import fits # Open the fits file -with fits.open('NectarCAM_Run2720_Results.fits') as hdulist: +with fits.open('NectarCAM_Run3645_Results.fits') as hdulist: # Get the first table by name # Print the tables hdulist.info() for i in range(1,4): - print(hdulist[i].columns.names) \ No newline at end of file + print(hdulist[i].columns.names) + + table_data1 = hdulist['Camera'].data + print(table_data1["CHARGE-INTEGRATION-IMAGE-ALL-AVERAGE-HIGH-GAIN"]) + + table_data2 = hdulist['MWF'].data + print(table_data2["WF-PHY-AVERAGE-CHAN-HIGH-GAIN"]) + + + table_data3 = hdulist['Trigger'].data + print(table_data3["TRIGGER-STATISTICS"]) \ No newline at end of file diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index f0be9038..3ee2b824 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -15,10 +15,26 @@ from trigger_statistics import TriggerStatistics from camera_monitoring import CameraMonitoring -print(sys.argv) -path = sys.argv[1] # path of the Run file: ./NectarCAM.Run2720.0000.fits.fz +import argparse + +# Create an ArgumentParser object +parser = argparse.ArgumentParser(description='My script') +parser.add_argument('-plot', action='store_true', help='Whether to plot the data') +parser.add_argument('input_paths', nargs='+', help='Input paths') +args = parser.parse_args() + +#Reading arguments, paths and plot-boolean +path = args.input_paths[0] +print("Input file paths:") +print(path) +for arg in args.input_paths[1:]: + print(arg) +PlotFig = args.plot +print("Plot:", PlotFig) + +#NectarPath +NectarPath = str(os.environ['NECTARDIR']) -NectarPath = str(os.environ["NECTARDIR"]) def GetName(RunFile): @@ -46,7 +62,8 @@ def CreateFigFolder(name, type): # INITIATE path = path -cmap = "gnuplot2" +print(path) +cmap = 'gnuplot2' # Read and seek reader = EventSource(input_url=path) @@ -113,11 +130,14 @@ def CreateFigFolder(name, type): p.ConfigureForRun(path, Chan, Samp, reader1) for i, evt in enumerate(reader): - for p in processors: - p.ProcessEvent(evt) - -for arg in sys.argv[2:]: - reader = EventSource(input_url=arg) + for p in processors: + p.ProcessEvent(evt) + +#for the rest of the event files +for arg in args.input_paths[1:]: + print(arg) + + reader=EventSource(input_url=arg) seeker = EventSeeker(reader) for i, evt in enumerate(reader): @@ -136,20 +156,22 @@ def CreateFigFolder(name, type): name = name #in order to allow to change the name easily p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 pickle file we do this. -''' -for p in processors: - processor_figure_dict, processor_figure_name_dict = p.PlotResults(name, FigPath) - - for fig_plot in processor_figure_dict: - fig = processor_figure_dict[fig_plot] - SavePath = processor_figure_name_dict[fig_plot] - plt.gcf() - fig.savefig(SavePath) - -plt.clf() -plt.cla() -plt.close() -''' + +#if -plot in args it will construct the figures and save them +if PlotFig == True: + for p in processors: + processor_figure_dict, processor_figure_name_dict = p.PlotResults(name, FigPath) + + for fig_plot in processor_figure_dict: + fig = processor_figure_dict[fig_plot] + SavePath = processor_figure_name_dict[fig_plot] + plt.gcf() + fig.savefig(SavePath) + + plt.clf() + plt.cla() + plt.close() + end = time.time() print("Processing time:", end - start) From 4ff24092c8ed52530c9d8a83f70fe6cc8fceb19b Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Fri, 24 Feb 2023 13:57:51 +0100 Subject: [PATCH 04/21] Fixed a bug in charge integration (high and low gains were treated the same). Also added in the processor the handling of the case where not all the processors are chosen (in wrtiting the results FITS file) --- src/nectarchain/dqm/charge_integration.py | 6 +++--- src/nectarchain/dqm/dqm_summary_processor.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 0492a9f6..a8b5de40 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -45,9 +45,9 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.peakpos_ped = [] def ProcessEvent(self, evt): - # print("test", evt.r0.tel[0].waveform[0]) - waveform = evt.r0.tel[0].waveform[0] - image, peakpos = self.integrator(waveform, 0, np.zeros(self.Chan, dtype=int)) + #print("test", evt.r0.tel[0].waveform[0]) + waveform=evt.r0.tel[0].waveform[self.k] + image, peakpos = self.integrator(waveform,0,np.zeros(self.Chan, dtype = int)) if evt.trigger.event_type.value == 32: # count peds self.counter_ped += 1 diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index bc9da6a7..49a211f1 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -84,10 +84,18 @@ def WriteAllResults(self,path, DICT): data[n] = m hdu = fits.BinTableHDU(data) hdu.name = "Camera" - - hdulist.append(hdu2) - hdulist.append(hdu1) - hdulist.append(hdu) + try: + hdulist.append(hdu2) + except: + print("No trigger statistics requests") + try: + hdulist.append(hdu1) + except: + print("No MWF studies requests") + try: + hdulist.append(hdu) + except: + print("No Camera studies requests") FileName = path + '_Results.fits' hdulist.writeto(FileName, overwrite=True) return None From 8d25791ac8aa04cf1e2a6f0c11518c4b9bf04d0c Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Wed, 1 Mar 2023 12:47:22 +0100 Subject: [PATCH 05/21] Added an option to substract first 20 ns as pedestal (only applied for charge integration but could be done for all modules). Improvement of the code. Debugging. --- ...anectarcam_monitoring_db_2022-02-24.sqlite | 0 src/nectarchain/dqm/camera_monitoring.py | 10 +- src/nectarchain/dqm/charge_integration.py | 11 +- src/nectarchain/dqm/dqm_summary_processor.py | 4 +- src/nectarchain/dqm/mean_camera_display.py | 4 +- src/nectarchain/dqm/mean_waveforms.py | 5 +- src/nectarchain/dqm/start_calib.py | 15 +- .../dqm/start_calib_thermal_test.py | 213 ++++++++++++++++++ src/nectarchain/dqm/trigger_statistics.py | 2 +- 9 files changed, 246 insertions(+), 18 deletions(-) create mode 100644 src/nectarchain/dqm/UsershashkarDesktopashkar_nectardatanectarcam_monitoring_db_2022-02-24.sqlite create mode 100644 src/nectarchain/dqm/start_calib_thermal_test.py diff --git a/src/nectarchain/dqm/UsershashkarDesktopashkar_nectardatanectarcam_monitoring_db_2022-02-24.sqlite b/src/nectarchain/dqm/UsershashkarDesktopashkar_nectardatanectarcam_monitoring_db_2022-02-24.sqlite new file mode 100644 index 00000000..e69de29b diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index 835ebe71..40d39201 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -50,10 +50,10 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): except sqlite3.Error as err: print("Error Code: ", err) - print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") + print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") - def ProcessEvent(self, evt): + def ProcessEvent(self, evt, noped): trigger_time = evt.trigger.time.value trigger_id = evt.index.event_id @@ -100,7 +100,7 @@ def FinishRun(self): self.DrawerTemp_mean = (self.DrawerTemp1_mean + self.DrawerTemp2_mean)/2 except Exception as err: print("Error Code: ", err) - print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") + print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") def GetResults(self): @@ -109,7 +109,7 @@ def GetResults(self): self.CameraMonitoring_Results_Dict["CAMERA-TEMPERATURE-AVERAGE"] = self.DrawerTemp_mean except Exception as err: print("Error Code: ", err) - print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") + print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") return self.CameraMonitoring_Results_Dict @@ -168,6 +168,6 @@ def PlotResults(self, name, FigPath): except Exception as err: print("Error Code: ", err) - print("DRAWER TEMPERATURE COULD NOT BE RETREIVED!") + print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") return self.ChargeInt_Figures_Dict, self.ChargeInt_Figures_Names_Dict diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index a8b5de40..383dcfe7 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -44,10 +44,17 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.image_ped = [] self.peakpos_ped = [] - def ProcessEvent(self, evt): + + def ProcessEvent(self, evt, noped): #print("test", evt.r0.tel[0].waveform[0]) waveform=evt.r0.tel[0].waveform[self.k] - image, peakpos = self.integrator(waveform,0,np.zeros(self.Chan, dtype = int)) + if noped == True: + ped = np.mean(waveform[:, 20]) + w_noped = waveform - ped + image, peakpos = self.integrator(w_noped,0,np.zeros(self.Chan, dtype = int)) + + else: + image, peakpos = self.integrator(waveform,0,np.zeros(self.Chan, dtype = int)) if evt.trigger.event_type.value == 32: # count peds self.counter_ped += 1 diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index 49a211f1..f60adf42 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -44,8 +44,8 @@ def DefineForRun(self, reader1): def ConfigureForRun(self): print("Processor 1") - def ProcessEvent(self, evt): - print("Processor 2") + def ProcessEvent(self, evt, noped): + print('Processor 2') def FinishRun(self, M, M_ped, counter_evt, counter_ped): print("Processor 3") diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index 6229dfa3..de915188 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -26,8 +26,8 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.cmap = "gnuplot2" self.cmap2 = "gnuplot2" - def ProcessEvent(self, evt): - if evt.trigger.event_type.value == 32: # count peds + def ProcessEvent(self, evt, noped): + if evt.trigger.event_type.value == 32: #count peds self.counter_ped += 1 else: self.counter_evt += 1 diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py index ad660b3a..95a1bec4 100644 --- a/src/nectarchain/dqm/mean_waveforms.py +++ b/src/nectarchain/dqm/mean_waveforms.py @@ -28,8 +28,9 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): return None - def ProcessEvent(self, evt): - if evt.trigger.event_type.value == 32: # count peds + + def ProcessEvent(self, evt, noped): + if evt.trigger.event_type.value == 32: #count peds self.counter_ped += 1 else: self.counter_evt += 1 diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index 3ee2b824..0e000f74 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -19,7 +19,12 @@ # Create an ArgumentParser object parser = argparse.ArgumentParser(description='My script') -parser.add_argument('-plot', action='store_true', help='Whether to plot the data') +parser.add_argument('-p', '--plot', + action='store_true', + help='Enables plots to be generated') +parser.add_argument('-n', '--noped', + action='store_true', + help='Enables plots to be generated') parser.add_argument('input_paths', nargs='+', help='Input paths') args = parser.parse_args() @@ -30,7 +35,9 @@ for arg in args.input_paths[1:]: print(arg) PlotFig = args.plot +noped = args.noped print("Plot:", PlotFig) +print("Noped:", noped) #NectarPath NectarPath = str(os.environ['NECTARDIR']) @@ -131,7 +138,7 @@ def CreateFigFolder(name, type): for i, evt in enumerate(reader): for p in processors: - p.ProcessEvent(evt) + p.ProcessEvent(evt, noped) #for the rest of the event files for arg in args.input_paths[1:]: @@ -142,7 +149,7 @@ def CreateFigFolder(name, type): for i, evt in enumerate(reader): for p in processors: - p.ProcessEvent(evt) + p.ProcessEvent(evt, noped) for p in processors: p.FinishRun() @@ -155,7 +162,7 @@ def CreateFigFolder(name, type): name = name #in order to allow to change the name easily -p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 pickle file we do this. +p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 fits file we do this. #if -plot in args it will construct the figures and save them if PlotFig == True: diff --git a/src/nectarchain/dqm/start_calib_thermal_test.py b/src/nectarchain/dqm/start_calib_thermal_test.py new file mode 100644 index 00000000..5648fd43 --- /dev/null +++ b/src/nectarchain/dqm/start_calib_thermal_test.py @@ -0,0 +1,213 @@ +import os +import sys + +from matplotlib import pyplot as plt + +#from multiprocessing import Process + +import time + +from ctapipe.io import EventSource, EventSeeker + + +from mean_waveforms import MeanWaveForms_HighLowGain +from mean_camera_display import MeanCameraDisplay_HighLowGain +from charge_integration import ChargeIntegration_HighLowGain +from trigger_statistics import TriggerStatistics +from camera_monitoring import CameraMonitoring + +import argparse + +# Create an ArgumentParser object +parser = argparse.ArgumentParser(description='My script') +parser.add_argument('-p', '--plot', + action='store_true', + help='Enables plots to be generated') +parser.add_argument('-n', '--noped', + action='store_true', + help='Enables plots to be generated') +parser.add_argument('input_paths', nargs='+', help='Input paths') +args = parser.parse_args() + +#Reading arguments, paths and plot-boolean +path = args.input_paths[0] +print("Input file paths:") +print(path) +for arg in args.input_paths[1:]: + print(arg) +PlotFig = args.plot +noped = args.noped +print("Plot:", PlotFig) + +#NectarPath +NectarPath = str(os.environ['NECTARDIR']) + + + +def GetName(RunFile): + name = RunFile.split('/')[-1] + name = name.split('.')[0] + '_' + name.split('.')[1]# + '_' +name.split('.')[2] + print(name) + return name + +def CreateFigFolder(name, type): + if(type == 0): + folder = 'Plots' + + ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] + ChildrenFolderName = './' + ParentFolderName +'/' + name + '_calib' + FolderPath = NectarPath + 'output/%s/%s/' %(ChildrenFolderName, folder) + + if not os.path.exists(FolderPath): + os.makedirs(FolderPath) + + return ParentFolderName, ChildrenFolderName, FolderPath + + +start = time.time() + +#INITIATE +####################################################################################################################### +path = path +print(path) +cmap = 'gnuplot2' + +#Read and seek +reader=EventSource(input_url=path) +seeker = EventSeeker(reader) +reader1 = EventSource(input_url=path, max_events=1) +#print(reader.file_list) + +name = GetName(path) +ParentFolderName, ChildrenFolderName, FigPath = CreateFigFolder(name, 0) +ResPath = NectarPath + 'output/%s/%s' %(ChildrenFolderName, name) +####################################################################################################################### + + + + + ######################## + + + + + + + +#LIST OF PROCESSES TO RUN +####################################################################################################################### +a = TriggerStatistics(0) +b = MeanWaveForms_HighLowGain(0) #0 is for high gain and 1 is for low gain +c = MeanWaveForms_HighLowGain(1) +d = MeanCameraDisplay_HighLowGain(0) +e = MeanCameraDisplay_HighLowGain(1) +f = ChargeIntegration_HighLowGain(0) +g = ChargeIntegration_HighLowGain(1) +h = CameraMonitoring(0) + +processors = list() + +processors.append(a) +processors.append(b) +processors.append(c) +processors.append(d) +processors.append(e) +processors.append(f) +processors.append(g) +processors.append(h) +####################################################################################################################### + + + + + + ######################## + + + + +#LIST OF DICT RESULTS +####################################################################################################################### +Results_MeanWaveForms_HighGain ={} +Results_MeanWaveForms_LowGain = {} +Results_MeanCameraDisplay_HighGain = {} +Results_MeanCameraDisplay_LowGain = {} +Results_ChargeIntegration_HighGain = {} +Results_ChargeIntegration_LowGain = {} +Results_TriggerStatistics = {} +Results_CameraMonitoring = {} + +NESTED_DICT = {} #The final results dictionary +NESTED_DICT_KEYS = ["Results_TriggerStatistics", "Results_MeanWaveForms_HighGain", "Results_MeanWaveForms_LowGain", "Results_MeanCameraDisplay_HighGain", "Results_MeanCameraDisplay_LowGain", "Results_ChargeIntegration_HighGain", "Results_ChargeIntegration_LowGain", "Results_CameraMonitoring"] +#NESTED_DICT_KEYS = ["Results_CameraMonitoring"] + + +####################################################################################################################### + + + + + ######################## + + + +#START +####################################################################################################################### +for p in processors: + Chan, Samp = p.DefineForRun(reader1) + break + +for p in processors: + p.ConfigureForRun(path, Chan, Samp, reader1) + +for i, evt in enumerate(reader): + for p in processors: + p.ProcessEvent(evt, noped) + +#for the rest of the event files +for arg in args.input_paths[1:]: + print(arg) + + reader=EventSource(input_url=arg) + seeker = EventSeeker(reader) + + for i, evt in enumerate(reader): + for p in processors: + p.ProcessEvent(evt, noped) + +for p in processors: + p.FinishRun() + +dict_num = 0 +for p in processors: + NESTED_DICT[NESTED_DICT_KEYS[dict_num]] = p.GetResults() #True if want to compute plots, sedond true if want to save results + dict_num += 1 + + +name = name #in order to allow to change the name easily +p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 fits file we do this. + +#if -plot in args it will construct the figures and save them +if PlotFig == True: + for p in processors: + processor_figure_dict, processor_figure_name_dict = p.PlotResults(name, FigPath) + + for fig_plot in processor_figure_dict: + fig = processor_figure_dict[fig_plot] + SavePath = processor_figure_name_dict[fig_plot] + plt.gcf() + fig.savefig(SavePath) + + plt.clf() + plt.cla() + plt.close() + + +end = time.time() +print("Processing time:", end-start) + +#TODOS +####################################################################################################################### +#Reduce code by using loops: for figs and results +#MONGO: store results + diff --git a/src/nectarchain/dqm/trigger_statistics.py b/src/nectarchain/dqm/trigger_statistics.py index 354c760b..629cde95 100644 --- a/src/nectarchain/dqm/trigger_statistics.py +++ b/src/nectarchain/dqm/trigger_statistics.py @@ -20,7 +20,7 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.event_id = [] self.run_times = [] - def ProcessEvent(self, evt): + def ProcessEvent(self, evt, noped): trigger_type = evt.trigger.event_type.value trigger_time = evt.trigger.time.value trigger_id = evt.index.event_id From 079f71fe5c17d808139a4ad5b73fa7a277497aca Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Mon, 27 Mar 2023 10:13:53 +0200 Subject: [PATCH 06/21] Removing unwanted scripts --- src/nectarchain/dqm/ReadFitsFile.py | 20 -- src/nectarchain/dqm/ReadPickleFiles.py | 10 - ...anectarcam_monitoring_db_2022-02-24.sqlite | 0 src/nectarchain/dqm/dqm_summary_processor.py | 1 + .../dqm/start_calib_thermal_test.py | 213 ------------------ 5 files changed, 1 insertion(+), 243 deletions(-) delete mode 100644 src/nectarchain/dqm/ReadFitsFile.py delete mode 100644 src/nectarchain/dqm/ReadPickleFiles.py delete mode 100644 src/nectarchain/dqm/UsershashkarDesktopashkar_nectardatanectarcam_monitoring_db_2022-02-24.sqlite delete mode 100644 src/nectarchain/dqm/start_calib_thermal_test.py diff --git a/src/nectarchain/dqm/ReadFitsFile.py b/src/nectarchain/dqm/ReadFitsFile.py deleted file mode 100644 index f806167f..00000000 --- a/src/nectarchain/dqm/ReadFitsFile.py +++ /dev/null @@ -1,20 +0,0 @@ -from astropy.io import fits - -# Open the fits file -with fits.open('NectarCAM_Run3645_Results.fits') as hdulist: - # Get the first table by name - # Print the tables - hdulist.info() - - for i in range(1,4): - print(hdulist[i].columns.names) - - table_data1 = hdulist['Camera'].data - print(table_data1["CHARGE-INTEGRATION-IMAGE-ALL-AVERAGE-HIGH-GAIN"]) - - table_data2 = hdulist['MWF'].data - print(table_data2["WF-PHY-AVERAGE-CHAN-HIGH-GAIN"]) - - - table_data3 = hdulist['Trigger'].data - print(table_data3["TRIGGER-STATISTICS"]) \ No newline at end of file diff --git a/src/nectarchain/dqm/ReadPickleFiles.py b/src/nectarchain/dqm/ReadPickleFiles.py deleted file mode 100644 index 636090f4..00000000 --- a/src/nectarchain/dqm/ReadPickleFiles.py +++ /dev/null @@ -1,10 +0,0 @@ -import pickle -import sys - - -filename = sys.argv[1] - -infile = open(filename, "rb") -new_dict = pickle.load(infile) -infile.close() -print(new_dict) diff --git a/src/nectarchain/dqm/UsershashkarDesktopashkar_nectardatanectarcam_monitoring_db_2022-02-24.sqlite b/src/nectarchain/dqm/UsershashkarDesktopashkar_nectardatanectarcam_monitoring_db_2022-02-24.sqlite deleted file mode 100644 index e69de29b..00000000 diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index f60adf42..7bcf18f1 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -21,6 +21,7 @@ from ctapipe.image.extractor import * from ctapipe.io import EventSeeker from ctapipe.instrument import CameraGeometry +from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.io.hdf5tableio import HDF5TableWriter, HDF5TableReader diff --git a/src/nectarchain/dqm/start_calib_thermal_test.py b/src/nectarchain/dqm/start_calib_thermal_test.py deleted file mode 100644 index 5648fd43..00000000 --- a/src/nectarchain/dqm/start_calib_thermal_test.py +++ /dev/null @@ -1,213 +0,0 @@ -import os -import sys - -from matplotlib import pyplot as plt - -#from multiprocessing import Process - -import time - -from ctapipe.io import EventSource, EventSeeker - - -from mean_waveforms import MeanWaveForms_HighLowGain -from mean_camera_display import MeanCameraDisplay_HighLowGain -from charge_integration import ChargeIntegration_HighLowGain -from trigger_statistics import TriggerStatistics -from camera_monitoring import CameraMonitoring - -import argparse - -# Create an ArgumentParser object -parser = argparse.ArgumentParser(description='My script') -parser.add_argument('-p', '--plot', - action='store_true', - help='Enables plots to be generated') -parser.add_argument('-n', '--noped', - action='store_true', - help='Enables plots to be generated') -parser.add_argument('input_paths', nargs='+', help='Input paths') -args = parser.parse_args() - -#Reading arguments, paths and plot-boolean -path = args.input_paths[0] -print("Input file paths:") -print(path) -for arg in args.input_paths[1:]: - print(arg) -PlotFig = args.plot -noped = args.noped -print("Plot:", PlotFig) - -#NectarPath -NectarPath = str(os.environ['NECTARDIR']) - - - -def GetName(RunFile): - name = RunFile.split('/')[-1] - name = name.split('.')[0] + '_' + name.split('.')[1]# + '_' +name.split('.')[2] - print(name) - return name - -def CreateFigFolder(name, type): - if(type == 0): - folder = 'Plots' - - ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] - ChildrenFolderName = './' + ParentFolderName +'/' + name + '_calib' - FolderPath = NectarPath + 'output/%s/%s/' %(ChildrenFolderName, folder) - - if not os.path.exists(FolderPath): - os.makedirs(FolderPath) - - return ParentFolderName, ChildrenFolderName, FolderPath - - -start = time.time() - -#INITIATE -####################################################################################################################### -path = path -print(path) -cmap = 'gnuplot2' - -#Read and seek -reader=EventSource(input_url=path) -seeker = EventSeeker(reader) -reader1 = EventSource(input_url=path, max_events=1) -#print(reader.file_list) - -name = GetName(path) -ParentFolderName, ChildrenFolderName, FigPath = CreateFigFolder(name, 0) -ResPath = NectarPath + 'output/%s/%s' %(ChildrenFolderName, name) -####################################################################################################################### - - - - - ######################## - - - - - - - -#LIST OF PROCESSES TO RUN -####################################################################################################################### -a = TriggerStatistics(0) -b = MeanWaveForms_HighLowGain(0) #0 is for high gain and 1 is for low gain -c = MeanWaveForms_HighLowGain(1) -d = MeanCameraDisplay_HighLowGain(0) -e = MeanCameraDisplay_HighLowGain(1) -f = ChargeIntegration_HighLowGain(0) -g = ChargeIntegration_HighLowGain(1) -h = CameraMonitoring(0) - -processors = list() - -processors.append(a) -processors.append(b) -processors.append(c) -processors.append(d) -processors.append(e) -processors.append(f) -processors.append(g) -processors.append(h) -####################################################################################################################### - - - - - - ######################## - - - - -#LIST OF DICT RESULTS -####################################################################################################################### -Results_MeanWaveForms_HighGain ={} -Results_MeanWaveForms_LowGain = {} -Results_MeanCameraDisplay_HighGain = {} -Results_MeanCameraDisplay_LowGain = {} -Results_ChargeIntegration_HighGain = {} -Results_ChargeIntegration_LowGain = {} -Results_TriggerStatistics = {} -Results_CameraMonitoring = {} - -NESTED_DICT = {} #The final results dictionary -NESTED_DICT_KEYS = ["Results_TriggerStatistics", "Results_MeanWaveForms_HighGain", "Results_MeanWaveForms_LowGain", "Results_MeanCameraDisplay_HighGain", "Results_MeanCameraDisplay_LowGain", "Results_ChargeIntegration_HighGain", "Results_ChargeIntegration_LowGain", "Results_CameraMonitoring"] -#NESTED_DICT_KEYS = ["Results_CameraMonitoring"] - - -####################################################################################################################### - - - - - ######################## - - - -#START -####################################################################################################################### -for p in processors: - Chan, Samp = p.DefineForRun(reader1) - break - -for p in processors: - p.ConfigureForRun(path, Chan, Samp, reader1) - -for i, evt in enumerate(reader): - for p in processors: - p.ProcessEvent(evt, noped) - -#for the rest of the event files -for arg in args.input_paths[1:]: - print(arg) - - reader=EventSource(input_url=arg) - seeker = EventSeeker(reader) - - for i, evt in enumerate(reader): - for p in processors: - p.ProcessEvent(evt, noped) - -for p in processors: - p.FinishRun() - -dict_num = 0 -for p in processors: - NESTED_DICT[NESTED_DICT_KEYS[dict_num]] = p.GetResults() #True if want to compute plots, sedond true if want to save results - dict_num += 1 - - -name = name #in order to allow to change the name easily -p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 fits file we do this. - -#if -plot in args it will construct the figures and save them -if PlotFig == True: - for p in processors: - processor_figure_dict, processor_figure_name_dict = p.PlotResults(name, FigPath) - - for fig_plot in processor_figure_dict: - fig = processor_figure_dict[fig_plot] - SavePath = processor_figure_name_dict[fig_plot] - plt.gcf() - fig.savefig(SavePath) - - plt.clf() - plt.cla() - plt.close() - - -end = time.time() -print("Processing time:", end-start) - -#TODOS -####################################################################################################################### -#Reduce code by using loops: for figs and results -#MONGO: store results - From bddfc598f5bd85af0cddb1e526cb87aa569209b7 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Mon, 27 Mar 2023 12:44:24 +0200 Subject: [PATCH 07/21] Adding option to autimatically find runs, removing env variables, debugging --- src/nectarchain/dqm/camera_monitoring.py | 1 - src/nectarchain/dqm/dqm_summary_processor.py | 2 + src/nectarchain/dqm/start_calib.py | 99 ++++++++++++++------ 3 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index 40d39201..cfebc82b 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -168,6 +168,5 @@ def PlotResults(self, name, FigPath): except Exception as err: print("Error Code: ", err) - print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") return self.ChargeInt_Figures_Dict, self.ChargeInt_Figures_Names_Dict diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index 7bcf18f1..3acc7a2c 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -27,6 +27,7 @@ from ctapipe.io import EventSource import ctapipe.instrument.camera.readout +from ctapipe_io_nectarcam.constants import LOW_GAIN, HIGH_GAIN from astropy import time as astropytime @@ -98,5 +99,6 @@ def WriteAllResults(self,path, DICT): except: print("No Camera studies requests") FileName = path + '_Results.fits' + print(FileName) hdulist.writeto(FileName, overwrite=True) return None diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index 0e000f74..63696665 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -2,12 +2,12 @@ import sys from matplotlib import pyplot as plt - -# from multiprocessing import Process - +import argparse +#from multiprocessing import Process import time from ctapipe.io import EventSource, EventSeeker +from ctapipe_io_nectarcam.constants import LOW_GAIN, HIGH_GAIN from mean_waveforms import MeanWaveForms_HighLowGain from mean_camera_display import MeanCameraDisplay_HighLowGain @@ -15,7 +15,7 @@ from trigger_statistics import TriggerStatistics from camera_monitoring import CameraMonitoring -import argparse + # Create an ArgumentParser object parser = argparse.ArgumentParser(description='My script') @@ -24,23 +24,51 @@ help='Enables plots to be generated') parser.add_argument('-n', '--noped', action='store_true', - help='Enables plots to be generated') -parser.add_argument('input_paths', nargs='+', help='Input paths') + help='Enables pedestal subtraction in intergration') +parser.add_argument('-r', '--runnb', + action='store_true', + help='Emables findung run number automatically') + +parser.add_argument('input_paths', help='Input paths') +parser.add_argument('output_paths', help='Output paths') +parser.add_argument('input_files', nargs='+', help='Input files') + args = parser.parse_args() #Reading arguments, paths and plot-boolean -path = args.input_paths[0] -print("Input file paths:") +NectarPath = args.input_paths #str(os.environ['NECTARDIR']) +print("Input files path:", NectarPath) + +#Defining and priting the paths of the output files. +output_path = args.output_paths +print("Output path:", output_path) + + +#Defining and priting the paths of the input files. + +#Read run automatocally is the -r option is provided...NOT TESTED YET +AutomaticRun = args.runnb +if AutomaticRun == True: + from nectarchain.calibration.container import utils + dm = utils.DataManagement() + path1 = dm.findrun(args.input_files) +else: #OTHERWISE READ THE RUNS FROM ARGS + path1 = args.input_files[0] + +#THE PATH OF INPUT FILES +path = NectarPath + path1 +print("Input files:") print(path) -for arg in args.input_paths[1:]: +for arg in args.input_files[1:]: print(arg) + +#Defining and priting the options PlotFig = args.plot noped = args.noped + print("Plot:", PlotFig) print("Noped:", noped) -#NectarPath -NectarPath = str(os.environ['NECTARDIR']) @@ -51,13 +79,9 @@ def GetName(RunFile): return name -def CreateFigFolder(name, type): - if type == 0: - folder = "Plots" - - ParentFolderName = name.split("_")[0] + "_" + name.split("_")[1] - ChildrenFolderName = "./" + ParentFolderName + "/" + name + "_calib" - FolderPath = NectarPath + "output/%s/%s/" % (ChildrenFolderName, folder) + ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] + ChildrenFolderName = './' + ParentFolderName +'/' + name + '_calib' + FolderPath = output_path + 'output/%s/%s/' %(ChildrenFolderName, folder) if not os.path.exists(FolderPath): os.makedirs(FolderPath) @@ -80,18 +104,30 @@ def CreateFigFolder(name, type): name = GetName(path) ParentFolderName, ChildrenFolderName, FigPath = CreateFigFolder(name, 0) -ResPath = NectarPath + "output/%s/%s" % (ChildrenFolderName, name) +ResPath = output_path + 'output/%s/%s' %(ChildrenFolderName, name) +####################################################################################################################### + + + + + ######################## -# LIST OF PROCESSES TO RUN -a = TriggerStatistics(0) -b = MeanWaveForms_HighLowGain(0) # 0 is for high gain and 1 is for low gain -c = MeanWaveForms_HighLowGain(1) -d = MeanCameraDisplay_HighLowGain(0) -e = MeanCameraDisplay_HighLowGain(1) -f = ChargeIntegration_HighLowGain(0) -g = ChargeIntegration_HighLowGain(1) -h = CameraMonitoring(0) + + + + + +#LIST OF PROCESSES TO RUN +####################################################################################################################### +a = TriggerStatistics(HIGH_GAIN) +b = MeanWaveForms_HighLowGain(HIGH_GAIN) #0 is for high gain and 1 is for low gain +c = MeanWaveForms_HighLowGain(LOW_GAIN) +d = MeanCameraDisplay_HighLowGain(HIGH_GAIN) +e = MeanCameraDisplay_HighLowGain(LOW_GAIN) +f = ChargeIntegration_HighLowGain(HIGH_GAIN) +g = ChargeIntegration_HighLowGain(LOW_GAIN) +h = CameraMonitoring(HIGH_GAIN) processors = list() @@ -141,10 +177,11 @@ def CreateFigFolder(name, type): p.ProcessEvent(evt, noped) #for the rest of the event files -for arg in args.input_paths[1:]: - print(arg) +for arg in args.input_files[1:]: + path2 = NectarPath + arg + print(path2) - reader=EventSource(input_url=arg) + reader=EventSource(input_url=path2) seeker = EventSeeker(reader) for i, evt in enumerate(reader): From 51a159a5d0d0a93152ad55709afbadbc482475d4 Mon Sep 17 00:00:00 2001 From: hashkar <43830561+hashkar@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:37:27 +0200 Subject: [PATCH 08/21] Update README --- src/nectarchain/dqm/README | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/nectarchain/dqm/README b/src/nectarchain/dqm/README index 73a5e4af..78961301 100644 --- a/src/nectarchain/dqm/README +++ b/src/nectarchain/dqm/README @@ -1,6 +1,22 @@ -Environment variable needed: -export NECTARDIR=path/to/ParentFoderOfNectarchain/ #This is where the nectarchain folder is checked out. The output folder will be created there also. - To launch the script: source activate nectarchain #this is the environment required for the nectarchain analysis -python start_calib.py /path/NectarCAM.Run2720.0000.fits.fz /path/NectarCAM.Run2720.0001.fits.fz + +For automatic run files finding: +Arguments: +Input (data) directory +Output parent directory +file number +option -r to automatically find run files +Example: python python start_calib.py $NECTARDATA $NECTARDIR 2720 -r + +For manually inputing the run files: +Arguments: +Input (data) directory +Output parent directory +run files names +Example: python start_calib.py $NECTARDATA $NECTARDIR NectarCAM.Run2720.0000.fits.fz NectarCAM.Run2720.0001.fits.fz + + +More options: +-p to get plots and save them +-n to substract pedestal from integration From 2f1e568eee75d6e6784ff2323b3c57774ba240f1 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 11:50:35 +0200 Subject: [PATCH 09/21] Fixed typos --- src/nectarchain/dqm/start_calib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index 63696665..8dbe2582 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -27,7 +27,7 @@ help='Enables pedestal subtraction in intergration') parser.add_argument('-r', '--runnb', action='store_true', - help='Emables findung run number automatically') + help='Enables finding runs automatically on DIRAC') parser.add_argument('input_paths', help='Input paths') parser.add_argument('output_paths', help='Output paths') From 0f13cab5ee6176f5fbefdeb19b0ca172ed14041a Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 13:20:32 +0200 Subject: [PATCH 10/21] Corrected example call to `start_calib.py` --- src/nectarchain/dqm/README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/dqm/README b/src/nectarchain/dqm/README index 78961301..d10c7773 100644 --- a/src/nectarchain/dqm/README +++ b/src/nectarchain/dqm/README @@ -1,5 +1,5 @@ To launch the script: -source activate nectarchain #this is the environment required for the nectarchain analysis +source activate nectarchain # this is the environment required for the nectarchain analysis For automatic run files finding: Arguments: @@ -7,7 +7,7 @@ Input (data) directory Output parent directory file number option -r to automatically find run files -Example: python python start_calib.py $NECTARDATA $NECTARDIR 2720 -r +Example: python start_calib.py -r 2720 $NECTARCAMDATA $NECTARDIR For manually inputing the run files: Arguments: From 5d8c6c7485cb3c41e2c80dde8044876ccb666548 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 13:50:01 +0200 Subject: [PATCH 11/21] Fixed usage of `nectarchain.calibration.container.utils.DataManagement().findrun`, fixed bug in parsing input files, corrected optional `runnb` optional argument --- src/nectarchain/dqm/start_calib.py | 41 +++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index 8dbe2582..b27d5994 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -26,37 +26,44 @@ action='store_true', help='Enables pedestal subtraction in intergration') parser.add_argument('-r', '--runnb', - action='store_true', - help='Enables finding runs automatically on DIRAC') + help='Optional run number, automatically found on DIRAC', + type=int) +parser.add_argument('-i', '--input-files', + nargs='+', + help='Input files') parser.add_argument('input_paths', help='Input paths') parser.add_argument('output_paths', help='Output paths') -parser.add_argument('input_files', nargs='+', help='Input files') -args = parser.parse_args() +args, leftovers = parser.parse_known_args() -#Reading arguments, paths and plot-boolean -NectarPath = args.input_paths #str(os.environ['NECTARDIR']) -print("Input files path:", NectarPath) +# Reading arguments, paths and plot-boolean +NectarPath = args.input_paths # str(os.environ['NECTARDIR']) +print("Input file path:", NectarPath) -#Defining and priting the paths of the output files. +# Defining and printing the paths of the output files. output_path = args.output_paths print("Output path:", output_path) -#Defining and priting the paths of the input files. +# Defining and printing the paths of the input files. #Read run automatocally is the -r option is provided...NOT TESTED YET -AutomaticRun = args.runnb -if AutomaticRun == True: +if args.runnb is not None: from nectarchain.calibration.container import utils dm = utils.DataManagement() - path1 = dm.findrun(args.input_files) -else: #OTHERWISE READ THE RUNS FROM ARGS - path1 = args.input_files[0] - -#THE PATH OF INPUT FILES -path = NectarPath + path1 + _, filelist = dm.findrun(args.runnb) + args.input_files = filelist + path1 = filelist[0].name +else: # OTHERWISE READ THE RUNS FROM ARGS + if args.input_files is not None: + path1 = args.input_files[0] + else: + print('Input files should be provided, exiting...') + sys.exit(1) + +#THE PATH OF INPUT FILES +path = f'{NectarPath}/{path1}' print("Input files:") print(path) for arg in args.input_files[1:]: From daeb2f0bdfc2f6206c39147fab3b08ee882941a6 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 14:09:05 +0200 Subject: [PATCH 12/21] Definition line for `CreateFigFolder` was lost in a previous commit. --- src/nectarchain/dqm/start_calib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index b27d5994..6cfebf56 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -85,7 +85,7 @@ def GetName(RunFile): print(name) return name - +def CreateFigFolder(name, type): ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] ChildrenFolderName = './' + ParentFolderName +'/' + name + '_calib' FolderPath = output_path + 'output/%s/%s/' %(ChildrenFolderName, folder) From f4e5f582992db1b2190ffd620d0e31e1c309c1a9 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 14:11:34 +0200 Subject: [PATCH 13/21] Yet a few missing lines in `CreateFigFolder` which were lost in a previous commit. --- src/nectarchain/dqm/start_calib.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index 6cfebf56..e6fea830 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -86,6 +86,9 @@ def GetName(RunFile): return name def CreateFigFolder(name, type): + if type == 0: + folder = "Plots" + ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] ChildrenFolderName = './' + ParentFolderName +'/' + name + '_calib' FolderPath = output_path + 'output/%s/%s/' %(ChildrenFolderName, folder) From 86952ccbf4ef99a00b8e32ef9c71df847d77bbd4 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 14:15:37 +0200 Subject: [PATCH 14/21] Adapted examples in README to show how to launch the DQM on: - files automatically retrieved from DIRAC - local files --- src/nectarchain/dqm/README | 6 +++--- src/nectarchain/dqm/start_calib.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nectarchain/dqm/README b/src/nectarchain/dqm/README index d10c7773..fef4edd0 100644 --- a/src/nectarchain/dqm/README +++ b/src/nectarchain/dqm/README @@ -6,15 +6,15 @@ Arguments: Input (data) directory Output parent directory file number -option -r to automatically find run files +option -r to automatically find run files on DIRAC Example: python start_calib.py -r 2720 $NECTARCAMDATA $NECTARDIR For manually inputing the run files: Arguments: Input (data) directory Output parent directory -run files names -Example: python start_calib.py $NECTARDATA $NECTARDIR NectarCAM.Run2720.0000.fits.fz NectarCAM.Run2720.0001.fits.fz +Option -i : local run file names +Example: python start_calib.py $NECTARDATA $NECTARDIR -i NectarCAM.Run2720.0000.fits.fz NectarCAM.Run2720.0001.fits.fz More options: diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index e6fea830..ff4ccdc6 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -30,7 +30,7 @@ type=int) parser.add_argument('-i', '--input-files', nargs='+', - help='Input files') + help='Local input files') parser.add_argument('input_paths', help='Input paths') parser.add_argument('output_paths', help='Output paths') From 8b9b3beb41879ed62ae5e129a26afd232828c168 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 14:17:07 +0200 Subject: [PATCH 15/21] Add `__init__.py` into DQM, so that DQM is shipped when deploying on PyPI and conda-forge. --- src/nectarchain/dqm/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/nectarchain/dqm/__init__.py diff --git a/src/nectarchain/dqm/__init__.py b/src/nectarchain/dqm/__init__.py new file mode 100644 index 00000000..e69de29b From 4cbbfb406b39ca5b208e192d2d2380318d802ae2 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 14:24:18 +0200 Subject: [PATCH 16/21] Improved documentation --- src/nectarchain/dqm/README | 22 ---------------- src/nectarchain/dqm/README.md | 41 ++++++++++++++++++++++++++++++ src/nectarchain/dqm/start_calib.py | 4 +-- 3 files changed, 43 insertions(+), 24 deletions(-) delete mode 100644 src/nectarchain/dqm/README create mode 100644 src/nectarchain/dqm/README.md diff --git a/src/nectarchain/dqm/README b/src/nectarchain/dqm/README deleted file mode 100644 index fef4edd0..00000000 --- a/src/nectarchain/dqm/README +++ /dev/null @@ -1,22 +0,0 @@ -To launch the script: -source activate nectarchain # this is the environment required for the nectarchain analysis - -For automatic run files finding: -Arguments: -Input (data) directory -Output parent directory -file number -option -r to automatically find run files on DIRAC -Example: python start_calib.py -r 2720 $NECTARCAMDATA $NECTARDIR - -For manually inputing the run files: -Arguments: -Input (data) directory -Output parent directory -Option -i : local run file names -Example: python start_calib.py $NECTARDATA $NECTARDIR -i NectarCAM.Run2720.0000.fits.fz NectarCAM.Run2720.0001.fits.fz - - -More options: --p to get plots and save them --n to substract pedestal from integration diff --git a/src/nectarchain/dqm/README.md b/src/nectarchain/dqm/README.md new file mode 100644 index 00000000..13ec9e9f --- /dev/null +++ b/src/nectarchain/dqm/README.md @@ -0,0 +1,41 @@ +# Quick recipe for the Data Quality Monitoring (DQM) scripts + +To launch the DQM, first activate the `nectarchain` `conda` environment: + +```shell +source activate nectarchain +``` + +Usage: + +```shell +$ python start_calib.py -h + +usage: start_calib.py [-h] [-p] [-n] [-r RUNNB] [-i INPUT_FILES [INPUT_FILES ...]] input_paths output_paths + +NectarCAM Data Quality Monitoring tool + +positional arguments: + input_paths Input paths + output_paths Output paths + +optional arguments: + -h, --help show this help message and exit + -p, --plot Enables plots to be generated + -n, --noped Enables pedestal subtraction in charge integration + -r RUNNB, --runnb RUNNB + Optional run number, automatically found on DIRAC + -i INPUT_FILES [INPUT_FILES ...], --input-files INPUT_FILES [INPUT_FILES ...] + Local input files +``` + +To automatically find and retrieve run files from DIRAC, use the `-r` option: + +```shell +python start_calib.py -r 2720 $NECTARCAMDATA $NECTARDIR +``` + +To manually use local run files, use the `-i` option **after** indicating the positional arguments for input and output directories: +```shell +python start_calib.py $NECTARDATA $NECTARDIR -i NectarCAM.Run2720.0000.fits.fz NectarCAM.Run2720.0001.fits.fz +``` diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index ff4ccdc6..b0dae4ed 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -18,13 +18,13 @@ # Create an ArgumentParser object -parser = argparse.ArgumentParser(description='My script') +parser = argparse.ArgumentParser(description='NectarCAM Data Quality Monitoring tool') parser.add_argument('-p', '--plot', action='store_true', help='Enables plots to be generated') parser.add_argument('-n', '--noped', action='store_true', - help='Enables pedestal subtraction in intergration') + help='Enables pedestal subtraction in charge integration') parser.add_argument('-r', '--runnb', help='Optional run number, automatically found on DIRAC', type=int) From 7682ce3600915d4c18e7af88aec339ad896b7dad Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 16:37:51 +0200 Subject: [PATCH 17/21] Fixed bug to properly concatenate `PosixPath` and `str`. Only keep basename when parsing run paths from DIRAC. --- src/nectarchain/dqm/start_calib.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index b0dae4ed..e34eae7e 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -48,19 +48,18 @@ # Defining and printing the paths of the input files. -#Read run automatocally is the -r option is provided...NOT TESTED YET if args.runnb is not None: + # Grab runs automatically from DIRAC is the -r option is provided from nectarchain.calibration.container import utils dm = utils.DataManagement() _, filelist = dm.findrun(args.runnb) - args.input_files = filelist - path1 = filelist[0].name -else: # OTHERWISE READ THE RUNS FROM ARGS - if args.input_files is not None: - path1 = args.input_files[0] - else: - print('Input files should be provided, exiting...') - sys.exit(1) + args.input_files = [s.name for s in filelist] +elif args.input_files is None: + print('Input files should be provided, exiting...') + sys.exit(1) + +# OTHERWISE READ THE RUNS FROM ARGS +path1 = args.input_files[0] #THE PATH OF INPUT FILES path = f'{NectarPath}/{path1}' @@ -188,7 +187,7 @@ def CreateFigFolder(name, type): #for the rest of the event files for arg in args.input_files[1:]: - path2 = NectarPath + arg + path2 = f'{NectarPath}/{arg}' print(path2) reader=EventSource(input_url=path2) From 2fcfac9dfa47705582a01f458361636963481981 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 14 Apr 2023 17:06:17 +0200 Subject: [PATCH 18/21] Fix output paths. --- src/nectarchain/dqm/start_calib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index e34eae7e..6d482afe 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -90,7 +90,7 @@ def CreateFigFolder(name, type): ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] ChildrenFolderName = './' + ParentFolderName +'/' + name + '_calib' - FolderPath = output_path + 'output/%s/%s/' %(ChildrenFolderName, folder) + FolderPath = f'{output_path}/output/{ChildrenFolderName}/{folder}' if not os.path.exists(FolderPath): os.makedirs(FolderPath) @@ -113,7 +113,7 @@ def CreateFigFolder(name, type): name = GetName(path) ParentFolderName, ChildrenFolderName, FigPath = CreateFigFolder(name, 0) -ResPath = output_path + 'output/%s/%s' %(ChildrenFolderName, name) +ResPath = f'{output_path}/output/{ChildrenFolderName}/{name}' ####################################################################################################################### From 016ca9322b614dbfab820fe49374888a3b924f08 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Tue, 18 Apr 2023 10:37:24 +0200 Subject: [PATCH 19/21] fixing the camera geometry in the plots --- src/nectarchain/dqm/camera_monitoring.py | 3 ++- src/nectarchain/dqm/charge_integration.py | 14 +++++++++++++- src/nectarchain/dqm/mean_camera_display.py | 14 ++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index cfebc82b..198cb5cd 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -2,6 +2,7 @@ from matplotlib import pyplot as plt from ctapipe.visualization import CameraDisplay from ctapipe.instrument import CameraGeometry +from ctapipe.coordinates import EngineeringCameraFrame from astropy import time as astropytime import numpy as np import sqlite3 @@ -17,7 +18,7 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.Chan = Chan self.Samp = Samp - self.camera = CameraGeometry.from_name("NectarCam", 3) + self.camera = self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam", 3) self.cmap = "gnuplot2" self.subarray = Reader1.subarray diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 383dcfe7..c4c395b6 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -3,6 +3,7 @@ import numpy as np from ctapipe.visualization import CameraDisplay from ctapipe.instrument import CameraGeometry +from ctapipe.coordinates import EngineeringCameraFrame from traitlets.config.loader import Config import ctapipe.instrument.camera.readout from ctapipe.image import LocalPeakWindowSum @@ -21,7 +22,7 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 - self.camera = CameraGeometry.from_name("NectarCam-003") + self.camera = self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam", 3) self.cmap = "gnuplot2" # reader1=EventSource(input_url=path, max_events = 1) @@ -47,14 +48,25 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): def ProcessEvent(self, evt, noped): #print("test", evt.r0.tel[0].waveform[0]) + pixels = evt.nectarcam.tel[0].svc.pixel_ids + #pixel21 = np.arange(0,21,1,dtype = int) + pixels = list(pixels) + #pixel21 = list(pixel21) + #pixels = np.concatenate([pixel21,pixel]) + waveform=evt.r0.tel[0].waveform[self.k] + if noped == True: ped = np.mean(waveform[:, 20]) w_noped = waveform - ped image, peakpos = self.integrator(w_noped,0,np.zeros(self.Chan, dtype = int)) + image = image[pixels] + peakpos = peakpos[pixels] else: image, peakpos = self.integrator(waveform,0,np.zeros(self.Chan, dtype = int)) + image = image[pixels] + peakpos = peakpos[pixels] if evt.trigger.event_type.value == 32: # count peds self.counter_ped += 1 diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index de915188..1b796900 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -2,9 +2,11 @@ from matplotlib import pyplot as plt from ctapipe.visualization import CameraDisplay from ctapipe.instrument import CameraGeometry +from ctapipe.coordinates import EngineeringCameraFrame import numpy as np + class MeanCameraDisplay_HighLowGain(dqm_summary): def __init__(self, gaink): self.k = gaink @@ -20,13 +22,19 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 - self.camera = CameraGeometry.from_name("NectarCam-003") - self.camera2 = CameraGeometry.from_name("NectarCam-003") + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam-003") + self.camera2 = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam-003") self.cmap = "gnuplot2" self.cmap2 = "gnuplot2" def ProcessEvent(self, evt, noped): + pixels = evt.nectarcam.tel[0].svc.pixel_ids + #pixel21 = np.arange(0,21,1,dtype = int) + pixels = list(pixels) + #pixel21 = list(pixel21) + #pixels = np.concatenate([pixel21,pixel]) + if evt.trigger.event_type.value == 32: #count peds self.counter_ped += 1 else: @@ -36,10 +44,12 @@ def ProcessEvent(self, evt, noped): self.CameraAverage_ped += ( evt.r0.tel[0].waveform[self.k].sum(axis=1) ) # fill channels one by one and sum them for peds only + self.CameraAverage_ped = self.CameraAverage_ped[pixels] else: self.CameraAverage += ( evt.r0.tel[0].waveform[self.k].sum(axis=1) ) # fill channels one by one and sum them + self.CameraAverage = self.CameraAverage[pixels] return None def FinishRun(self): From 1bec9990f4e3d71dfe59088993b737e1c95956d3 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Tue, 18 Apr 2023 10:38:29 +0200 Subject: [PATCH 20/21] fixing camera geometry with missing first 21 pixels --- src/nectarchain/dqm/charge_integration.py | 10 +++++----- src/nectarchain/dqm/mean_camera_display.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index c4c395b6..efa2d32e 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -48,11 +48,11 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): def ProcessEvent(self, evt, noped): #print("test", evt.r0.tel[0].waveform[0]) - pixels = evt.nectarcam.tel[0].svc.pixel_ids - #pixel21 = np.arange(0,21,1,dtype = int) - pixels = list(pixels) - #pixel21 = list(pixel21) - #pixels = np.concatenate([pixel21,pixel]) + pixel = evt.nectarcam.tel[0].svc.pixel_ids + pixel21 = np.arange(0,21,1,dtype = int) + pixel = list(pixel) + pixel21 = list(pixel21) + pixels = np.concatenate([pixel21,pixel]) waveform=evt.r0.tel[0].waveform[self.k] diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index 1b796900..6016f14c 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -29,11 +29,11 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.cmap2 = "gnuplot2" def ProcessEvent(self, evt, noped): - pixels = evt.nectarcam.tel[0].svc.pixel_ids - #pixel21 = np.arange(0,21,1,dtype = int) - pixels = list(pixels) - #pixel21 = list(pixel21) - #pixels = np.concatenate([pixel21,pixel]) + pixel = evt.nectarcam.tel[0].svc.pixel_ids + pixel21 = np.arange(0,21,1,dtype = int) + pixel = list(pixel) + pixel21 = list(pixel21) + pixels = np.concatenate([pixel21,pixel]) if evt.trigger.event_type.value == 32: #count peds self.counter_ped += 1 From a05bed70dc10a42bfc5fb63d7691e9a4f52dea02 Mon Sep 17 00:00:00 2001 From: jlenain Date: Tue, 18 Apr 2023 11:15:40 +0200 Subject: [PATCH 21/21] Fixed code typo --- src/nectarchain/dqm/camera_monitoring.py | 2 +- src/nectarchain/dqm/charge_integration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index 198cb5cd..03d8c338 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -18,7 +18,7 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.Chan = Chan self.Samp = Samp - self.camera = self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam", 3) + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam", 3) self.cmap = "gnuplot2" self.subarray = Reader1.subarray diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index efa2d32e..f4c98b7c 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -22,7 +22,7 @@ def ConfigureForRun(self, path, Chan, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 - self.camera = self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam", 3) + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame())#CameraGeometry.from_name("NectarCam", 3) self.cmap = "gnuplot2" # reader1=EventSource(input_url=path, max_events = 1)