From 0f246eac6f0bff3a18d165f4a67a535e33c26cc4 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 1 Dec 2019 09:43:34 -0500 Subject: [PATCH] Initial steps to modernizing task_schedule --- .gitignore | 4 ++ Makefile | 97 +++++++++++++++++-------------- t/data/basic/basic.config | 12 ++-- t/data/exception/exception.config | 10 ++-- t/data/fail/fail.config | 18 +++--- t/data/send_mail/send_mail.config | 6 +- t/data/test.config | 8 +-- task_schedule3.pl | 70 ++++++++++++---------- 8 files changed, 122 insertions(+), 103 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d486a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/data +/bin +/dat +/doc \ No newline at end of file diff --git a/Makefile b/Makefile index 1041956..bf1c974 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,11 @@ # Set the task name TASK = task_schedule -# Uncomment the correct choice indicating either SKA or TST flight environment -FLIGHT_ENV = SKA - # Set the names of all files that get installed BIN = task_schedule3.pl -include /proj/sot/ska/include/Makefile.FLIGHT # Define outside data and bin dependencies required for testing, -# i.e. all tools and data required by the task which are NOT +# i.e. all tools and data required by the task which are NOT # created by or internal to the task itself. These will be copied # first from the local test directory t/ and if not found from the # ROOT_FLIGHT area. @@ -22,56 +18,67 @@ TEST_DEP = data/basic/basic.config \ bin/watch_cron_logs.pl \ bin/exception.py +TEST_SKA = $(PWD)/test_ska + +# Define installation PREFIX as the sys.prefix of python in the PATH. +PREFIX = $(shell python -c 'import sys; print(sys.prefix)') + # To 'test', first check that the INSTALL root is not the same as the FLIGHT # root with 'check_install' (defined in Makefile.FLIGHT). Typically this means # doing 'setenv TST .'. Then copy any outside data or bin dependencies into local # directory via dependency rules defined in Makefile.FLIGHT. Finally install -# the task, typically in '.'. +# the task, typically in '.'. test: test_basic test_exception test_basic_full test_fail test_send_mail test_send_mail_quiet -test_basic: check_install $(TEST_DEP) install - rm -f $(INSTALL)/data/basic/task_sched_heartbeat - rm -f $(INSTALL)/data/basic/task_sched_heart_attack - rm -f $(INSTALL)/data/basic/task_sched_disable_alerts - perl $(INSTALL_BIN)/task_schedule3.pl -alert $(USER) -config $(INSTALL)/data/basic/basic.config -fast 6 -no-email -loud - -test_exception:check_install $(TEST_DEP) install - rm -f $(INSTALL)/data/exception/task_sched_heartbeat - rm -f $(INSTALL)/data/exception/task_sched_heart_attack - rm -f $(INSTALL)/data/exception/task_sched_disable_alerts - perl $(INSTALL_BIN)/task_schedule3.pl -alert $(USER) -config $(INSTALL)/data/exception/exception.config -fast 6 -no-email -loud - -test_basic_full: check_install $(TEST_DEP) install - rm -f $(INSTALL)/data/basic/task_sched_heartbeat - rm -f $(INSTALL)/data/basic/task_sched_heart_attack - rm -f $(INSTALL)/data/basic/task_sched_disable_alerts - perl $(INSTALL_BIN)/task_schedule3.pl -alert $(USER) -config $(INSTALL)/data/basic/basic.config -loud - -test_fail: check_install $(TEST_DEP) install - rm -f $(INSTALL)/data/fail/task_sched_heartbeat - rm -f $(INSTALL)/data/fail/task_sched_heart_attack - rm -f $(INSTALL)/data/fail/task_sched_disable_alerts - perl $(INSTALL_BIN)/task_schedule3.pl -alert $(USER) -config $(INSTALL)/data/fail/fail.config -fast 20 -loud - -test_send_mail: check_install $(TEST_DEP) install - rm -f $(INSTALL)/data/send_mail/task_sched_heartbeat - rm -f $(INSTALL)/data/send_mail/task_sched_heart_attack - rm -f $(INSTALL)/data/send_mail/task_sched_disable_alerts - perl $(INSTALL_BIN)/task_schedule3.pl -alert $(USER) -config $(INSTALL)/data/send_mail/send_mail.config -fast 5 -loud - -test_send_mail_quiet: check_install $(TEST_DEP) install - rm -f $(INSTALL)/data/send_mail/task_sched_heartbeat - rm -f $(INSTALL)/data/send_mail/task_sched_heart_attack - rm -f $(INSTALL)/data/send_mail/task_sched_disable_alerts - perl $(INSTALL_BIN)/task_schedule3.pl -alert $(USER) -config $(INSTALL)/data/send_mail/send_mail.config -fast 5 +show_prefix: + echo $(PREFIX) + +test_basic: + rm -rf $(TEST_SKA) + rsync -av t/ $(TEST_SKA)/ + env SKA=$(TEST_SKA) ./task_schedule3.pl -alert $(USER) \ + -config $(TEST_SKA)/data/basic/basic.config -fast 6 -no-email -loud + +test_exception: + rm -rf $(TEST_SKA) + rsync -av t/ $(TEST_SKA)/ + env SKA=$(TEST_SKA) ./task_schedule3.pl -alert $(USER) \ + -config $(TEST_SKA)/data/exception/exception.config -fast 6 -no-email -loud + +test_basic_full: + rm -rf $(TEST_SKA) + rsync -av t/ $(TEST_SKA)/ + env SKA=$(TEST_SKA) ./task_schedule3.pl -alert $(USER) \ + -config $(TEST_SKA)/data/basic/basic.config -loud + +test_fail: + rm -rf $(TEST_SKA) + rsync -av t/ $(TEST_SKA)/ + env SKA=$(TEST_SKA) ./task_schedule3.pl -alert $(USER) \ + -config $(TEST_SKA)/data/fail/fail.config -fast 20 -loud + +test_send_mail: + rm -rf $(TEST_SKA) + rsync -av t/ $(TEST_SKA)/ + env SKA=$(TEST_SKA) ./task_schedule3.pl -alert $(USER) \ + -config $(TEST_SKA)/data/send_mail/send_mail.config -fast 5 -loud + +test_send_mail_quiet: + rm -rf $(TEST_SKA) + rsync -av t/ $(TEST_SKA)/ + env SKA=$(TEST_SKA) ./task_schedule3.pl -alert $(USER) \ + -config $(TEST_SKA)/data/send_mail/send_mail.config -fast 5 install: - mkdir -p $(INSTALL_BIN); rsync --times --cvs-exclude $(BIN) $(INSTALL_BIN)/ - mkdir -p $(INSTALL_DOC) - pod2html task_schedule3.pl > $(INSTALL_DOC)/index.html + mkdir -p $(PREFIX)/bin + rsync --times --cvs-exclude $(BIN) $(PREFIX)/bin/ + +install_doc: + mkdir -p $(SKA)/doc/$(TASK) + pod2html task_schedule3.pl > $(SKA)/doc/$(TASK)/index.html rm -f pod2htm?.tmp clean: - rm -r bin data doc + rm -r $(TEST_SKA) diff --git a/t/data/basic/basic.config b/t/data/basic/basic.config index 1c4e1ec..302911c 100644 --- a/t/data/basic/basic.config +++ b/t/data/basic/basic.config @@ -3,14 +3,14 @@ subject Basic tests # subject of email timeout 1000 # Default tool timeout heartbeat_timeout 120 # Maximum age of heartbeat file (seconds) -iterations 5 +iterations 3 # Data files and directories. The *_dir vars can have $ENV{} vars which # get interpolated. (Note lack of task name after TST_DATA because this is just for test). -data_dir $ENV{SKA_DATA}/basic # Data file directory -log_dir $ENV{SKA_DATA}/basic/logs # Log file directory -bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) +data_dir $ENV{SKA}/data/basic # Data file directory +log_dir $ENV{SKA}/data/basic/logs # Log file directory +bin_dir $ENV{SKA}/bin # Bin dir (optional, see task def'n) # Email addresses that receive an alert if there was a severe error in # running jobs (i.e. couldn't start jobs or couldn't open log file). @@ -20,7 +20,7 @@ bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) # Define task parameters # cron: Job repetition specification ala crontab -# exec: Name of executable. Can have $ENV{} vars which get interpolated. +# exec: Name of executable. Can have $ENV{} vars which get interpolated. # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. # log: Name of log. Can have $ENV{} vars which get interpolated. # If log is set to '' then no log file will be created @@ -37,7 +37,7 @@ bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) cron * * * * * check_cron = 0-59/2 * * * * exec 1 : task1.pl 1 - exec 1 : $ENV{SKA_BIN}/task1.pl 2 + exec 1 : $ENV{SKA}/bin/task1.pl 2 timeout 100 context 1 diff --git a/t/data/exception/exception.config b/t/data/exception/exception.config index 2956bff..03d9137 100644 --- a/t/data/exception/exception.config +++ b/t/data/exception/exception.config @@ -8,9 +8,9 @@ iterations 1 # Data files and directories. The *_dir vars can have $ENV{} vars which # get interpolated. (Note lack of task name after TST_DATA because this is just for test). -data_dir $ENV{SKA_DATA}/exception # Data file directory -log_dir $ENV{SKA_DATA}/exception/logs # Log file directory -bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) +data_dir $ENV{SKA}/data/exception # Data file directory +log_dir $ENV{SKA}/data/exception/logs # Log file directory +bin_dir $ENV{SKA}/bin # Bin dir (optional, see task def'n) # Email addresses that receive an alert if there was a severe error in # running jobs (i.e. couldn't start jobs or couldn't open log file). @@ -20,7 +20,7 @@ bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) # Define task parameters # cron: Job repetition specification ala crontab -# exec: Name of executable. Can have $ENV{} vars which get interpolated. +# exec: Name of executable. Can have $ENV{} vars which get interpolated. # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. # log: Name of log. Can have $ENV{} vars which get interpolated. # If log is set to '' then no log file will be created @@ -37,7 +37,7 @@ bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) cron * * * * * check_cron = * * * * * exec task1.pl 1 - exec python $ENV{SKA_BIN}/exception.py + exec $ENV{SKA}/bin/exception.py timeout 100 context 1 diff --git a/t/data/fail/fail.config b/t/data/fail/fail.config index c2a4049..def0b9d 100644 --- a/t/data/fail/fail.config +++ b/t/data/fail/fail.config @@ -9,9 +9,9 @@ heartbeat_timeout 120 # Maximum age of heartbeat file (seconds) # Data files and directories. The *_dir vars can have $ENV{} vars which # get interpolated. (Note lack of task name after TST_DATA because this is just for test). -data_dir $ENV{SKA_DATA}/fail # Data file directory -log_dir $ENV{SKA_DATA}/fail/logs # Log file directory -bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) +data_dir $ENV{SKA}/data/fail # Data file directory +log_dir $ENV{SKA}/data/fail/logs # Log file directory +bin_dir $ENV{SKA}/bin # Bin dir (optional, see task def'n) # Email addresses that receive an alert if there was a severe error in # running jobs (i.e. couldn't start jobs or couldn't open log file). @@ -21,7 +21,7 @@ bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) # Define task parameters # cron: Job repetition specification ala crontab -# exec: Name of executable. Can have $ENV{} vars which get interpolated. +# exec: Name of executable. Can have $ENV{} vars which get interpolated. # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. # log: Name of log. Can have $ENV{} vars which get interpolated. # If log is set to '' then no log file will be created @@ -35,25 +35,25 @@ bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) exec task1.pll 20 log task1_nonstandard.log - + # Task that times out cron * * * * * exec task2.pl 10 timeout 5 - + # Can't write log file cron * * * * * exec task1.pl 10 log /cant/write/to/this/logfile - + # Executable that returns non-zero status cron * * * * * exec /bin/ls doesntexist - - + + diff --git a/t/data/send_mail/send_mail.config b/t/data/send_mail/send_mail.config index 1504ee7..8dda4ee 100644 --- a/t/data/send_mail/send_mail.config +++ b/t/data/send_mail/send_mail.config @@ -18,9 +18,9 @@ NOTIFY # Data files and directories. The *_dir vars can have $ENV{} vars which # get interpolated. (Note lack of task name after TST_DATA because this is just for test). -data_dir $ENV{SKA_DATA}/send_mail # Data file directory -log_dir $ENV{SKA_DATA}/send_mail/logs # Log file directory -bin_dir $ENV{SKA_BIN} # Bin dir ($ENV{SKA_BIN} is typical) +data_dir $ENV{SKA}/data/send_mail # Data file directory +log_dir $ENV{SKA}/data/send_mail/logs # Log file directory +bin_dir $ENV{SKA}/bin # Bin dir ($ENV{SKA_BIN} is typical) # Email addresses that receive notification that the task ran diff --git a/t/data/test.config b/t/data/test.config index cd31d31..ce36f3a 100644 --- a/t/data/test.config +++ b/t/data/test.config @@ -8,9 +8,9 @@ heartbeat_timeout 120 # Maximum age of heartbeat file (seconds) # Data files and directories. The *_dir vars can have $ENV{} vars which # get interpolated. (Note lack of task name after TST_DATA because this is just for test). -data_dir $ENV{SKA_DATA} # Data file directory -log_dir $ENV{SKA_DATA}/Logs # Log file directory -bin_dir $ENV{SKA_BIN} # Bin dir (optional, see task def'n) +data_dir $ENV{SKA}/data # Data file directory +log_dir $ENV{SKA}/data/Logs # Log file directory +bin_dir $ENV{SKA}/bin # Bin dir (optional, see task def'n) master_log Master.log # Composite master log (created in log_dir) heartbeat heartbeat # File to ensure sched. running (in data_dir) @@ -57,7 +57,7 @@ heartbeat heartbeat # File to ensure sched. running (in data_dir) cron * * * * * exec task1.pl 1 - exec 2 : $ENV{SKA_BIN}/task1.pl 2 + exec 2 : $ENV{SKA}/bin/task1.pl 2 exec 4 : task1.pl 3 timeout 100 context 1 diff --git a/task_schedule3.pl b/task_schedule3.pl index 4c3dffe..21102a3 100755 --- a/task_schedule3.pl +++ b/task_schedule3.pl @@ -2,7 +2,7 @@ ##*************************************************************************** # Schedule a set a tasks -# +# # Author: T. Aldcroft # Created: 28-Dec-04 ##*************************************************************************** @@ -25,13 +25,17 @@ use Mail::Send; use Ska::Process qw(send_mail); - + ##*************************************************************************** ## Some initialization ##*************************************************************************** $| = 1; +if (not defined $ENV{SKA_ARCH_OS}) { + chomp($ENV{SKA_ARCH_OS} = `python -c 'import sys; print(sys.prefix)'`); +} + ##*************************************************************************** ## Get config and cmd line options @@ -41,7 +45,7 @@ my $hostname = Sys::Hostname::hostname; my $start_time = time(); -our %opt = (master_heart_attack => "$ENV{SKA_DATA}/${task}/master_heart_attack", +our %opt = (master_heart_attack => "$ENV{SKA}/data/${task}/master_heart_attack", heartbeat => 'task_sched_heartbeat', heart_attack => 'task_sched_heart_attack', no_master_heart_attack => 'task_sched_no_master_heart_attack', @@ -82,7 +86,7 @@ ); ##*************************************************************************** -## Interpolate (safely) some of the options to allow for generalized paths +## Interpolate (safely) some of the options to allow for generalized paths ## based on environment vars. Prepend default paths for bin and log directories ## if required. ##*************************************************************************** @@ -107,10 +111,14 @@ $_->{cmd} = $safe->reval(qq/"$_->{cmd}"/); # If (after interpolation) the exec isn't an absolute path - # and there is a bin_dir defined, then prepend that to path + # and there is a bin_dir defined and there is an executable + # at bin_dir/cmd, then prepend that to path if (not $_->{cmd} =~ m|\A \s* /|x and $opt{bin_dir}) { - $_->{cmd} = "$opt{bin_dir}/$_->{cmd}"; - } + my ($bin_dir_cmd, @cmd_args) = split(" ", "$opt{bin_dir}/$_->{cmd}"); + if (-e $bin_dir_cmd and -x $bin_dir_cmd) { + $_->{cmd} = $bin_dir_cmd; + } + } } # Do the same for the log file, except that a value of undef @@ -192,7 +200,7 @@ ## Fork so each job is launched in a separate process which waits until ## the job finishes or a timeout alarm expires. The parent just schedules ## the next execution event for the job. -## Touch a "heartbeat" file each minute +## Touch a "heartbeat" file each minute ##*************************************************************************** $SIG{CHLD} = 'IGNORE'; # Avoid zombies from dead children @@ -340,7 +348,7 @@ sub check_outputs { # 1 0 1 -email # 1 0 0 -noemail my $email_flag = ($opt{email} and (not $opt{disable_alerts} or not -e $opt{disable_alerts})) - ? '-email' + ? '-email' : '-noemail'; my $print_error_flag = $opt{print_error} ? '-printerror' : ''; @@ -448,7 +456,7 @@ sub run { ($cmd_root) = split ' ', $cmd->{cmd}; dbg "Running '$cmd->{cmd}' $cmd->{count} $cmd->{repeat_count}"; $cmd_pid = open CMD, "$cmd->{cmd} 2>&1 |" or die "ERROR - Could not start $cmd_root command: $!\n"; - + my $exec_out = ''; while () { $exec_out .= $_; @@ -483,7 +491,7 @@ sub run { my $warning = "WARNING - $cmd_root command timed out ".localtime()."\n"; dbg $warning; print $LOG_FH $warning; - + # Kill the cmd process. See Ska::Process for a more detailed routine that # finds all child processes with same group pid and kills them one at a time. # This is a bad idea here because there may be other "cron" processes with @@ -554,7 +562,7 @@ =head1 OPTIONS =item B<-config > This option is mandatory and gives the name of a file containing the -task scheduler configuration. This file specifies the jobs to be +task scheduler configuration. This file specifies the jobs to be run, email addresses for alerts, and all other program options. The test config file (t/data/test.config) has further documentation. @@ -564,7 +572,7 @@ =head1 OPTIONS =item B<-no-email> -Print error alerts but do not actually send emails. +Print error alerts but do not actually send emails. =item B<-fast