From e14e6167e227cbd0607a061749d8538ba6e33236 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 29 Dec 2022 20:19:42 -0500 Subject: [PATCH 01/30] Apparently working w/ time conversions --- starcheck/server.py | 64 ++++++++++ starcheck/src/lib/Ska/Parse_CM_File.pm | 142 +++++++++++----------- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 3 +- starcheck/src/lib/Ska/Starcheck/Python.pm | 84 +++++++++++++ starcheck/src/starcheck.pl | 43 ++++++- starcheck/utils.py | 36 +++++- 6 files changed, 291 insertions(+), 81 deletions(-) create mode 100644 starcheck/server.py create mode 100644 starcheck/src/lib/Ska/Starcheck/Python.pm diff --git a/starcheck/server.py b/starcheck/server.py new file mode 100644 index 00000000..00512be0 --- /dev/null +++ b/starcheck/server.py @@ -0,0 +1,64 @@ +import importlib +import json +import socketserver +import traceback + + +PORT = 44123 +HOST = "localhost" + + +class MyTCPHandler(socketserver.StreamRequestHandler): + """ + The request handler class for our server. + + It is instantiated once per connection to the server, and must + override the handle() method to implement communication to the + client. + """ + + def handle(self): + # self.request is the TCP socket connected to the client + data = self.rfile.readline() + # print(f"SERVER receive: {data.decode('utf-8')}") + + # Decode self.data from JSON + cmd = json.loads(data) + # print(f"SERVER receive func: {cmd['func']}") + # print(f"SERVER receive args: {cmd['args']}") + # print(f"SERVER receive kwargs: {cmd['kwargs']}") + + # For security reasons, only allow functions in the public API of starcheck module + parts = cmd["func"].split(".") + package = '.'.join(['starcheck'] + parts[:-1]) + func = parts[-1] + module = importlib.import_module(package) + func = getattr(module, func) + args = cmd["args"] + kwargs = cmd["kwargs"] + + try: + result = func(*args, **kwargs) + except Exception: + result = None + exc = traceback.format_exc() + else: + exc = None + + resp = json.dumps({"result": result, "exception": exc}) + # print(f"SERVER send: {resp}") + + self.request.sendall(resp.encode("utf-8")) + + +def main(): + # Create the server, binding to localhost on port 9999 + with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server: + # Activate the server; this will keep running until you + # interrupt the program with Ctrl-C + server.serve_forever() + + +if __name__ == "__main__": + print("SERVER: starting on port", PORT) + main() diff --git a/starcheck/src/lib/Ska/Parse_CM_File.pm b/starcheck/src/lib/Ska/Parse_CM_File.pm index b8b9d937..2fec66c5 100644 --- a/starcheck/src/lib/Ska/Parse_CM_File.pm +++ b/starcheck/src/lib/Ska/Parse_CM_File.pm @@ -3,7 +3,7 @@ package Ska::Parse_CM_File; ############################################################### # -# Parse one of several types of files produced in OFLS +# Parse one of several types of files produced in OFLS # command management # # Part of the starcheck cvs project @@ -11,18 +11,14 @@ package Ska::Parse_CM_File; ############################################################### use strict; -use warnings; +use warnings; use POSIX qw( ceil); use IO::All; use Carp; +use Ska::Starcheck::Python qw(date2time time2date); -use Inline Python => q{ - -from starcheck.utils import date2time, time2date - -}; my $VERSION = '$Id$'; # ' 1; @@ -69,11 +65,11 @@ sub TLR_load_segments{ } - + return @segment_times; } - + ############################################################### @@ -112,15 +108,19 @@ sub dither { # 2002262.095427395 | ENDITH AOENDITH my $dith_hist_fh = IO::File->new($dh_file, "r") or $dither_error = "Dither history file read err"; - while (<$dith_hist_fh>) { + my $date_start = time2date($bs_arr->[0]->{time} - 60 * 86400); + while (<$dith_hist_fh>) { if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d) \d* \s+ \| \s+ (ENDITH|DSDITH)/x) { - my ($yr, $doy, $hr, $min, $sec, $state) = ($1,$2,$3,$4,$5,$6); - my $time = date2time("$yr:$doy:$hr:$min:$sec"); - push @dh_state, $dith_enab_cmd_map{$state}; - push @dh_time, $time; - push @dh_params, {}; - } - } + my ($yr, $doy, $hr, $min, $sec, $state) = ($1,$2,$3,$4,$5,$6); + my $date = "$yr:$doy:$hr:$min:$sec"; + if ($date gt $date_start) { + my $time = date2time($date); + push @dh_state, $dith_enab_cmd_map{$state}; + push @dh_time, $time; + push @dh_params, {}; + } + } + } $dith_hist_fh->close(); if (not defined $dither_error){ @@ -201,7 +201,7 @@ sub dither { ############################################################### -sub radmon { +sub radmon { ############################################################### my $h_file = shift; # Radmon history file name my $bs_arr = shift; # Backstop array reference @@ -213,7 +213,7 @@ sub radmon { my @h_state; my @h_time; my @h_date; - my %cmd = ('DS' => 'DISA', + my %cmd = ('DS' => 'DISA', 'EN' => 'ENAB'); my %obs; # First get everything from backstop @@ -228,20 +228,24 @@ sub radmon { } } } - + # Now get everything from RADMON.txt # Parse lines like: # 2012222.011426269 | ENAB OORMPEN # 2012224.051225059 | DISA OORMPDS - my $hist_fh = IO::File->new($h_file, "r") or return (undef, undef); + my $hist_fh = IO::File->new($h_file, "r") or return (undef, undef); + my $date_start = time2date($bs_arr->[0]->{time} - 60 * 86400); while (<$hist_fh>) { - if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d) \d* \s+ \| \s+ (DISA|ENAB) \s+ (OORMPDS|OORMPEN)/x) { - my ($yr, $doy, $hr, $min, $sec, $state) = ($1,$2,$3,$4,$5,$6); - my $time = date2time("$yr:$doy:$hr:$min:$sec"); - my $date = "$yr:$doy:$hr:$min:$sec"; - push @h_date, $date; - push @h_state, $state; - push @h_time, $time; + if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d) \d* \s+ \| \s+ (DISA|ENAB) \s+ (OORMPDS|OORMPEN)/x) { + my ($yr, $doy, $hr, $min, $sec, $state) = ($1,$2,$3,$4,$5,$6); + my $date = "$yr:$doy:$hr:$min:$sec"; + if ($date gt $date_start) { + my $time = date2time($date); + my $date = "$yr:$doy:$hr:$min:$sec"; + push @h_date, $date; + push @h_state, $state; + push @h_time, $time; + } } } $hist_fh->close(); @@ -279,23 +283,23 @@ sub fidsel { my $bs = shift; # Reference to backstop array my $error = []; my %time_hash = (); # Hash of time stamps of fid cmds - + my @fs = (); foreach (0 .. 14) { $fs[$_] = []; } - + my ($actions, $times, $fid_time_violation) = get_fid_actions($fidsel_file, $bs); - + # Check for duplicate commanding map { $time_hash{$_}++ } @{$times}; # foreach (sort keys %time_hash) { # push @{$error}, "ERROR - $time_hash{$_} fid hardware commands at time $_\n" # if ($time_hash{$_} > 1); # } - + for (my $i = 0; $i <= $#{$times}; $i++) { - # If command contains RESET, then turn off (i.e. set tstop) any + # If command contains RESET, then turn off (i.e. set tstop) any # fid light that is on if ($actions->[$i] =~ /RESET/) { foreach my $fid (1 .. 14) { @@ -311,9 +315,9 @@ sub fidsel { push @{$error}, "Parse_cm_file::fidsel: WARNING - Could not parse $actions->[$i]"; } } - + return ($fid_time_violation, $error, \@fs); -} +} ############################################################### sub get_fid_actions { @@ -325,7 +329,7 @@ sub get_fid_actions { my @bs_time; my @fs_action; my @fs_time; - my $fidsel_fh; + my $fidsel_fh; # First get everything from backstop foreach $bs (@{$bs_arr}) { @@ -362,14 +366,14 @@ sub get_fid_actions { if ($action =~ /(RESET|FID.+ON)/) { # Convert to time, and subtract 10 seconds so that fid lights are # on slightly before end of manuever. In actual commanding, they - # come on about 1-2 seconds *after*. + # come on about 1-2 seconds *after*. push @fs_action, $action; push @fs_time, $time; } } } - + # $fidsel_fh->close(); } @@ -386,7 +390,7 @@ sub get_fid_actions { if ($fs_time[-1] >= $bs_arr->[0]->{time}){ $fid_time_violation = 1; } - + return (\@action, \@time, $fid_time_violation); } @@ -458,7 +462,7 @@ sub backstop { }; } close $BACKSTOP; - + return @bs; } @@ -531,32 +535,32 @@ sub guide{ # return hash that contains # target obsid, target dec, target ra, target roll # and an array of the lines of the catalog info - + my $guide_file = shift; my %guidesumm; # Let's slurp the file instead of reading it a line at a time my $whole_guide_file = io($guide_file)->slurp; - + # And then, let's split that file into chunks by processing request # By chunking I can guarantee that an error parsing the ID doesn't cause the # script to blindly overwrite the RA and DEC and keep adding to the starcat.. my @file_chunk = split /\n\n\n\*\*\*\* PROCESSING REQUEST \*\*\*\*\n/, $whole_guide_file; - + # Skip the first block in the file (which has no catalog) by using the index 1-end for my $chunk_number (1 .. $#file_chunk){ - + # Then, for each chunk, split into a line array my @file_chunk_lines = split /\n/, $file_chunk[$chunk_number]; # Now, since my loop is chunk by chunk, I can clear these for every chunk. my ($ra, $dec, $roll); my ($oflsid, $gsumid); - + foreach my $line (@file_chunk_lines){ - + # Look for an obsid, ra, dec, or roll if ($line =~ /\s+ID:\s+([[:ascii:]]{5})\s+\((\S{3,5})\)/) { my @field = ($1, $2); @@ -568,16 +572,16 @@ sub guide{ ($oflsid = $1) =~ s/^0*//; $oflsid =~ s/00$//; } - + # Skip the rest of the block for each line if # oflsid hasn't been found/defined - + next unless (defined $oflsid); - + if ($line =~ /\s+RA:\s*([^ ]+) DEG/){ $ra = $1; $guidesumm{$oflsid}{ra}=$ra; - } + } if ($line =~ /\s+DEC:\s*([^ ]+) DEG/){ $dec = $1; $guidesumm{$oflsid}{dec}=$dec; @@ -586,10 +590,10 @@ sub guide{ $roll = $1; $guidesumm{$oflsid}{roll}=$roll; } - + if ($line =~ /^(FID|ACQ|GUI|BOT)/) { push @{$guidesumm{$oflsid}{info}}, $line; - + } if ($line =~ /^MON/){ my @l= split ' ', $line; @@ -602,9 +606,9 @@ sub guide{ } } } - + return %guidesumm; - + } @@ -615,10 +619,10 @@ sub OR { my %or; my %obs; my $obs; - - + + open (my $OR, $or_file) || die "Couldn't open OR file $or_file\n"; - my $in_obs_statement = 0; + my $in_obs_statement = 0; while (<$OR>) { chomp; if ($in_obs_statement) { @@ -649,7 +653,7 @@ sub OR_parse_obs { my %obs = (); # print STDERR "In OR_Parse_obs \n"; foreach (@obs_columns) { - $obs{$_} = ''; + $obs{$_} = ''; } ($obs{TARGET_RA}, $obs{TARGET_DEC}) = (0.0, 0.0); ($obs{TARGET_OFFSET_Y}, $obs{TARGET_OFFSET_Z}) = (0.0, 0.0); @@ -687,7 +691,7 @@ sub OR_parse_obs { ############################################################### sub PS { # Parse processing summary -# Actually, just read in the juicy lines in the middle +# Actually, just read in the juicy lines in the middle # which are maneuvers or observations and store them # to a line array ############################################################### @@ -779,10 +783,10 @@ sub MM { $manvr_hash{tstop} = date2time($manvr_hash{stop_date}); # let's just add those 10 seconds to the summary tstart so it lines up with - # AOMANUVR in backstop + # AOMANUVR in backstop $manvr_hash{tstart} += $manvr_offset; $manvr_hash{start_date} = time2date($manvr_hash{tstart}); - + # clean up obsids (remove prepended 0s) if (defined $manvr_hash{initial_obsid}) { $manvr_hash{initial_obsid} =~ s/^0+//; @@ -806,7 +810,7 @@ sub MM { } # create a manvr_dest key to record the eventual destination of - # all manvrs. + # all manvrs. for my $i (0 .. $#mm_array){ # by default the destination is just the final_obsid $mm_array[$i]->{manvr_dest} = $mm_array[$i]->{final_obsid}; @@ -885,7 +889,7 @@ sub mechcheck { $evt{val} = $2; $evt{from}= $1; } - + push @mc, { %evt } if ($evt{var}); } close $MC; @@ -901,15 +905,15 @@ sub SOE { # read the SOE record formats into the hashes of lists $fld and $len my (%fld, %len, $obsidx); - my (%rlen, %templ); - while () { + my (%rlen, %templ); + while () { my ($rtype,$rfld,$rlen,$rdim1,$rdim2) = split; - for my $j (0 .. $rdim2-1) { + for my $j (0 .. $rdim2-1) { for my $i (0 .. $rdim1-1) { my $idx = ''; $idx .= "[$i]" if ($rdim1 > 1); $idx .= "[$j]" if ($rdim2 > 1); - push @{ $fld{$rtype} },$rfld.$idx; + push @{ $fld{$rtype} },$rfld.$idx; push @{ $len{$rtype} },$rlen; } } @@ -947,7 +951,7 @@ sub SOE { die "Parse_CM_File::SOE: Cannot identify record of type $typ\n "; } } - + return %SOE; } @@ -983,7 +987,7 @@ sub odb { close $ODB; return (%odb); -} +} diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 20803f20..f9b9d749 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -20,12 +20,13 @@ package Ska::Starcheck::Obsid; use strict; use warnings; +use Ska::Starcheck::Python qw(date2time time2date call_python); use Inline Python => q{ import numpy as np from astropy.table import Table -from starcheck.utils import time2date, date2time, de_bytestr +from starcheck.utils import de_bytestr from mica.archive import aca_dark from chandra_aca.star_probs import guide_count from chandra_aca.transform import (yagzag_to_pixels, pixels_to_yagzag, diff --git a/starcheck/src/lib/Ska/Starcheck/Python.pm b/starcheck/src/lib/Ska/Starcheck/Python.pm new file mode 100644 index 00000000..a66e3572 --- /dev/null +++ b/starcheck/src/lib/Ska/Starcheck/Python.pm @@ -0,0 +1,84 @@ +package Ska::Starcheck::Python; + +use strict; +use warnings; +use IO::Socket; +use JSON; +use Carp qw(confess); +use Data::Dumper; + +use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); +require Exporter; + +our @ISA = qw(Exporter); +our @EXPORT = qw(); +our @EXPORT_OK = qw(call_python date2time time2date); +%EXPORT_TAGS = ( all => \@EXPORT_OK ); + +STDOUT->autoflush(1); + +$Data::Dumper::Terse = 1; + +my $host = "localhost"; +my $port = 44123; + + +sub call_python { + my $func = shift; + my $args = shift; + my $kwargs = shift; + if (!defined $args) { $args = []; } + if (!defined $kwargs) { $kwargs = {}; } + + my $command = { + "func" => $func, + "args" => $args, + "kwargs" => $kwargs, + }; + my $command_json = encode_json $command; + # print "CLIENT: Sending command $command_json\n"; + + my $handle; + my $iter = 0; + while ($iter++ < 10) { + $handle = IO::Socket::INET->new(Proto => "tcp", + PeerAddr => $host, + PeerPort => $port); + last if defined($handle); + sleep 1; + } + if (!defined($handle)) { + die "Unable to connect to port $port on $host: $!"; + } + $handle->autoflush(1); # so output gets there right away + $handle->write("$command_json\n"); + + my $response = <$handle>; + $handle->close(); + + my $data = decode_json $response; + if (defined $data->{exception}) { + my $msg = "\nPython exception:\n"; + $msg .= "command = " . Dumper($command) . "\n"; + $msg .= "$data->{exception}\n"; + Carp::confess $msg; + } + + # print "CLIENT: Got result: $data->{result}\n"; + return $data->{result}; +} + + +sub date2time { + my $date = shift; + # print "date2time: $date\n"; + return call_python("utils.date2time", [$date]); +} + + +sub time2date { + my $time = shift; + # print "time2date: $time\n"; + return call_python("utils.time2date", [$time]); +} + diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index c9a20a19..efb5bcc0 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -24,6 +24,7 @@ use PoorTextFormat; use Ska::Starcheck::Obsid; +use Ska::Starcheck::Python qw(date2time time2date call_python); use Ska::Parse_CM_File; use Carp; use YAML; @@ -36,6 +37,16 @@ use Carp 'verbose'; $SIG{ __DIE__ } = sub { Carp::confess( @_ )}; +# Start a server that can run Python code +my $pid; +if ($pid = fork) {} else { + exec('python', '-m', 'starcheck.server'); +} + + +print "Starting starcheck.pl\n"; +print call_python("utils.time2date", [150000000.]) . "\n"; +print time2date(300000000.) . "\n"; use Inline Python => q{ @@ -45,7 +56,6 @@ from starcheck.pcad_att_check import check_characteristics_date from starcheck.utils import (_make_pcad_attitude_check_report, plot_cat_wrapper, - date2time, time2date, config_logging, set_kadi_scenario_default, ccd_temp_wrapper, @@ -211,6 +221,7 @@ # First read the Backstop file, and split into components +print "Reading backstop file $backstop\n"; my @bs = Ska::Parse_CM_File::backstop($backstop); my $i = 0; @@ -223,6 +234,7 @@ } # Read DOT, which is used to figure out the Obsid for each command +print "Reading DOT file $dot_file\n"; my ($dot_ref, $dot_touched_by_sausage) = Ska::Parse_CM_File::DOT($dot_file) if ($dot_file); my %dot = %{$dot_ref}; @@ -231,23 +243,29 @@ # print STDERR "$dotkey $dot{$dotkey}{cmd_identifier} $dot{$dotkey}{anon_param3} $dot{$dotkey}{anon_param4} \n"; #} +print "Reading TLR file $tlr_file\n"; my @load_segments = Ska::Parse_CM_File::TLR_load_segments($tlr_file); +print "Reading MM file $mm_file\n"; # Read momentum management (maneuvers + SIM move) summary file my %mm = Ska::Parse_CM_File::MM({file => $mm_file, ret_type => 'hash'}) if ($mm_file); # Read maneuver management summary for handy obsid time checks +print "Reading process summary $ps_file\n"; my @ps = Ska::Parse_CM_File::PS($ps_file) if ($ps_file); # Read mech check file and parse +print "Reading mech check file $mech_file\n"; my @mc = Ska::Parse_CM_File::mechcheck($mech_file) if ($mech_file); # Read OR file and integrate into %obs +print "Reading OR file $or_file\n"; my %or = Ska::Parse_CM_File::OR($or_file) if ($or_file); # Read FIDSEL (fid light) history file and ODB (for fid # characteristics) and parse; use fid_time_violation later (when global_warn set up +print "Reading FIDSEL file $fidsel_file\n"; my ($fid_time_violation, $error, $fidsel) = Ska::Parse_CM_File::fidsel($fidsel_file, \@bs) ; map { warning("$_\n") } @{$error}; @@ -271,6 +289,7 @@ Ska::Starcheck::Obsid::set_config($config_ref); # Read Maneuver error file containing more accurate maneuver errors +print "Reading Maneuver Error file $manerr_file\n"; my @manerr; if ($manerr_file) { @manerr = Ska::Parse_CM_File::man_err($manerr_file); @@ -282,11 +301,14 @@ # in review, and the RUNNING_LOAD_TERMINATION_TIME backstop "pseudo" command is available, that # command will be the first command ($bs[0]) and the kadi dither state will be fetched at that time. # This is expected and appropriate. +print "Getting dither state from kadi at $bs[0]->{date} \n"; my $kadi_dither = get_dither_kadi_state($bs[0]->{date}); # Read DITHER history file and backstop to determine expected dither state +print "Reading DITHER file $dither_file\n"; my ($dither_error, $dither) = Ska::Parse_CM_File::dither($dither_file, \@bs, $kadi_dither); +print "Reading RADMON file $radmon_file\n"; my ($radmon_time_violation, $radmon) = Ska::Parse_CM_File::radmon($radmon_file, \@bs); # if dither history runs into load or kadi mismatch @@ -339,6 +361,7 @@ my $obsid; my %obs; my @obsid_id; +my $n_obsid = 0; for my $i (0 .. $#cmd) { # Get obsid (aka ofls_id) for this cmd by matching up with corresponding # commands from DOT. Returns undef if it isn't "interesting" @@ -347,8 +370,9 @@ # If obsid hasn't been seen before, create obsid object unless ($obs{$obsid}) { - push @obsid_id, $obsid; - $obs{$obsid} = Ska::Starcheck::Obsid->new($obsid, $date[$i]); + push @obsid_id, $obsid; + $obs{$obsid} = Ska::Starcheck::Obsid->new($obsid, $date[$i]); + $n_obsid++; } # Add the command to the correct obs object @@ -358,6 +382,10 @@ date => $date[$i], time => $time[$i], cmd => $cmd[$i] } ); + + if ($n_obsid > 4) { + last; + } } # Read guide star summary file $guide_summ. This file is the OFLS summary of @@ -397,7 +425,7 @@ foreach my $oflsid (keys %guidesumm){ unless (defined $obs{$oflsid}){ - warning("OFLS ID $oflsid in Guide Summ but not in DOT! \n"); + #warning("OFLS ID $oflsid in Guide Summ but not in DOT! \n"); } } @@ -1109,8 +1137,11 @@ sub usage exit($exit) if ($exit); } - - +END { + print("Killing python server with pid=$pid\n"); + kill 9, $pid; # must it be 9 (SIGKILL)? + my $gone_pid = waitpid $pid, 0; # then check that it's gone +}; =pod diff --git a/starcheck/utils.py b/starcheck/utils.py index 3d42537a..35a624cc 100644 --- a/starcheck/utils.py +++ b/starcheck/utils.py @@ -6,7 +6,7 @@ import numpy as np import proseco.characteristics as proseco_char from Chandra.Time import DateTime -from Chandra.Time import date2secs, secs2date +import cxotime from kadi.commands import states from testr import test_helper @@ -18,6 +18,36 @@ from starcheck.plot import make_plots_for_obsid +def date2secs(val): + """Convert date to seconds since 1998.0""" + out = cxotime.date2secs(val) + # if isinstance(out, (np.number, np.ndarray)): + # out = out.tolist() + return out + + +def secs2date(val): + """Convert date to seconds since 1998.0""" + out = cxotime.secs2date(val) + # if isinstance(out, (np.number, np.ndarray)): + # out = out.tolist() + return out + + +def date2time(val): + out = cxotime.date2secs(val) + if isinstance(out, (np.number, np.ndarray)): + out = out.tolist() + return out + + +def time2date(val): + out = cxotime.secs2date(val) + if isinstance(out, (np.number, np.ndarray)): + out = out.tolist() + return out + + def python_from_perl(func): """Decorator to facilitate calling Python from Perl inline. @@ -52,10 +82,6 @@ def de_bytestr(data): return data -date2time = python_from_perl(date2secs) -time2date = python_from_perl(secs2date) - - @python_from_perl def ccd_temp_wrapper(kwargs): return get_ccd_temps(**kwargs) From c0b7b7bccf7e4af5cbe6d08cc7629c8c979f3361 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 29 Dec 2022 21:25:43 -0500 Subject: [PATCH 02/30] Vectorize date2time for backstop --- starcheck/src/lib/Ska/Parse_CM_File.pm | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/starcheck/src/lib/Ska/Parse_CM_File.pm b/starcheck/src/lib/Ska/Parse_CM_File.pm index 2fec66c5..5d2079f7 100644 --- a/starcheck/src/lib/Ska/Parse_CM_File.pm +++ b/starcheck/src/lib/Ska/Parse_CM_File.pm @@ -448,21 +448,27 @@ sub backstop { my $backstop = shift; my @bs = (); + my @dates = (); open (my $BACKSTOP, $backstop) || die "Couldn't open backstop file $backstop for reading\n"; while (<$BACKSTOP>) { - my ($date, $vcdu, $cmd, $params) = split '\s*\|\s*', $_; - $vcdu =~ s/ +.*//; # Get rid of second field in vcdu - my %command = parse_params($params); - push @bs, { date => $date, + my ($date, $vcdu, $cmd, $params) = split '\s*\|\s*', $_; + $vcdu =~ s/ +.*//; # Get rid of second field in vcdu + my %command = parse_params($params); + push @bs, { date => $date, vcdu => $vcdu, cmd => $cmd, params => $params, - time => date2time($date), command => \%command, }; + push @dates, $date; } close $BACKSTOP; + my $times = date2time(\@dates); + for (my $i = 0; $i <= $#bs; $i++) { + $bs[$i]->{time} = $times->[$i]; + } + return @bs; } From c552e4c6ba5789266414277290f9f0a89f87b091 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Dec 2022 06:11:48 -0500 Subject: [PATCH 03/30] Apply black/isort to utils.py --- starcheck/utils.py | 47 ++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/starcheck/utils.py b/starcheck/utils.py index 35a624cc..f4d0b166 100644 --- a/starcheck/utils.py +++ b/starcheck/utils.py @@ -3,10 +3,10 @@ import os from pathlib import Path +import cxotime import numpy as np import proseco.characteristics as proseco_char from Chandra.Time import DateTime -import cxotime from kadi.commands import states from testr import test_helper @@ -54,6 +54,7 @@ def python_from_perl(func): - Convert byte strings to unicode. - Print stack trace on exceptions which perl inline suppresses. """ + @functools.wraps(func) def wrapper(*args, **kwargs): args = de_bytestr(args) @@ -62,8 +63,10 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) except Exception: import traceback + traceback.print_exc() raise + return wrapper @@ -108,26 +111,26 @@ def set_kadi_scenario_default(): # This is aimed at running in production where the commands archive is # updated hourly. In this case no network resources are used. if test_helper.on_head_network(): - os.environ.setdefault('KADI_SCENARIO', 'flight') + os.environ.setdefault("KADI_SCENARIO", "flight") @python_from_perl def get_cheta_source(): sources = starcheck.calc_ccd_temps.fetch.data_source.sources() - if len(sources) == 1 and sources[0] == 'cxc': - return 'cxc' + if len(sources) == 1 and sources[0] == "cxc": + return "cxc" else: return str(sources) @python_from_perl def get_kadi_scenario(): - return os.getenv('KADI_SCENARIO', default="None") + return os.getenv("KADI_SCENARIO", default="None") @python_from_perl def get_data_dir(): - sc_data = os.path.join(os.path.dirname(starcheck.__file__), 'data') + sc_data = os.path.join(os.path.dirname(starcheck.__file__), "data") return sc_data if os.path.exists(sc_data) else "" @@ -143,15 +146,26 @@ def make_ir_check_report(kwargs): @python_from_perl def get_dither_kadi_state(date): - cols = ['dither', 'dither_ampl_pitch', 'dither_ampl_yaw', - 'dither_period_pitch', 'dither_period_yaw'] + cols = [ + "dither", + "dither_ampl_pitch", + "dither_ampl_yaw", + "dither_period_pitch", + "dither_period_yaw", + ] state = states.get_continuity(date, cols) # Cast the numpy floats as plain floats - for key in ['dither_ampl_pitch', 'dither_ampl_yaw', - 'dither_period_pitch', 'dither_period_yaw']: + for key in [ + "dither_ampl_pitch", + "dither_ampl_yaw", + "dither_period_pitch", + "dither_period_yaw", + ]: state[key] = float(state[key]) # get most recent change time - state['time'] = float(np.max([DateTime(state['__dates__'][key]).secs for key in cols])) + state["time"] = float( + np.max([DateTime(state["__dates__"][key]).secs for key in cols]) + ) return state @@ -204,12 +218,13 @@ def config_logging(outdir, verbose, name): class NullHandler(logging.Handler): def emit(self, record): pass + rootlogger = logging.getLogger() rootlogger.addHandler(NullHandler()) - loglevel = {0: logging.CRITICAL, - 1: logging.INFO, - 2: logging.DEBUG}.get(int(verbose), logging.INFO) + loglevel = {0: logging.CRITICAL, 1: logging.INFO, 2: logging.DEBUG}.get( + int(verbose), logging.INFO + ) logger = logging.getLogger(name) logger.setLevel(loglevel) @@ -218,7 +233,7 @@ def emit(self, record): for handler in list(logger.handlers): logger.removeHandler(handler) - formatter = logging.Formatter('%(message)s') + formatter = logging.Formatter("%(message)s") console = logging.StreamHandler() console.setFormatter(formatter) @@ -226,6 +241,6 @@ def emit(self, record): outdir = Path(outdir) outdir.mkdir(parents=True, exist_ok=True) - filehandler = logging.FileHandler(filename=outdir / 'run.dat', mode='w') + filehandler = logging.FileHandler(filename=outdir / "run.dat", mode="w") filehandler.setFormatter(formatter) logger.addHandler(filehandler) From d6450f4eb93afbfc016836f74ec55409fdda260a Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Dec 2022 09:47:55 -0500 Subject: [PATCH 04/30] Running to completion --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 331 ++------------------- starcheck/src/lib/Ska/Starcheck/Python.pm | 3 +- starcheck/src/starcheck.pl | 102 ++++--- starcheck/utils.py | 344 ++++++++++++++++++---- 4 files changed, 376 insertions(+), 404 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index f9b9d749..c8e26285 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -22,220 +22,13 @@ use strict; use warnings; use Ska::Starcheck::Python qw(date2time time2date call_python); -use Inline Python => q{ -import numpy as np -from astropy.table import Table - -from starcheck.utils import de_bytestr -from mica.archive import aca_dark -from chandra_aca.star_probs import guide_count -from chandra_aca.transform import (yagzag_to_pixels, pixels_to_yagzag, - count_rate_to_mag, mag_to_count_rate) -import Quaternion -from Ska.quatutil import radec2yagzag -import agasc - -from proseco.core import ACABox -from proseco.catalog import get_effective_t_ccd -from proseco.guide import get_imposter_mags -import proseco.characteristics as char - -import mica.stats.acq_stats -import mica.stats.guide_stats - -ACQS = mica.stats.acq_stats.get_stats() -GUIDES = mica.stats.guide_stats.get_stats() - -def _get_aca_limits(): - return float(char.aca_t_ccd_planning_limit), float(char.aca_t_ccd_penalty_limit) - -def _pixels_to_yagzag(i, j): - """ - Call chandra_aca.transform.pixels_to_yagzag. - This wrapper is set to pass allow_bad=True, as exceptions from the Python side - in this case would not be helpful, and the very small bad pixel list should be - on the CCD. - :params i: pixel row - :params j: pixel col - :returns tuple: yag, zag as floats - """ - yag, zag = pixels_to_yagzag(i, j, allow_bad=True) - return float(yag), float(zag) - - -def _yagzag_to_pixels(yag, zag): - """ - Call chandra_aca.transform.yagzag_to_pixels. - This wrapper is set to pass allow_bad=True, as exceptions from the Python side - in this case would not be helpful, and the boundary checks and such will work fine - on the Perl side even if the returned row/col is off the CCD. - :params yag: y-angle arcsecs (hopefully as a number from the Perl) - :params zag: z-angle arcsecs (hopefully as a number from the Perl) - :returns tuple: row, col as floats - """ - row, col = yagzag_to_pixels(yag, zag, allow_bad=True) - return float(row), float(col) - - -def _guide_count(mags, t_ccd, count_9th=False): - eff_t_ccd = get_effective_t_ccd(t_ccd) - return float(guide_count(np.array(mags), eff_t_ccd, count_9th)) - -def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z): - """ - Return a list of info to make warnings on guide stars or fid lights with local dark map that gives an - 'imposter_mag' that could perturb a centroid. The potential worst-case offsets (ignoring effects - at the background pixels) are returned and checking against offset limits needs to be done - from calling code. - - This fetches the dark current before the date of the observation and passes it to - proseco.get_imposter_mags with the star candidate positions to fetch the brightest - 2x2 for each and calculates the mag for that region. The worse case offset is then - added to an entry for the star index. - - :param idxs: catalog indexes as list or array - :param yags: catalog yangs as list or array - :param zags: catalog zangs as list or array - :param mags: catalog mags (AGASC mags for stars estimated fid mags for fids) list or array - :param types: catalog TYPE (ACQ|BOT|FID|MON|GUI) as list or array - :param t_ccd: observation t_ccd in deg C (should be max t_ccd in guide phase) - :param date: observation date (bytestring via Inline) - :param dither_y: dither_y in arcsecs (guide dither) - :param dither_z: dither_z in arcsecs (guide dither) - :param yellow_lim: yellow limit centroid offset threshold limit (in arcsecs) - :param red_lim: red limit centroid offset threshold limit (in arcsecs) - :return imposters: list of dictionaries with keys that define the index, the imposter mag, - a 'status' key that has value 0 if the code to get the imposter mag ran successfully, - calculated centroid offset, and star or fid info to make a warning. - """ - - types = [t.decode('ascii') for t in types] - date = date.decode('ascii') - eff_t_ccd = get_effective_t_ccd(t_ccd) - - dark = aca_dark.get_dark_cal_image(date=date, t_ccd_ref=eff_t_ccd, aca_image=True) - - - def imposter_offset(cand_mag, imposter_mag): - """ - For a given candidate star and the pseudomagnitude of the brightest 2x2 imposter - calculate the max offset of the imposter counts are at the edge of the 6x6 - (as if they were in one pixel). This is somewhat the inverse of proseco.get_pixmag_for_offset - """ - cand_counts = mag_to_count_rate(cand_mag) - spoil_counts = mag_to_count_rate(imposter_mag) - return spoil_counts * 3 * 5 / (spoil_counts + cand_counts) - - imposters = [] - for idx, yag, zag, mag, ctype in zip(idxs, yags, zags, mags, types): - if ctype in ['BOT', 'GUI', 'FID']: - if ctype in ['BOT', 'GUI']: - dither = ACABox((dither_y, dither_z)) - else: - dither = ACABox((5.0, 5.0)) - row, col = yagzag_to_pixels(yag, zag, allow_bad=True) - # Handle any errors in get_imposter_mags with a try/except. This doesn't - # try to pass back a message. Most likely this will only fail if the star - # or fid is completely off the CCD and will have other warning. - try: - # get_imposter_mags takes a Table of candidates as its first argument, so construct - # a single-candidate table `entries` - entries = Table([{'idx': idx, 'row': row, 'col': col, 'mag': mag, 'type': ctype}]) - imp_mags, imp_rows, imp_cols = get_imposter_mags(entries, dark, dither) - offset = imposter_offset(mag, imp_mags[0]) - imposters.append({'idx': int(idx), 'status': int(0), - 'entry_row': float(row), 'entry_col': float(col), - 'bad2_row': float(imp_rows[0]), 'bad2_col': float(imp_cols[0]), - 'bad2_mag': float(imp_mags[0]), 'offset': float(offset)}) - except: - imposters.append({'idx': int(idx), 'status': int(1)}) - return imposters - - -def _get_agasc_stars(ra, dec, roll, radius, date, agasc_file): - """ - Fetch the cone of agasc stars. Update the table with the yag and zag of each star. - Return as a dictionary with the agasc ids as keys and all of the values as - simple Python types (int, float) - """ - stars = agasc.get_agasc_cone(float(ra), float(dec), float(radius), date.decode('ascii'), - agasc_file.decode('ascii')) - q_aca = Quaternion.Quat([float(ra), float(dec), float(roll)]) - yags, zags = radec2yagzag(stars['RA_PMCORR'], stars['DEC_PMCORR'], q_aca) - yags *= 3600 - zags *= 3600 - stars['yang'] = yags - stars['zang'] = zags - - # Get a dictionary of the stars with the columns that are used - # This needs to be de-numpy-ified to pass back into Perl - stars_dict = {} - for star in stars: - stars_dict[str(star['AGASC_ID'])] = { - 'id': int(star['AGASC_ID']), - 'class': int(star['CLASS']), - 'ra': float(star['RA_PMCORR']), - 'dec': float(star['DEC_PMCORR']), - 'mag_aca': float(star['MAG_ACA']), - 'bv': float(star['COLOR1']), - 'color1': float(star['COLOR1']), - 'mag_aca_err': float(star['MAG_ACA_ERR']), - 'poserr': float(star['POS_ERR']), - 'yag': float(star['yang']), - 'zag': float(star['zang']), - 'aspq': int(star['ASPQ1']), - 'var': int(star['VAR']), - 'aspq1': int(star['ASPQ1'])} - - return stars_dict - - -def get_mica_star_stats(agasc_id, time): - """ - Get the acq and guide star statistics for a star before a given time. - The time filter is just there to make this play well when run in regression. - The mica acq and guide stats are fetched into globals ACQS and GUIDES - and this method just filters for the relevant ones for a star and returns - a dictionary of summarized statistics. - - :param agasc_id: agasc id of star - :param time: time used as end of range to retrieve statistics. - - :return: dictionary of stats for the observed history of the star - """ - - # Cast the inputs - time = float(time) - agasc_id = int(agasc_id) - - acqs = Table(ACQS[(ACQS['agasc_id'] == agasc_id) - & (ACQS['guide_tstart'] < time)]) - ok = acqs['img_func'] == 'star' - guides = Table(GUIDES[(GUIDES['agasc_id'] == agasc_id) - & (GUIDES['kalman_tstart'] < time)]) - mags = np.concatenate( - [acqs['mag_obs'][acqs['mag_obs'] != 0], - guides['aoacmag_mean'][guides['aoacmag_mean'] != 0]]) - - avg_mag = float(np.mean(mags)) if (len(mags) > 0) else float(13.94) - stats = {'acq': len(acqs), - 'acq_noid': int(np.count_nonzero(~ok)), - 'gui' : len(guides), - 'gui_bad': int(np.count_nonzero(guides['f_track'] < .95)), - 'gui_fail': int(np.count_nonzero(guides['f_track'] < .01)), - 'gui_obc_bad': int(np.count_nonzero(guides['f_obc_bad'] > .05)), - 'avg_mag': avg_mag} - return stats - -}; - - use List::Util qw(min max); use Quat; use File::Basename; use POSIX qw(floor); use English; use IO::All; +use Data::Dumper qw(Dumper); use RDB; @@ -332,7 +125,8 @@ sub set_config { # Set the ACA planning (red) and penalty (yellow) limits. Under the hood # this uses proseco to get the relevant values from the current release of # the ACA thermal model in chandra_models. - ($config{'ccd_temp_red_limit'}, $config{'ccd_temp_yellow_limit'}) = _get_aca_limits(); + my $vals = call_python("utils._get_aca_limits"); + ($config{'ccd_temp_red_limit'}, $config{'ccd_temp_yellow_limit'}) = @$vals; } @@ -356,7 +150,8 @@ sub set_ACA_bad_pixels { foreach my $j ($line[2]..$line[3]) { my $pixel = {'row' => $i, 'col' => $j}; - my ($yag,$zag) = _pixels_to_yagzag($i, $j); + my $call_vals = call_python("utils._pixels_to_yagzag", [$i, $j]); + my ($yag,$zag) = @$call_vals; $pixel->{yag} = $yag; $pixel->{zag} = $zag; push @bad_pixels, $pixel; @@ -1115,7 +910,8 @@ sub check_bright_perigee{ } # Pass 1 to _guide_count as third arg to use the count_9th mode - my $bright_count = sprintf("%.1f", _guide_count(\@mags, $self->{ccd_temp}, 1)); + my $bright_count = sprintf("%.1f", + call_python("utils._guide_count", [\@mags, $self->{ccd_temp}, 1])); if ($bright_count < $min_n_stars){ push @{$self->{warn}}, "$bright_count star(s) brighter than scaled 9th mag. " . "Perigee requires at least $min_n_stars\n"; @@ -1324,8 +1120,11 @@ sub check_star_catalog { } # Run the hot pixel region check on the Python side on FID|GUI|BOT - my @imposters = check_hot_pix(\@idxs, \@yags, \@zags, \@mags, \@types, - $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z); + my $call_vals = call_python( + "utils.check_hot_pix", + [\@idxs, \@yags, \@zags, \@mags, \@types, + $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z]); + my @imposters = @$call_vals; # Assign warnings based on those hot pixel region checks IMPOSTER: @@ -1489,7 +1288,8 @@ sub check_star_catalog { # Star/fid outside of CCD boundaries # ACA-019 ACA-020 ACA-021 - my ($pixel_row, $pixel_col) = _yagzag_to_pixels($yag, $zag); + my $call_vals = call_python("utils._yagzag_to_pixels", [$yag, $zag]); + my ($pixel_row, $pixel_col) = @$call_vals; # Set "acq phase" dither to acq dither or 20.0 if undefined my $dither_acq_y = $self->{dither_acq}->{ampl_y} or 20.0; @@ -2413,12 +2213,10 @@ sub get_agasc_stars { return unless ($c = find_command($self, "MP_TARGQUAT")); # Use Python agasc to fetch the stars into a hash - $self->{agasc_hash} = _get_agasc_stars($self->{ra}, - $self->{dec}, - $self->{roll}, - 1.3, - $self->{date}, - $agasc_file); + $self->{agasc_hash} = call_python( + "utils._get_agasc_stars", + [$self->{ra}, $self->{dec}, $self->{roll}, 1.3, $self->{date}, $agasc_file] + ); foreach my $star (values %{$self->{agasc_hash}}) { if ($star->{'mag_aca'} < -10 or $star->{'mag_aca_err'} < -10) { @@ -2543,7 +2341,7 @@ sub star_dbhist { my $obs_tstart_minus_day = $obs_tstart - 86400; - return get_mica_star_stats($star_id, $obs_tstart_minus_day); + return call_python("utils.get_mica_star_stats", [$star_id, $obs_tstart_minus_day]); } @@ -2588,7 +2386,8 @@ sub star_image_map { my $sid = $cat_star->{id}; my $yag = $cat_star->{yag}; my $zag = $cat_star->{zag}; - my ($pix_row, $pix_col) = _yagzag_to_pixels($yag, $zag); + my $call_vals = call_python("utils._yagzag_to_pixels", [$yag, $zag]); + my ($pix_row, $pix_col) = @$call_vals; my $image_x = 54 + ((2900 - $yag) * $pix_scale); my $image_y = 39 + ((2900 - $zag) * $pix_scale); my $star = '{ccd_temp})); + return sprintf("%.1f", call_python("utils._guide_count", [\@mags, $self->{ccd_temp}])); } @@ -2708,12 +2507,14 @@ sub set_ccd_temps{ # Add info for having a penalty temperature too if ($self->{ccd_temp} > $config{ccd_temp_yellow_limit}){ push @{$self->{fyi}}, sprintf("Effective guide temperature %.1f C\n", - get_effective_t_ccd($self->{ccd_temp})); + call_python("utils.get_effective_t_ccd", + [$self->{ccd_temp}])); } if ($self->{ccd_temp_acq} > $config{ccd_temp_yellow_limit}){ push @{$self->{fyi}}, sprintf("Effective acq temperature %.1f C\n", - get_effective_t_ccd($self->{ccd_temp_acq})); + call_python("utils.get_effective_t_ccd", + [$self->{ccd_temp_acq}])); } # Clip the acq ccd temperature to the calibrated range of the grid acq probability model @@ -2728,60 +2529,6 @@ sub set_ccd_temps{ } } -use Inline Python => q{ - -from proseco.catalog import get_aca_catalog - - -def proseco_probs(kwargs): - """ - Call proseco's get_acq_catalog with the parameters supplied in `kwargs` for a specific obsid catalog - and return the individual acq star probabilities, the P2 value for the catalog, and the expected - number of acq stars. - - `kwargs` will be a Perl hash converted to dict (by Inline) of the expected keyword params. These keys - must be defined: - - 'q1', 'q2', 'q3', 'q4' = the target quaternion - 'man_angle' the maneuver angle to the target quaternion in degrees. - 'acq_ids' list of acq star ids - 'halfwidths' list of acq star halfwidths in arcsecs - 't_ccd_acq' acquisition temperature in deg C - 'date' observation date (in Chandra.Time compatible format) - 'detector' science detector - 'sim_offset' SIM offset - - As these values are from a Perl hash, bytestrings will be converted by de_bytestr early in this method. - - :param kwargs: dict of expected keywords - :return tuple: (list of floats of star acq probabilties, float P2, float expected acq stars) - - """ - - kw = de_bytestr(kwargs) - args = dict(obsid=0, - att=Quaternion.normalize(kw['att']), - date=kw['date'], - n_acq=kw['n_acq'], - man_angle=kw['man_angle'], - t_ccd_acq=kw['t_ccd_acq'], - t_ccd_guide=kw['t_ccd_guide'], - dither_acq=ACABox(kw['dither_acq']), - dither_guide=ACABox(kw['dither_guide']), - include_ids_acq=kw['include_ids_acq'], - include_halfws_acq=kw['include_halfws_acq'], - detector=kw['detector'], sim_offset=kw['sim_offset'], - n_fid=0, n_guide=0, focus_offset=0) - aca = get_aca_catalog(**args) - acq_cat = aca.acqs - - # Assign the proseco probabilities back into an array. - p_acqs = [float(acq_cat['p_acq'][acq_cat['id'] == acq_id][0]) for acq_id in kw['include_ids_acq']] - - return p_acqs, float(-np.log10(acq_cat.calc_p_safe())), float(np.sum(p_acqs)) -}; - - ################################################################################### sub proseco_args{ ################################################################################### @@ -2891,7 +2638,8 @@ sub set_proseco_probs_and_check_P2{ if (not %{$args}){ return; } - my ($p_acqs, $P2, $expected) = proseco_probs($args); + my $call_vals = call_python("utils.proseco_probs", [], $args); + my ($p_acqs, $P2, $expected) = @$call_vals; $P2 = sprintf("%.1f", $P2); @@ -2933,23 +2681,6 @@ sub set_proseco_probs_and_check_P2{ } - -use Inline Python => q{ - -from chandra_aca.star_probs import mag_for_p_acq - -def _mag_for_p_acq(p_acq, date, t_ccd): - """ - Call mag_for_p_acq, but cast p_acq and t_ccd as floats (may or may not be needed) and - convert date from a bytestring (from the Perl interface). - """ - eff_t_ccd = get_effective_t_ccd(t_ccd) - return mag_for_p_acq(float(p_acq), date.decode(), float(eff_t_ccd)) - -}; - - - sub set_dynamic_mag_limits{ # Use the t_ccd at time of acquistion and time to set the mag limits corresponding to the the magnitude # for a 75% acquisition succes (yellow limit) and a 50% acquisition success (red limit) @@ -2961,7 +2692,9 @@ sub set_dynamic_mag_limits{ my $t_ccd = $self->{ccd_temp_acq}; # Dynamic mag limits based on 75% and 50% chance of successful star acq # Maximum limits of 10.3 and 10.6 - $self->{mag_faint_yellow} = min(10.3, _mag_for_p_acq(0.75, $date, $t_ccd)); - $self->{mag_faint_red} = min(10.6, _mag_for_p_acq(0.5, $date, $t_ccd)); + $self->{mag_faint_yellow} = min(10.3, call_python( + "utils._mag_for_p_acq", [0.75, $date, $t_ccd])); + $self->{mag_faint_red} = min(10.6, call_python( + "utils._mag_for_p_acq", [0.5, $date, $t_ccd])); } diff --git a/starcheck/src/lib/Ska/Starcheck/Python.pm b/starcheck/src/lib/Ska/Starcheck/Python.pm index a66e3572..cb6d13d1 100644 --- a/starcheck/src/lib/Ska/Starcheck/Python.pm +++ b/starcheck/src/lib/Ska/Starcheck/Python.pm @@ -57,6 +57,8 @@ sub call_python { $handle->close(); my $data = decode_json $response; + # print "CLIENT: Got response: $response\n"; + # print Dumper($data); if (defined $data->{exception}) { my $msg = "\nPython exception:\n"; $msg .= "command = " . Dumper($command) . "\n"; @@ -64,7 +66,6 @@ sub call_python { Carp::confess $msg; } - # print "CLIENT: Got result: $data->{result}\n"; return $data->{result}; } diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index efb5bcc0..547e77d5 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -48,29 +48,7 @@ print call_python("utils.time2date", [150000000.]) . "\n"; print time2date(300000000.) . "\n"; -use Inline Python => q{ - -import os -import traceback - -from starcheck.pcad_att_check import check_characteristics_date -from starcheck.utils import (_make_pcad_attitude_check_report, - plot_cat_wrapper, - config_logging, - set_kadi_scenario_default, - ccd_temp_wrapper, - starcheck_version, get_data_dir, - get_chandra_models_version, - get_dither_kadi_state, - get_run_start_time, - get_kadi_scenario, get_cheta_source, - make_ir_check_report) - -}; - - -my $version = starcheck_version(); - +my $version = call_python("utils.starcheck_version"); # Set some global vars with directory locations my $SKA = $ENV{SKA} || '/proj/sot/ska'; @@ -108,7 +86,7 @@ exit( 1 ); -my $Starcheck_Data = $par{sc_data} || get_data_dir(); +my $Starcheck_Data = $par{sc_data} || call_python("utils.get_data_dir"); my $STARCHECK = $par{out} || ($par{vehicle} ? 'v_starcheck' : 'starcheck'); @@ -127,8 +105,8 @@ # verbose=1) is too chatty for the default. Instead allow only verbose=0 # (CRITICAL) or verbose=2 (DEBUG). my $kadi_verbose = $par{verbose} eq '2' ? '2' : '0'; -config_logging($STARCHECK, $kadi_verbose, "kadi"); -set_kadi_scenario_default(); +call_python("utils.config_logging", [$STARCHECK, $kadi_verbose, "kadi"]); +call_python("utils.set_kadi_scenario_default"); # Find backstop, guide star summary, OR, and maneuver files. my %input_files = (); @@ -199,6 +177,8 @@ my $ACA_badpix_date; my $ACA_badpix_firstline = io($ACA_bad_pixel_file)->getline; +Ska::Starcheck::Obsid::set_config($config_ref); + if ($ACA_badpix_firstline =~ /Bad Pixel.*\d{7}\s+\d{7}\s+(\d{7}).*/ ){ $ACA_badpix_date = $1; print STDERR "Using ACABadPixel file from $ACA_badpix_date Dark Cal \n"; @@ -286,7 +266,6 @@ Ska::Starcheck::Obsid::set_odb(%odb); -Ska::Starcheck::Obsid::set_config($config_ref); # Read Maneuver error file containing more accurate maneuver errors print "Reading Maneuver Error file $manerr_file\n"; @@ -302,7 +281,7 @@ # command will be the first command ($bs[0]) and the kadi dither state will be fetched at that time. # This is expected and appropriate. print "Getting dither state from kadi at $bs[0]->{date} \n"; -my $kadi_dither = get_dither_kadi_state($bs[0]->{date}); +my $kadi_dither = call_python("utils.get_dither_kadi_state", [$bs[0]->{date}]); # Read DITHER history file and backstop to determine expected dither state print "Reading DITHER file $dither_file\n"; @@ -523,19 +502,27 @@ sub json_obsids{ # Set the thermal model run start time to be either the supplied # run_start_time or now. -my $run_start_time = get_run_start_time($par{run_start_time}, $bs[0]->{date}); +my $run_start_time = call_python( + "utils.get_run_start_time", + [$par{run_start_time}, $bs[0]->{date}] +); my $json_text = json_obsids(); my $obsid_temps; my $json_obsid_temps; -$json_obsid_temps = ccd_temp_wrapper({oflsdir=> $par{dir}, - outdir=>$STARCHECK, - json_obsids => $json_text, - orlist => $or_file, - run_start_time => $run_start_time, - verbose => $par{verbose}, - maude => $par{maude}, - }); +$json_obsid_temps = call_python( + "utils.ccd_temp_wrapper", + [], + { + oflsdir=> $par{dir}, + outdir=>$STARCHECK, + json_obsids => $json_text, + orlist => $or_file, + run_start_time => $run_start_time, + verbose => $par{verbose}, + maude => $par{maude}, + } +); # convert back from JSON outside $obsid_temps = JSON::from_json($json_obsid_temps); @@ -566,7 +553,7 @@ sub json_obsids{ starcat_time=>"$obs{$obsid}->{date}", duration=>($obs{$obsid}->{obs_tstop} - $obs{$obsid}->{obs_tstart}), outdir=>$STARCHECK, agasc_file=>$agasc_file); - plot_cat_wrapper(\%plot_args); + call_python("utils.plot_cat_wrapper", [], \%plot_args); $obs{$obsid}->{plot_file} = "$STARCHECK/stars_$obs{$obsid}->{obsid}.png"; $obs{$obsid}->{plot_field_file} = "$STARCHECK/star_view_$obs{$obsid}->{obsid}.png"; $obs{$obsid}->{compass_file} = "$STARCHECK/compass$obs{$obsid}->{obsid}.png"; @@ -606,14 +593,14 @@ sub json_obsids{ $out .= "------------ Starcheck $version -----------------\n"; $out .= " Run on $date by $ENV{USER} from $hostname\n"; $out .= " Configuration: Using AGASC at $agasc_file\n"; -my $chandra_models_version = get_chandra_models_version(); +my $chandra_models_version = call_python("utils.get_chandra_models_version"); $out .= " chandra_models version: $chandra_models_version\n"; -my $kadi_scenario = get_kadi_scenario(); +my $kadi_scenario = call_python("utils.get_kadi_scenario"); if ($kadi_scenario ne "flight") { $kadi_scenario = "${red_font_start}${kadi_scenario}${font_stop}"; } $out .= " Kadi scenario: $kadi_scenario\n"; -my $cheta_source = get_cheta_source(); +my $cheta_source = call_python("utils.get_cheta_source"); if ($cheta_source ne 'cxc'){ $cheta_source = "${red_font_start}${cheta_source}${font_stop}"; } @@ -659,9 +646,14 @@ sub json_obsids{ # Check for just NMAN during high IR Zone $out .= "------------ HIGH IR ZONE CHECK -----------------\n\n"; my $ir_report = "${STARCHECK}/high_ir_check.txt"; -my $ir_ok = make_ir_check_report({ - backstop_file=> $backstop, - out=> $ir_report}); +my $ir_ok = call_python( + "utils.make_ir_check_report", + [], + { + backstop_file=> $backstop, + out=> $ir_report + } +); if ($ir_ok){ $out .= "[OK] In NMAN during High IR Zones.\n"; } @@ -690,10 +682,19 @@ sub json_obsids{ } else{ my $att_report = "${STARCHECK}/pcad_att_check.txt"; - my $att_ok = _make_pcad_attitude_check_report({ - backstop_file=> $backstop, or_list_file=>$or_file, - simtrans_file=>$simtrans_file, simfocus_file=>$simfocus_file, attitude_file=>$attitude_file, - ofls_characteristics_file=>$char_file, out=>$att_report, dynamic_offsets_file=>$aimpoint_file}); + my $att_ok = call_python( + "utils._make_pcad_attitude_check_report", + [], + { + backstop_file=> $backstop, + or_list_file=>$or_file, + simtrans_file=>$simtrans_file, + simfocus_file=>$simfocus_file, + attitude_file=>$attitude_file, + ofls_characteristics_file=>$char_file, + out=>$att_report, + dynamic_offsets_file=>$aimpoint_file + }); if ($att_ok){ $out .= "[OK] Coordinates as expected.\n"; } @@ -703,7 +704,10 @@ sub json_obsids{ # Only check that characteristics file is less than 30 days old if backstop starts before 01-Aug-2016 my $CHAR_DATE_CHECK_BEFORE = '2016:214:00:00:00.000'; if ($bs[0]->{time} < date2time($CHAR_DATE_CHECK_BEFORE)){ - if (check_characteristics_date($char_file, $date[0])){ + if (call_python( + "pcad_att_check.check_characteristics_date", + [$char_file, $date[0]]) + ) { $out .= "[OK] Characteristics file newer than 30 days\n\n"; } else{ diff --git a/starcheck/utils.py b/starcheck/utils.py index f4d0b166..3d17883e 100644 --- a/starcheck/utils.py +++ b/starcheck/utils.py @@ -1,13 +1,25 @@ -import functools import logging import os from pathlib import Path +import agasc import cxotime +import mica.stats.acq_stats +import mica.stats.guide_stats import numpy as np import proseco.characteristics as proseco_char +import proseco.characteristics as char +import Quaternion +from astropy.table import Table from Chandra.Time import DateTime +from chandra_aca.star_probs import guide_count, mag_for_p_acq +from chandra_aca.transform import mag_to_count_rate, pixels_to_yagzag, yagzag_to_pixels from kadi.commands import states +from mica.archive import aca_dark +from proseco.catalog import get_aca_catalog, get_effective_t_ccd +from proseco.core import ACABox +from proseco.guide import get_imposter_mags +from Ska.quatutil import radec2yagzag from testr import test_helper import starcheck @@ -17,6 +29,9 @@ from starcheck.pcad_att_check import make_pcad_attitude_check_report from starcheck.plot import make_plots_for_obsid +ACQS = mica.stats.acq_stats.get_stats() +GUIDES = mica.stats.guide_stats.get_stats() + def date2secs(val): """Convert date to seconds since 1998.0""" @@ -48,64 +63,22 @@ def time2date(val): return out -def python_from_perl(func): - """Decorator to facilitate calling Python from Perl inline. - - - Convert byte strings to unicode. - - Print stack trace on exceptions which perl inline suppresses. - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - args = de_bytestr(args) - kwargs = de_bytestr(kwargs) - try: - return func(*args, **kwargs) - except Exception: - import traceback - - traceback.print_exc() - raise - - return wrapper - - -# Borrowed from https://stackoverflow.com/a/33160507 -def de_bytestr(data): - if isinstance(data, bytes): - return data.decode() - if isinstance(data, dict): - return dict(map(de_bytestr, data.items())) - if isinstance(data, tuple): - return tuple(map(de_bytestr, data)) - if isinstance(data, list): - return list(map(de_bytestr, data)) - if isinstance(data, set): - return set(map(de_bytestr, data)) - return data - - -@python_from_perl -def ccd_temp_wrapper(kwargs): +def ccd_temp_wrapper(**kwargs): return get_ccd_temps(**kwargs) -@python_from_perl -def plot_cat_wrapper(kwargs): +def plot_cat_wrapper(**kwargs): return make_plots_for_obsid(**kwargs) -@python_from_perl def starcheck_version(): return version -@python_from_perl def get_chandra_models_version(): return proseco_char.chandra_models_version -@python_from_perl def set_kadi_scenario_default(): # For kadi commands v2 running on HEAD set the default scenario to flight. # This is aimed at running in production where the commands archive is @@ -114,7 +87,6 @@ def set_kadi_scenario_default(): os.environ.setdefault("KADI_SCENARIO", "flight") -@python_from_perl def get_cheta_source(): sources = starcheck.calc_ccd_temps.fetch.data_source.sources() if len(sources) == 1 and sources[0] == "cxc": @@ -123,28 +95,23 @@ def get_cheta_source(): return str(sources) -@python_from_perl def get_kadi_scenario(): return os.getenv("KADI_SCENARIO", default="None") -@python_from_perl def get_data_dir(): sc_data = os.path.join(os.path.dirname(starcheck.__file__), "data") return sc_data if os.path.exists(sc_data) else "" -@python_from_perl -def _make_pcad_attitude_check_report(kwargs): +def _make_pcad_attitude_check_report(**kwargs): return make_pcad_attitude_check_report(**kwargs) -@python_from_perl -def make_ir_check_report(kwargs): +def make_ir_check_report(**kwargs): return ir_zone_ok(**kwargs) -@python_from_perl def get_dither_kadi_state(date): cols = [ "dither", @@ -169,7 +136,6 @@ def get_dither_kadi_state(date): return state -@python_from_perl def get_run_start_time(run_start_time, backstop_start): """ Determine a reasonable reference run start time based on the supplied @@ -206,7 +172,6 @@ def get_run_start_time(run_start_time, backstop_start): return ref_time.date -@python_from_perl def config_logging(outdir, verbose, name): """Set up file and console logger. See http://docs.python.org/library/logging.html @@ -244,3 +209,272 @@ def emit(self, record): filehandler = logging.FileHandler(filename=outdir / "run.dat", mode="w") filehandler.setFormatter(formatter) logger.addHandler(filehandler) + + +def _get_aca_limits(): + return float(char.aca_t_ccd_planning_limit), float(char.aca_t_ccd_penalty_limit) + + +def _pixels_to_yagzag(i, j): + """ + Call chandra_aca.transform.pixels_to_yagzag. + This wrapper is set to pass allow_bad=True, as exceptions from the Python side + in this case would not be helpful, and the very small bad pixel list should be + on the CCD. + :params i: pixel row + :params j: pixel col + :returns tuple: yag, zag as floats + """ + yag, zag = pixels_to_yagzag(i, j, allow_bad=True) + return float(yag), float(zag) + + +def _yagzag_to_pixels(yag, zag): + """ + Call chandra_aca.transform.yagzag_to_pixels. + This wrapper is set to pass allow_bad=True, as exceptions from the Python side + in this case would not be helpful, and the boundary checks and such will work fine + on the Perl side even if the returned row/col is off the CCD. + :params yag: y-angle arcsecs (hopefully as a number from the Perl) + :params zag: z-angle arcsecs (hopefully as a number from the Perl) + :returns tuple: row, col as floats + """ + row, col = yagzag_to_pixels(yag, zag, allow_bad=True) + return float(row), float(col) + + +def _guide_count(mags, t_ccd, count_9th=False): + eff_t_ccd = get_effective_t_ccd(t_ccd) + return float(guide_count(np.array(mags), eff_t_ccd, count_9th)) + + +def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z): + """ + Return a list of info to make warnings on guide stars or fid lights with + local dark map that gives an 'imposter_mag' that could perturb a centroid. + The potential worst-case offsets (ignoring effects at the background pixels) + are returned and checking against offset limits needs to be done from + calling code. + + This fetches the dark current before the date of the observation and passes + it to proseco.get_imposter_mags with the star candidate positions to fetch + the brightest 2x2 for each and calculates the mag for that region. The + worse case offset is then added to an entry for the star index. + + :param idxs: catalog indexes as list or array + :param yags: catalog yangs as list or array + :param zags: catalog zangs as list or array + :param mags: catalog mags (AGASC mags for stars estimated fid mags for fids) + list or array + :param types: catalog TYPE (ACQ|BOT|FID|MON|GUI) as list or array + :param t_ccd: observation t_ccd in deg C (should be max t_ccd in guide + phase) + :param date: observation date (str) + :param dither_y: dither_y in arcsecs (guide dither) + :param dither_z: dither_z in arcsecs (guide dither) + :param yellow_lim: yellow limit centroid offset threshold limit (in arcsecs) + :param red_lim: red limit centroid offset threshold limit (in arcsecs) + :return imposters: list of dictionaries with keys that define the index, the + imposter mag, a 'status' key that has value 0 if the code to get + the imposter mag ran successfully, calculated centroid offset, and + star or fid info to make a warning. + """ + + eff_t_ccd = get_effective_t_ccd(t_ccd) + + dark = aca_dark.get_dark_cal_image(date=date, t_ccd_ref=eff_t_ccd, aca_image=True) + + def imposter_offset(cand_mag, imposter_mag): + """ + For a given candidate star and the pseudomagnitude of the brightest 2x2 + imposter calculate the max offset of the imposter counts are at the edge + of the 6x6 (as if they were in one pixel). This is somewhat the inverse + of proseco.get_pixmag_for_offset + """ + cand_counts = mag_to_count_rate(cand_mag) + spoil_counts = mag_to_count_rate(imposter_mag) + return spoil_counts * 3 * 5 / (spoil_counts + cand_counts) + + imposters = [] + for idx, yag, zag, mag, ctype in zip(idxs, yags, zags, mags, types): + if ctype in ["BOT", "GUI", "FID"]: + if ctype in ["BOT", "GUI"]: + dither = ACABox((dither_y, dither_z)) + else: + dither = ACABox((5.0, 5.0)) + row, col = yagzag_to_pixels(yag, zag, allow_bad=True) + # Handle any errors in get_imposter_mags with a try/except. This doesn't + # try to pass back a message. Most likely this will only fail if the star + # or fid is completely off the CCD and will have other warning. + try: + # get_imposter_mags takes a Table of candidates as its first argument, so construct + # a single-candidate table `entries` + entries = Table( + [{"idx": idx, "row": row, "col": col, "mag": mag, "type": ctype}] + ) + imp_mags, imp_rows, imp_cols = get_imposter_mags(entries, dark, dither) + offset = imposter_offset(mag, imp_mags[0]) + imposters.append( + { + "idx": int(idx), + "status": int(0), + "entry_row": float(row), + "entry_col": float(col), + "bad2_row": float(imp_rows[0]), + "bad2_col": float(imp_cols[0]), + "bad2_mag": float(imp_mags[0]), + "offset": float(offset), + } + ) + except Exception: + imposters.append({"idx": int(idx), "status": int(1)}) + return imposters + + +def _get_agasc_stars(ra, dec, roll, radius, date, agasc_file): + """ + Fetch the cone of agasc stars. Update the table with the yag and zag of each star. + Return as a dictionary with the agasc ids as keys and all of the values as + simple Python types (int, float) + """ + stars = agasc.get_agasc_cone( + float(ra), + float(dec), + float(radius), + date, + agasc_file, + ) + q_aca = Quaternion.Quat([float(ra), float(dec), float(roll)]) + yags, zags = radec2yagzag(stars["RA_PMCORR"], stars["DEC_PMCORR"], q_aca) + yags *= 3600 + zags *= 3600 + stars["yang"] = yags + stars["zang"] = zags + + # Get a dictionary of the stars with the columns that are used + # This needs to be de-numpy-ified to pass back into Perl + stars_dict = {} + for star in stars: + stars_dict[str(star["AGASC_ID"])] = { + "id": int(star["AGASC_ID"]), + "class": int(star["CLASS"]), + "ra": float(star["RA_PMCORR"]), + "dec": float(star["DEC_PMCORR"]), + "mag_aca": float(star["MAG_ACA"]), + "bv": float(star["COLOR1"]), + "color1": float(star["COLOR1"]), + "mag_aca_err": float(star["MAG_ACA_ERR"]), + "poserr": float(star["POS_ERR"]), + "yag": float(star["yang"]), + "zag": float(star["zang"]), + "aspq": int(star["ASPQ1"]), + "var": int(star["VAR"]), + "aspq1": int(star["ASPQ1"]), + } + + return stars_dict + + +def get_mica_star_stats(agasc_id, time): + """ + Get the acq and guide star statistics for a star before a given time. + The time filter is just there to make this play well when run in regression. + The mica acq and guide stats are fetched into globals ACQS and GUIDES + and this method just filters for the relevant ones for a star and returns + a dictionary of summarized statistics. + + :param agasc_id: agasc id of star + :param time: time used as end of range to retrieve statistics. + + :return: dictionary of stats for the observed history of the star + """ + + # Cast the inputs + time = float(time) + agasc_id = int(agasc_id) + + acqs = Table(ACQS[(ACQS["agasc_id"] == agasc_id) & (ACQS["guide_tstart"] < time)]) + ok = acqs["img_func"] == "star" + guides = Table( + GUIDES[(GUIDES["agasc_id"] == agasc_id) & (GUIDES["kalman_tstart"] < time)] + ) + mags = np.concatenate( + [ + acqs["mag_obs"][acqs["mag_obs"] != 0], + guides["aoacmag_mean"][guides["aoacmag_mean"] != 0], + ] + ) + + avg_mag = float(np.mean(mags)) if (len(mags) > 0) else float(13.94) + stats = { + "acq": len(acqs), + "acq_noid": int(np.count_nonzero(~ok)), + "gui": len(guides), + "gui_bad": int(np.count_nonzero(guides["f_track"] < 0.95)), + "gui_fail": int(np.count_nonzero(guides["f_track"] < 0.01)), + "gui_obc_bad": int(np.count_nonzero(guides["f_obc_bad"] > 0.05)), + "avg_mag": avg_mag, + } + return stats + + +def _mag_for_p_acq(p_acq, date, t_ccd): + """ + Call mag_for_p_acq, but cast p_acq and t_ccd as floats. + """ + eff_t_ccd = get_effective_t_ccd(t_ccd) + return mag_for_p_acq(float(p_acq), date, float(eff_t_ccd)) + + +def proseco_probs(**kw): + """ + Call proseco's get_acq_catalog with the parameters supplied in `kwargs` for + a specific obsid catalog and return the individual acq star probabilities, + the P2 value for the catalog, and the expected number of acq stars. + + `kwargs` will be a Perl hash converted to dict (by Inline) of the expected + keyword params. These keys must be defined: + + 'q1', 'q2', 'q3', 'q4' = the target quaternion 'man_angle' the maneuver + angle to the target quaternion in degrees. 'acq_ids' list of acq star ids + 'halfwidths' list of acq star halfwidths in arcsecs 't_ccd_acq' acquisition + temperature in deg C 'date' observation date (in Chandra.Time compatible + format) 'detector' science detector 'sim_offset' SIM offset + + As these values are from a Perl hash, bytestrings will be converted by + de_bytestr early in this method. + + :param **kw: dict of expected keywords + :return tuple: (list of floats of star acq probabilties, float P2, float + expected acq stars) + + """ + + args = dict( + obsid=0, + att=Quaternion.normalize(kw["att"]), + date=kw["date"], + n_acq=kw["n_acq"], + man_angle=kw["man_angle"], + t_ccd_acq=kw["t_ccd_acq"], + t_ccd_guide=kw["t_ccd_guide"], + dither_acq=ACABox(kw["dither_acq"]), + dither_guide=ACABox(kw["dither_guide"]), + include_ids_acq=kw["include_ids_acq"], + include_halfws_acq=kw["include_halfws_acq"], + detector=kw["detector"], + sim_offset=kw["sim_offset"], + n_fid=0, + n_guide=0, + focus_offset=0, + ) + aca = get_aca_catalog(**args) + acq_cat = aca.acqs + + # Assign the proseco probabilities back into an array. + p_acqs = [ + float(acq_cat["p_acq"][acq_cat["id"] == acq_id][0]) + for acq_id in kw["include_ids_acq"] + ] + + return p_acqs, float(-np.log10(acq_cat.calc_p_safe())), float(np.sum(p_acqs)) From a795acc3a9b6ca8906825037c605896ad93941a2 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Dec 2022 09:50:14 -0500 Subject: [PATCH 05/30] Remove final INLINE refs (env vars) --- sandbox_starcheck | 6 ------ starcheck/src/starcheck | 6 ------ starcheck/utils.py | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/sandbox_starcheck b/sandbox_starcheck index 89b9c04b..a6c5b837 100755 --- a/sandbox_starcheck +++ b/sandbox_starcheck @@ -23,12 +23,6 @@ then exit 1 fi -PERL_INLINE_DIRECTORY=`mktemp -d -t starcheck_inline.XXXXXX` || exit 1 -export PERL_INLINE_DIRECTORY perl -I ${DIR}/starcheck/src/lib ${DIR}/starcheck/src/starcheck.pl "$@" SC_STATUS=$? -if [[ -d $PERL_INLINE_DIRECTORY && $PERL_INLINE_DIRECTORY =~ .*starcheck_inline.* ]]; -then - rm -r $PERL_INLINE_DIRECTORY -fi exit $SC_STATUS diff --git a/starcheck/src/starcheck b/starcheck/src/starcheck index a2a8f70b..bbc7cd1c 100755 --- a/starcheck/src/starcheck +++ b/starcheck/src/starcheck @@ -29,13 +29,7 @@ then exit 1 fi -PERL_INLINE_DIRECTORY=`mktemp -d -t starcheck_inline.XXXXXX` || exit 1 -export PERL_INLINE_DIRECTORY STARCHECK=`python -c 'import starcheck; print(starcheck.__path__[0])'` perl -I ${STARCHECK}/src/lib ${STARCHECK}/src/starcheck.pl "$@" SC_STATUS=$? -if [[ -d $PERL_INLINE_DIRECTORY && $PERL_INLINE_DIRECTORY =~ .*starcheck_inline.* ]]; -then - rm -r $PERL_INLINE_DIRECTORY -fi exit $SC_STATUS diff --git a/starcheck/utils.py b/starcheck/utils.py index 3d17883e..04d026ed 100644 --- a/starcheck/utils.py +++ b/starcheck/utils.py @@ -432,7 +432,7 @@ def proseco_probs(**kw): a specific obsid catalog and return the individual acq star probabilities, the P2 value for the catalog, and the expected number of acq stars. - `kwargs` will be a Perl hash converted to dict (by Inline) of the expected + `kwargs` will be a Perl hash converted to dict of the expected keyword params. These keys must be defined: 'q1', 'q2', 'q3', 'q4' = the target quaternion 'man_angle' the maneuver From dce71d88b6e82756ef3c050ad0686fcaed6d2272 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Dec 2022 09:56:33 -0500 Subject: [PATCH 06/30] Minor cleanup --- starcheck/src/starcheck.pl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 547e77d5..8aa2d612 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -43,10 +43,9 @@ exec('python', '-m', 'starcheck.server'); } - -print "Starting starcheck.pl\n"; -print call_python("utils.time2date", [150000000.]) . "\n"; -print time2date(300000000.) . "\n"; +# DEBUG, limit number of obsids. TODO make this a command line option. +# Set to undef for no limit. +my $MAX_OBSIDS = 8; my $version = call_python("utils.starcheck_version"); @@ -362,7 +361,7 @@ time => $time[$i], cmd => $cmd[$i] } ); - if ($n_obsid > 4) { + if (defined $MAX_OBSIDS and $n_obsid > $MAX_OBSIDS) { last; } } From 58c11e303def486d1258eac96b195143fbb2510e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Dec 2022 07:12:42 -0500 Subject: [PATCH 07/30] Get free port and secure comm w/ key --- starcheck/server.py | 24 +++++++++++---- starcheck/src/lib/Ska/Starcheck/Python.pm | 23 +++++++++++---- starcheck/src/starcheck.pl | 36 +++++++++++++++++++---- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/starcheck/server.py b/starcheck/server.py index 00512be0..e705560f 100644 --- a/starcheck/server.py +++ b/starcheck/server.py @@ -1,11 +1,11 @@ import importlib import json import socketserver +import sys import traceback - -PORT = 44123 HOST = "localhost" +KEY = None class MyTCPHandler(socketserver.StreamRequestHandler): @@ -24,13 +24,18 @@ def handle(self): # Decode self.data from JSON cmd = json.loads(data) + + if cmd["key"] != KEY: + print(f"SERVER: bad key {cmd['key']!r}") + return + # print(f"SERVER receive func: {cmd['func']}") # print(f"SERVER receive args: {cmd['args']}") # print(f"SERVER receive kwargs: {cmd['kwargs']}") # For security reasons, only allow functions in the public API of starcheck module parts = cmd["func"].split(".") - package = '.'.join(['starcheck'] + parts[:-1]) + package = ".".join(["starcheck"] + parts[:-1]) func = parts[-1] module = importlib.import_module(package) func = getattr(module, func) @@ -52,13 +57,20 @@ def handle(self): def main(): - # Create the server, binding to localhost on port 9999 - with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server: + global KEY + + # Read a line from STDIN + port = int(sys.stdin.readline().strip()) + KEY = sys.stdin.readline().strip() + + print("SERVER: starting on port", port) + + # Create the server, binding to localhost on supplied port + with socketserver.TCPServer((HOST, port), MyTCPHandler) as server: # Activate the server; this will keep running until you # interrupt the program with Ctrl-C server.serve_forever() if __name__ == "__main__": - print("SERVER: starting on port", PORT) main() diff --git a/starcheck/src/lib/Ska/Starcheck/Python.pm b/starcheck/src/lib/Ska/Starcheck/Python.pm index cb6d13d1..c7ec3b0e 100644 --- a/starcheck/src/lib/Ska/Starcheck/Python.pm +++ b/starcheck/src/lib/Ska/Starcheck/Python.pm @@ -12,16 +12,26 @@ require Exporter; our @ISA = qw(Exporter); our @EXPORT = qw(); -our @EXPORT_OK = qw(call_python date2time time2date); +our @EXPORT_OK = qw(call_python date2time time2date set_port set_key); %EXPORT_TAGS = ( all => \@EXPORT_OK ); STDOUT->autoflush(1); $Data::Dumper::Terse = 1; -my $host = "localhost"; -my $port = 44123; +my $HOST = "localhost"; +my $PORT = 40000; +my $KEY = "fff"; +sub set_port { + $PORT = shift; + print "CLIENT: Setting port to $PORT\n"; +} + +sub set_key { + $KEY = shift; + print "CLIENT: Setting key to $KEY\n"; +} sub call_python { my $func = shift; @@ -34,6 +44,7 @@ sub call_python { "func" => $func, "args" => $args, "kwargs" => $kwargs, + "key" => $KEY, }; my $command_json = encode_json $command; # print "CLIENT: Sending command $command_json\n"; @@ -42,13 +53,13 @@ sub call_python { my $iter = 0; while ($iter++ < 10) { $handle = IO::Socket::INET->new(Proto => "tcp", - PeerAddr => $host, - PeerPort => $port); + PeerAddr => $HOST, + PeerPort => $PORT); last if defined($handle); sleep 1; } if (!defined($handle)) { - die "Unable to connect to port $port on $host: $!"; + die "Unable to connect to port $PORT on $HOST: $!"; } $handle->autoflush(1); # so output gets there right away $handle->write("$command_json\n"); diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 8aa2d612..e9afe191 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -15,6 +15,7 @@ use Getopt::Long; use IO::File; use IO::All; +use IO::Socket::INET; use Sys::Hostname; use English; use File::Basename; @@ -33,14 +34,35 @@ use Cwd qw( abs_path ); use HTML::TableExtract; - use Carp 'verbose'; $SIG{ __DIE__ } = sub { Carp::confess( @_ )}; -# Start a server that can run Python code +my $sock = IO::Socket::INET->new( + LocalAddr => '', LocalPort => 0, Proto => 'tcp', Listen => 1); +my $server_port = $sock->sockport(); +close($sock); + +# Generate a 16-character random string of letters and numbers that gets used +# as a key to authenticate the client to the server. +my $server_key = join '', map +(0..9,'a'..'z','A'..'Z')[rand 62], 1..16; + +# Configure the Python interface +Ska::Starcheck::Python::set_port($server_port); +Ska::Starcheck::Python::set_key($server_key); + +# Global process ID for fork that gets set in the parent process. This is +# used to kill the forked process when the parent finishes. my $pid; + +# Start a server that can call functions in the starcheck package if ($pid = fork) {} else { - exec('python', '-m', 'starcheck.server'); + open(SERVER, "| python -m starcheck.server"); + SERVER->autoflush(1); + # Send the port and key to the server + print SERVER "$server_port\n"; + print SERVER "$server_key\n"; + # Forked process waits until it gets killed by the parent finishing + sleep; } # DEBUG, limit number of obsids. TODO make this a command line option. @@ -1141,9 +1163,11 @@ sub usage } END { - print("Killing python server with pid=$pid\n"); - kill 9, $pid; # must it be 9 (SIGKILL)? - my $gone_pid = waitpid $pid, 0; # then check that it's gone + if (defined $pid) { + print("Killing python server with pid=$pid\n"); + kill 9, $pid; # must it be 9 (SIGKILL)? + my $gone_pid = waitpid $pid, 0; # then check that it's gone + } }; =pod From b7aeb60dafdf27f22e9993e1c6b1aff05c30111a Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Dec 2022 09:34:27 -0500 Subject: [PATCH 08/30] Add functionality to collect call statistics --- starcheck/server.py | 36 ++++++++++++++++++++++-------------- starcheck/src/starcheck.pl | 7 ++++--- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/starcheck/server.py b/starcheck/server.py index e705560f..f7aeffeb 100644 --- a/starcheck/server.py +++ b/starcheck/server.py @@ -1,3 +1,4 @@ +import collections import importlib import json import socketserver @@ -7,6 +8,8 @@ HOST = "localhost" KEY = None +func_calls = collections.Counter() + class MyTCPHandler(socketserver.StreamRequestHandler): """ @@ -33,22 +36,27 @@ def handle(self): # print(f"SERVER receive args: {cmd['args']}") # print(f"SERVER receive kwargs: {cmd['kwargs']}") + exc = None + # For security reasons, only allow functions in the public API of starcheck module - parts = cmd["func"].split(".") - package = ".".join(["starcheck"] + parts[:-1]) - func = parts[-1] - module = importlib.import_module(package) - func = getattr(module, func) - args = cmd["args"] - kwargs = cmd["kwargs"] - - try: - result = func(*args, **kwargs) - except Exception: - result = None - exc = traceback.format_exc() + if cmd["func"] == "get_server_calls": + # Sort func calls by the items in the counter + result = dict(func_calls) else: - exc = None + func_calls[cmd["func"]] += 1 + parts = cmd["func"].split(".") + package = ".".join(["starcheck"] + parts[:-1]) + func = parts[-1] + module = importlib.import_module(package) + func = getattr(module, func) + args = cmd["args"] + kwargs = cmd["kwargs"] + + try: + result = func(*args, **kwargs) + except Exception: + result = None + exc = traceback.format_exc() resp = json.dumps({"result": result, "exception": exc}) # print(f"SERVER send: {resp}") diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index e9afe191..eaba05fe 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -106,12 +106,9 @@ ) || exit( 1 ); - my $Starcheck_Data = $par{sc_data} || call_python("utils.get_data_dir"); my $STARCHECK = $par{out} || ($par{vehicle} ? 'v_starcheck' : 'starcheck'); - - my $empty_font_start = qq{}; my $red_font_start = qq{}; my $orange_font_start = qq{}; @@ -1164,6 +1161,10 @@ sub usage END { if (defined $pid) { + my $server_calls = call_python("get_server_calls"); + # print the server_calls hash sorted by value in descending order + print("Python server calls:"); + print Dump($server_calls); print("Killing python server with pid=$pid\n"); kill 9, $pid; # must it be 9 (SIGKILL)? my $gone_pid = waitpid $pid, 0; # then check that it's gone From a4f57d91a3d0462124efd16441a69a4e2b5af76f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Dec 2022 10:16:48 -0500 Subject: [PATCH 09/30] Reduce some Python calls and cut unused processing --- starcheck/src/lib/Ska/Parse_CM_File.pm | 77 +++++++++++++------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/starcheck/src/lib/Ska/Parse_CM_File.pm b/starcheck/src/lib/Ska/Parse_CM_File.pm index 5d2079f7..07b3c85d 100644 --- a/starcheck/src/lib/Ska/Parse_CM_File.pm +++ b/starcheck/src/lib/Ska/Parse_CM_File.pm @@ -99,26 +99,25 @@ sub dither { 'DITPAR' => undef); my @dh_state; - my @dh_time; my @dh_params; - # First get everything from DITHER + # First get everything from DITHER history. # Parse lines like: # 2002262.094827395 | DSDITH AODSDITH # 2002262.095427395 | ENDITH AOENDITH + # Check that the dither history does not run into the load. This is the only thing + # that is checked for the dither history file, the values are not used. + # Starcheck v2.0: skip it? + # Read the last 10000 bytes in the $dh_file. my $dith_hist_fh = IO::File->new($dh_file, "r") or $dither_error = "Dither history file read err"; - my $date_start = time2date($bs_arr->[0]->{time} - 60 * 86400); + $dith_hist_fh->seek(-10000, 2); + my $dh_date; + my $dh_state; while (<$dith_hist_fh>) { if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d) \d* \s+ \| \s+ (ENDITH|DSDITH)/x) { - my ($yr, $doy, $hr, $min, $sec, $state) = ($1,$2,$3,$4,$5,$6); - my $date = "$yr:$doy:$hr:$min:$sec"; - if ($date gt $date_start) { - my $time = date2time($date); - push @dh_state, $dith_enab_cmd_map{$state}; - push @dh_time, $time; - push @dh_params, {}; - } + $dh_date = "$1:$2:$3:$4:$5"; + $dh_state = $6; } } $dith_hist_fh->close(); @@ -126,16 +125,17 @@ sub dither { if (not defined $dither_error){ # If the most recent/last entry in the dither file has a timestamp newer than # the first entry in the load, return string error and undef dither history. - if ($dh_time[-1] >= $bs_arr->[0]->{time}){ - $dither_error = "Dither history runs into load\n"; + if ($dh_date ge $bs_arr->[0]->{date}){ + $dither_error = "Dither history runs into load\n"; } # Confirm that last state matches kadi continuity ENAB/DISA. # Otherwise, return string error and undef dither history. - if ($kadi_dither->{'dither'} ne $dh_state[-1]){ + if ($kadi_dither->{'dither'} ne $dith_enab_cmd_map{$dh_state}){ $dither_error = "Dither status in kadi commands does not match DITHER history\n" - . sprintf("kadi '%s' ; History '%s' \n", - $kadi_dither->{'dither'}, $dh_state[-1]); + . sprintf("kadi '%s' ; History '%s' \n", + $kadi_dither->{'dither'}, + $dith_enab_cmd_map{$dh_state}); } } @@ -329,19 +329,20 @@ sub get_fid_actions { my @bs_time; my @fs_action; my @fs_time; + my @fs_date; my $fidsel_fh; # First get everything from backstop foreach $bs (@{$bs_arr}) { - if ($bs->{cmd} eq 'COMMAND_HW') { - my %params = %{$bs->{command}}; - if ($params{TLMSID} eq 'AFIDP') { - my $msid = $params{MSID}; - push @bs_action, "$msid FID $1 ON" if ($msid =~ /AFLC(\d+)/); - push @bs_action, "RESET" if ($msid =~ /AFLCRSET/); - push @bs_time, $bs->{time} - 10; # see comment below about timing - } - } + if ($bs->{cmd} eq 'COMMAND_HW') { + my %params = %{$bs->{command}}; + if ($params{TLMSID} eq 'AFIDP') { + my $msid = $params{MSID}; + push @bs_action, "$msid FID $1 ON" if ($msid =~ /AFLC(\d+)/); + push @bs_action, "RESET" if ($msid =~ /AFLCRSET/); + push @bs_time, $bs->{time} - 10; # see comment below about timing + } + } } # printf("first bs entry at %s, last entry at %s \n", $bs_time[0], $bs_time[-1]); @@ -358,22 +359,24 @@ sub get_fid_actions { @reduced_fidsel_text = @fidsel_text[($#fidsel_text-1000) ... $#fidsel_text]; } # if ($fs_file && ($fidsel_fh = new IO::File $fs_file, "r")) { - for my $fidsel_line (@reduced_fidsel_text){ + for my $fidsel_line (@reduced_fidsel_text) { if ($fidsel_line =~ /(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d)\S*\s+\|\s+(AFL.+)/) { - my ($yr, $doy, $hr, $min, $sec, $action) = ($1,$2,$3,$4,$5,$6); - my $time = date2time("$yr:$doy:$hr:$min:$sec") - 10; + my ($yr, $doy, $hr, $min, $sec, $action) = ($1,$2,$3,$4,$5,$6); - if ($action =~ /(RESET|FID.+ON)/) { - # Convert to time, and subtract 10 seconds so that fid lights are - # on slightly before end of manuever. In actual commanding, they - # come on about 1-2 seconds *after*. + if ($action =~ /(RESET|FID.+ON)/) { - push @fs_action, $action; - push @fs_time, $time; - } + push @fs_action, $action; + push @fs_date, "$yr:$doy:$hr:$min:$sec"; + } } } + # Convert to time, and subtract 10 seconds so that fid lights are on + # slightly before end of manuever. In actual commanding, they come on about + # 1-2 seconds *after*. + my $times = date2time(\@fs_date); + @fs_time = map { $_ - 10 } @{$times}; + # $fidsel_fh->close(); } @@ -512,8 +515,8 @@ sub DOT { %{$dot{$_}} = parse_params($command{$_}); $dot{$_}{time} = date2time($dot{$_}{TIME}) if ($dot{$_}{TIME}); - # MANSTART is in the dot as a "relative" time like "000:00:00:00.000", so just pass it - # to the rel_date2time routine designed to handle that. + # MANSTART is in the dot as a "relative" time like "000:00:00:00.000", so just pass it + # to the rel_date2time routine designed to handle that. $dot{$_}{time} += rel_date2time($dot{$_}{MANSTART}) if ($dot{$_}{TIME} && $dot{$_}{MANSTART}); $dot{$_}{cmd_identifier} = "$dot{$_}{anon_param1}_$dot{$_}{anon_param2}" if ($dot{$_}{anon_param1} and $dot{$_}{anon_param2}); From 38beb17c98398725e4c8cb25006cfc7f77cb94c0 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Dec 2022 10:30:23 -0500 Subject: [PATCH 10/30] No limit on number of obsids --- starcheck/src/starcheck.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index eaba05fe..18c65c11 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -67,7 +67,7 @@ # DEBUG, limit number of obsids. TODO make this a command line option. # Set to undef for no limit. -my $MAX_OBSIDS = 8; +my $MAX_OBSIDS = undef; my $version = call_python("utils.starcheck_version"); From 4ada1e6c328a828ffaef90ae56ec9493d3ff5ed8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 1 Jan 2023 05:35:38 -0500 Subject: [PATCH 11/30] Vectorize yagzag_to_pixels call --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 20 +++++++++++++++----- starcheck/utils.py | 6 ++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index c8e26285..f3951ea1 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -2351,7 +2351,8 @@ sub star_image_map { my $self = shift; my $c; return unless ($c = find_command($self, 'MP_STARCAT')); - return unless ((defined $self->{ra}) and (defined $self->{dec}) and (defined $self->{roll})); my $obsid = $self->{obsid}; + return unless ((defined $self->{ra}) and (defined $self->{dec}) and (defined $self->{roll})); + my $obsid = $self->{obsid}; # a hash of the agasc ids we want to plot my %plot_ids; @@ -2380,21 +2381,30 @@ sub star_image_map { # top left +54+39 # 2900x2900 my $pix_scale = 330 / (2900. * 2); + + # Convert all the yag/zags to pixel rows/cols + my @yags = map { $self->{agasc_hash}->{$_}->{yag} } keys %plot_ids; + my @zags = map { $self->{agasc_hash}->{$_}->{zag} } keys %plot_ids; + my $call_vals = call_python("utils._yagzag_to_pixels", [\@yags, \@zags]); + my ($pix_rows, $pix_cols) = @$call_vals; + my $map = " \n"; - for my $star_id (keys %plot_ids){ + my @star_ids = keys %plot_ids; + for my $idx (0 .. $#star_ids) { + my $star_id = $star_ids[$idx]; + my $pix_row = $pix_rows->[$idx]; + my $pix_col = $pix_cols->[$idx]; my $cat_star = $self->{agasc_hash}->{$star_id}; my $sid = $cat_star->{id}; my $yag = $cat_star->{yag}; my $zag = $cat_star->{zag}; - my $call_vals = call_python("utils._yagzag_to_pixels", [$yag, $zag]); - my ($pix_row, $pix_col) = @$call_vals; my $image_x = 54 + ((2900 - $yag) * $pix_scale); my $image_y = 39 + ((2900 - $zag) * $pix_scale); my $star = '" . sprintf("yag,zag=%.2f,%.2f
", $yag, $zag) - . "row,col=$pix_row,$pix_col
" + . sprintf("row,col=%.2f,%.2f
", $pix_row,$pix_col) . sprintf("mag_aca=%.2f
", $cat_star->{mag_aca}) . sprintf("mag_aca_err=%.2f
", $cat_star->{mag_aca_err} / 100.0) . sprintf("class=%s
", $cat_star->{class}) diff --git a/starcheck/utils.py b/starcheck/utils.py index 04d026ed..0a6276a5 100644 --- a/starcheck/utils.py +++ b/starcheck/utils.py @@ -226,7 +226,8 @@ def _pixels_to_yagzag(i, j): :returns tuple: yag, zag as floats """ yag, zag = pixels_to_yagzag(i, j, allow_bad=True) - return float(yag), float(zag) + # Convert to lists or floats to avoid numpy types which are not JSON serializable + return yag.tolist(), zag.tolist() def _yagzag_to_pixels(yag, zag): @@ -240,7 +241,8 @@ def _yagzag_to_pixels(yag, zag): :returns tuple: row, col as floats """ row, col = yagzag_to_pixels(yag, zag, allow_bad=True) - return float(row), float(col) + # Convert to lists or floats to avoid numpy types which are not JSON serializable + return row.tolist(), col.tolist() def _guide_count(mags, t_ccd, count_9th=False): From d6087bdb975d58ee3755f7deb327a5477954f1c3 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 1 Jan 2023 05:49:52 -0500 Subject: [PATCH 12/30] Don't call pixel_to_yagzag for bad pixels --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index f3951ea1..42176be1 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -145,18 +145,14 @@ sub set_ACA_bad_pixels { my @tmp = io($pixel_file)->slurp; my @lines = grep { /^\s+(\d|-)/ } @tmp; foreach (@lines) { - my @line = split /;|,/, $_; - foreach my $i ($line[0]..$line[1]) { - foreach my $j ($line[2]..$line[3]) { - my $pixel = {'row' => $i, + my @line = split /;|,/, $_; + foreach my $i ($line[0]..$line[1]) { + foreach my $j ($line[2]..$line[3]) { + my $pixel = {'row' => $i, 'col' => $j}; - my $call_vals = call_python("utils._pixels_to_yagzag", [$i, $j]); - my ($yag,$zag) = @$call_vals; - $pixel->{yag} = $yag; - $pixel->{zag} = $zag; - push @bad_pixels, $pixel; - } - } + push @bad_pixels, $pixel; + } + } } print STDERR "Read ", ($#bad_pixels+1), " ACA bad pixels from $pixel_file\n"; @@ -1445,8 +1441,8 @@ sub check_star_catalog { my @dr; if ($type =~ /GUI|BOT/){ foreach my $pixel (@bad_pixels) { - my $dy = abs($yag-$pixel->{yag}); - my $dz = abs($zag-$pixel->{zag}); + my $dy = abs($pixel_row-$pixel->{row}) * 5; + my $dz = abs($pixel_col-$pixel->{col}) * 5; my $dr = sqrt($dy**2 + $dz**2); next unless ($dz < $self->{dither_guide}->{ampl_p} + 25 and $dy < $self->{dither_guide}->{ampl_y} + 25); push @close_pixels, sprintf(" row, col (%d, %d), dy, dz (%d, %d) \n", From c95b9105c08d9c351433a2dbf94f8074c9c110dc Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sun, 1 Jan 2023 06:47:35 -0500 Subject: [PATCH 13/30] More performance and code tidy --- starcheck/src/lib/Ska/Parse_CM_File.pm | 77 +++++++++++------------- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 14 ++--- 2 files changed, 39 insertions(+), 52 deletions(-) diff --git a/starcheck/src/lib/Ska/Parse_CM_File.pm b/starcheck/src/lib/Ska/Parse_CM_File.pm index 07b3c85d..486bd3af 100644 --- a/starcheck/src/lib/Ska/Parse_CM_File.pm +++ b/starcheck/src/lib/Ska/Parse_CM_File.pm @@ -109,9 +109,9 @@ sub dither { # Check that the dither history does not run into the load. This is the only thing # that is checked for the dither history file, the values are not used. # Starcheck v2.0: skip it? - # Read the last 10000 bytes in the $dh_file. - my $dith_hist_fh = IO::File->new($dh_file, "r") or $dither_error = "Dither history file read err"; - $dith_hist_fh->seek(-10000, 2); + # Read the last 1000 bytes in the $dh_file (guaranteed to be enough). + my $dith_hist_fh = IO::File->new($dh_file, "r") or croak "Can't open $dh_file: $!"; + $dith_hist_fh->seek(-1000, 2); my $dh_date; my $dh_state; while (<$dith_hist_fh>) { @@ -233,22 +233,20 @@ sub radmon { # Parse lines like: # 2012222.011426269 | ENAB OORMPEN # 2012224.051225059 | DISA OORMPDS - my $hist_fh = IO::File->new($h_file, "r") or return (undef, undef); - my $date_start = time2date($bs_arr->[0]->{time} - 60 * 86400); + my $hist_fh = IO::File->new($h_file, "r") or croak "Can't open $h_file: $!"; + # Get the last 1000 characters. This is guaranteed to get the most recent + # entries that we care about. + $hist_fh->seek(-1000, 2); while (<$hist_fh>) { - if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d) \d* \s+ \| \s+ (DISA|ENAB) \s+ (OORMPDS|OORMPEN)/x) { + if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d) \d* \s+ \| \s+ (DISA|ENAB) \s+ (OORMPDS|OORMPEN)/x) { my ($yr, $doy, $hr, $min, $sec, $state) = ($1,$2,$3,$4,$5,$6); my $date = "$yr:$doy:$hr:$min:$sec"; - if ($date gt $date_start) { - my $time = date2time($date); - my $date = "$yr:$doy:$hr:$min:$sec"; - push @h_date, $date; - push @h_state, $state; - push @h_time, $time; - } + push @h_date, $date; + push @h_state, $state; } } $hist_fh->close(); + @h_time = @{date2time(\@h_date)}; my @ok = grep { $h_time[$_] < $bs_arr->[0]->{time} } (0 .. $#h_time); my @state = (@h_state[@ok], @bs_state); @@ -337,10 +335,10 @@ sub get_fid_actions { if ($bs->{cmd} eq 'COMMAND_HW') { my %params = %{$bs->{command}}; if ($params{TLMSID} eq 'AFIDP') { - my $msid = $params{MSID}; - push @bs_action, "$msid FID $1 ON" if ($msid =~ /AFLC(\d+)/); - push @bs_action, "RESET" if ($msid =~ /AFLCRSET/); - push @bs_time, $bs->{time} - 10; # see comment below about timing + my $msid = $params{MSID}; + push @bs_action, "$msid FID $1 ON" if ($msid =~ /AFLC(\d+)/); + push @bs_action, "RESET" if ($msid =~ /AFLCRSET/); + push @bs_time, $bs->{time} - 10; # see comment below about timing } } } @@ -352,32 +350,25 @@ sub get_fid_actions { # 2001211.190730558 | AFLCRSET RESET # 2001211.190731558 | AFLC02D1 FID 02 ON if (defined $fs_file){ - my @fidsel_text = io($fs_file)->slurp; - # take the last thousand entries if there are more than a 1000 - my @reduced_fidsel_text = @fidsel_text; - if ($#fidsel_text > 1000){ - @reduced_fidsel_text = @fidsel_text[($#fidsel_text-1000) ... $#fidsel_text]; - } -# if ($fs_file && ($fidsel_fh = new IO::File $fs_file, "r")) { - for my $fidsel_line (@reduced_fidsel_text) { - if ($fidsel_line =~ /(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d)\S*\s+\|\s+(AFL.+)/) { - my ($yr, $doy, $hr, $min, $sec, $action) = ($1,$2,$3,$4,$5,$6); - - if ($action =~ /(RESET|FID.+ON)/) { - - push @fs_action, $action; - push @fs_date, "$yr:$doy:$hr:$min:$sec"; + my $fh = IO::File->new($fs_file, "r") or croak "Can't open $fs_file: $!"; + # Last 1000 bytes of history file is sufficient + $fh->seek(-1000, 2); + while (<$fh>) { + if (/(\d\d\d\d)(\d\d\d)\.(\d\d)(\d\d)(\d\d)\S*\s+\|\s+(AFL.+)/) { + my ($yr, $doy, $hr, $min, $sec, $action) = ($1,$2,$3,$4,$5,$6); + if ($action =~ /(RESET|FID.+ON)/) { + push @fs_action, $action; + push @fs_date, "$yr:$doy:$hr:$min:$sec"; + } } - } - } - - # Convert to time, and subtract 10 seconds so that fid lights are on - # slightly before end of manuever. In actual commanding, they come on about - # 1-2 seconds *after*. - my $times = date2time(\@fs_date); - @fs_time = map { $_ - 10 } @{$times}; - -# $fidsel_fh->close(); + } + + # Convert to time, and subtract 10 seconds so that fid lights are on + # slightly before end of manuever. In actual commanding, they come on about + # 1-2 seconds *after*. + my $times = date2time(\@fs_date); + @fs_time = map { $_ - 10 } @{$times}; + $fh->close(); } # printf("count of fid entries is %s \n", scalar(@fs_time)); @@ -391,7 +382,7 @@ sub get_fid_actions { # if the fid history extends into the current load if ($fs_time[-1] >= $bs_arr->[0]->{time}){ - $fid_time_violation = 1; + $fid_time_violation = 1; } return (\@action, \@time, $fid_time_violation); diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 42176be1..8ca7e93c 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -1116,11 +1116,10 @@ sub check_star_catalog { } # Run the hot pixel region check on the Python side on FID|GUI|BOT - my $call_vals = call_python( + my @imposters = @{call_python( "utils.check_hot_pix", [\@idxs, \@yags, \@zags, \@mags, \@types, - $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z]); - my @imposters = @$call_vals; + $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z]);}; # Assign warnings based on those hot pixel region checks IMPOSTER: @@ -1284,8 +1283,7 @@ sub check_star_catalog { # Star/fid outside of CCD boundaries # ACA-019 ACA-020 ACA-021 - my $call_vals = call_python("utils._yagzag_to_pixels", [$yag, $zag]); - my ($pixel_row, $pixel_col) = @$call_vals; + my ($pixel_row, $pixel_col) = @{call_python("utils._yagzag_to_pixels", [$yag, $zag])}; # Set "acq phase" dither to acq dither or 20.0 if undefined my $dither_acq_y = $self->{dither_acq}->{ampl_y} or 20.0; @@ -2381,8 +2379,7 @@ sub star_image_map { # Convert all the yag/zags to pixel rows/cols my @yags = map { $self->{agasc_hash}->{$_}->{yag} } keys %plot_ids; my @zags = map { $self->{agasc_hash}->{$_}->{zag} } keys %plot_ids; - my $call_vals = call_python("utils._yagzag_to_pixels", [\@yags, \@zags]); - my ($pix_rows, $pix_cols) = @$call_vals; + my ($pix_rows, $pix_cols) = @{call_python("utils._yagzag_to_pixels", [\@yags, \@zags])}; my $map = " \n"; my @star_ids = keys %plot_ids; @@ -2644,8 +2641,7 @@ sub set_proseco_probs_and_check_P2{ if (not %{$args}){ return; } - my $call_vals = call_python("utils.proseco_probs", [], $args); - my ($p_acqs, $P2, $expected) = @$call_vals; + my ($p_acqs, $P2, $expected) = @{call_python("utils.proseco_probs", [], $args)}; $P2 = sprintf("%.1f", $P2); From 79c351f622653a5f12d280aa48214b18bf09ec3d Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 5 Jan 2023 09:15:20 -0500 Subject: [PATCH 14/30] Start server in the main starcheck process (don't fork) --- starcheck/src/starcheck.pl | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 18c65c11..3d17ce98 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -50,20 +50,13 @@ Ska::Starcheck::Python::set_port($server_port); Ska::Starcheck::Python::set_key($server_key); -# Global process ID for fork that gets set in the parent process. This is -# used to kill the forked process when the parent finishes. -my $pid; - # Start a server that can call functions in the starcheck package -if ($pid = fork) {} else { - open(SERVER, "| python -m starcheck.server"); - SERVER->autoflush(1); - # Send the port and key to the server - print SERVER "$server_port\n"; - print SERVER "$server_key\n"; - # Forked process waits until it gets killed by the parent finishing - sleep; -} +print "Here in the forked process\n"; +my $pid = open(SERVER, "| python -m starcheck.server"); +SERVER->autoflush(1); +# Send the port and key to the server +print SERVER "$server_port\n"; +print SERVER "$server_key\n"; # DEBUG, limit number of obsids. TODO make this a command line option. # Set to undef for no limit. From 86c39d6a73ccdd4ecd412d6f8ac57f29c4a45384 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 5 Jan 2023 09:22:36 -0500 Subject: [PATCH 15/30] Add a server timeout (probably not needed?) --- starcheck/server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/starcheck/server.py b/starcheck/server.py index f7aeffeb..bf5aa85e 100644 --- a/starcheck/server.py +++ b/starcheck/server.py @@ -11,6 +11,15 @@ func_calls = collections.Counter() +class PythonServer(socketserver.TCPServer): + timeout = 2 + + def handle_timeout(self) -> None: + print("SERVER: timeout") + # self.shutdown() # DOES NOT WORK, just hangs + sys.exit(1) + + class MyTCPHandler(socketserver.StreamRequestHandler): """ The request handler class for our server. @@ -74,10 +83,11 @@ def main(): print("SERVER: starting on port", port) # Create the server, binding to localhost on supplied port - with socketserver.TCPServer((HOST, port), MyTCPHandler) as server: + with PythonServer((HOST, port), MyTCPHandler) as server: # Activate the server; this will keep running until you # interrupt the program with Ctrl-C - server.serve_forever() + while True: + server.handle_request() if __name__ == "__main__": From 69105ce11fcd26cf63a25b4f9f8332b286e5afee Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 10:22:58 -0500 Subject: [PATCH 16/30] More verbose server timeout text --- starcheck/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/starcheck/server.py b/starcheck/server.py index bf5aa85e..780337ad 100644 --- a/starcheck/server.py +++ b/starcheck/server.py @@ -15,7 +15,8 @@ class PythonServer(socketserver.TCPServer): timeout = 2 def handle_timeout(self) -> None: - print("SERVER: timeout") + print(f"SERVER: starcheck python server timeout after {self.timeout}s idle", + file=sys.stderr) # self.shutdown() # DOES NOT WORK, just hangs sys.exit(1) From dc8a6b666b45b8563c63dff0517736de6b9d7027 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 10:29:20 -0500 Subject: [PATCH 17/30] Put GetOptions earlier in the code --- starcheck/src/starcheck.pl | 55 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 3d17ce98..2fd21da4 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -37,33 +37,6 @@ use Carp 'verbose'; $SIG{ __DIE__ } = sub { Carp::confess( @_ )}; -my $sock = IO::Socket::INET->new( - LocalAddr => '', LocalPort => 0, Proto => 'tcp', Listen => 1); -my $server_port = $sock->sockport(); -close($sock); - -# Generate a 16-character random string of letters and numbers that gets used -# as a key to authenticate the client to the server. -my $server_key = join '', map +(0..9,'a'..'z','A'..'Z')[rand 62], 1..16; - -# Configure the Python interface -Ska::Starcheck::Python::set_port($server_port); -Ska::Starcheck::Python::set_key($server_key); - -# Start a server that can call functions in the starcheck package -print "Here in the forked process\n"; -my $pid = open(SERVER, "| python -m starcheck.server"); -SERVER->autoflush(1); -# Send the port and key to the server -print SERVER "$server_port\n"; -print SERVER "$server_key\n"; - -# DEBUG, limit number of obsids. TODO make this a command line option. -# Set to undef for no limit. -my $MAX_OBSIDS = undef; - -my $version = call_python("utils.starcheck_version"); - # Set some global vars with directory locations my $SKA = $ENV{SKA} || '/proj/sot/ska'; @@ -99,6 +72,34 @@ ) || exit( 1 ); + +my $sock = IO::Socket::INET->new( + LocalAddr => '', LocalPort => 0, Proto => 'tcp', Listen => 1); +my $server_port = $sock->sockport(); +close($sock); + +# Generate a 16-character random string of letters and numbers that gets used +# as a key to authenticate the client to the server. +my $server_key = join '', map +(0..9,'a'..'z','A'..'Z')[rand 62], 1..16; + +# Configure the Python interface +Ska::Starcheck::Python::set_port($server_port); +Ska::Starcheck::Python::set_key($server_key); + +# Start a server that can call functions in the starcheck package +my $pid = open(SERVER, "| python -m starcheck.server"); +SERVER->autoflush(1); +# Send the port and key to the server +print SERVER "$server_port\n"; +print SERVER "$server_key\n"; + +# DEBUG, limit number of obsids. TODO make this a command line option. +# Set to undef for no limit. +my $MAX_OBSIDS = undef; + +my $version = call_python("utils.starcheck_version"); + + my $Starcheck_Data = $par{sc_data} || call_python("utils.get_data_dir"); my $STARCHECK = $par{out} || ($par{vehicle} ? 'v_starcheck' : 'starcheck'); From 3cc1cf4cb0618d247d8fb3ea899bb6d7f9eb2026 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 10:31:42 -0500 Subject: [PATCH 18/30] Only show server calls if verbose > 1 --- starcheck/src/starcheck.pl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 2fd21da4..19d01a5b 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -1155,11 +1155,13 @@ sub usage END { if (defined $pid) { - my $server_calls = call_python("get_server_calls"); - # print the server_calls hash sorted by value in descending order - print("Python server calls:"); - print Dump($server_calls); - print("Killing python server with pid=$pid\n"); + if ($par{verbose} gt 1){ + my $server_calls = call_python("get_server_calls"); + # print the server_calls hash sorted by value in descending order + print("Python server calls:"); + print Dump($server_calls); + } + print("Shutting down python starcheck server with pid=$pid\n"); kill 9, $pid; # must it be 9 (SIGKILL)? my $gone_pid = waitpid $pid, 0; # then check that it's gone } From 7f73242ffedcf3ccc7d63e1eca555f82e0af0fe1 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 10:37:35 -0500 Subject: [PATCH 19/30] Put usage() right after GetOptions Put usage() right after GetOptions so one can get -help without starting the python server. --- starcheck/src/starcheck.pl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 19d01a5b..77f1f541 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -72,6 +72,9 @@ ) || exit( 1 ); +usage( 1 ) + if $par{help}; + my $sock = IO::Socket::INET->new( LocalAddr => '', LocalPort => 0, Proto => 'tcp', Listen => 1); @@ -110,8 +113,6 @@ my $blue_font_start = qq{}; my $font_stop = qq{}; -usage( 1 ) - if $par{help}; # kadi log levels are a little different and INFO (corresponding to the default # verbose=1) is too chatty for the default. Instead allow only verbose=0 From 9a54ead63f92c9fb5a6e474f77ccf45c5695489e Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 10:38:51 -0500 Subject: [PATCH 20/30] Set max_obsids by cmdline option --- starcheck/src/starcheck.pl | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 77f1f541..c6cfea6a 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -50,6 +50,7 @@ fid_char => "fid_CHARACTERISTICS", verbose => 1, maude => 0, + max_obsids => 0, ); @@ -69,6 +70,7 @@ 'config_file=s', 'run_start_time=s', 'maude!', + 'max_obsids:i', ) || exit( 1 ); @@ -96,9 +98,9 @@ print SERVER "$server_port\n"; print SERVER "$server_key\n"; -# DEBUG, limit number of obsids. TODO make this a command line option. -# Set to undef for no limit. -my $MAX_OBSIDS = undef; +# DEBUG, limit number of obsids. +# Set to undef for no limit (though option defaults to 0) +my $MAX_OBSIDS = $par{max_obsids} > 0 ? $par{max_obsids} : undef; my $version = call_python("utils.starcheck_version"); @@ -414,10 +416,12 @@ } # Check that every Guide summary OFLS ID has a matching OFLS ID in DOT - -foreach my $oflsid (keys %guidesumm){ - unless (defined $obs{$oflsid}){ - #warning("OFLS ID $oflsid in Guide Summ but not in DOT! \n"); +# Skip this check if developing code with MAX_OBSIDS set +if (not defined $MAX_OBSIDS){ + foreach my $oflsid (keys %guidesumm){ + unless (defined $obs{$oflsid}){ + warning("OFLS ID $oflsid in Guide Summ but not in DOT! \n"); + } } } @@ -1220,6 +1224,10 @@ =head1 OPTIONS Use MAUDE for telemetry instead of default cheta cxc archive. MAUDE will also be used if no AACCCDPT telemetry can be found in cheta archive for initial conditions. +=item B<-max_obsids > + +Limit starcheck review to first N obsids (for testing). + =item B<-agasc_file > Specify location of agasc h5 file. Default is SKA/data/agasc/agasc1p7.h5 . From 05a1002603402c00b0c13948a4614075bcf9832b Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 11:02:48 -0500 Subject: [PATCH 21/30] Update dither parsing comments --- starcheck/src/lib/Ska/Parse_CM_File.pm | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/starcheck/src/lib/Ska/Parse_CM_File.pm b/starcheck/src/lib/Ska/Parse_CM_File.pm index 486bd3af..5117152b 100644 --- a/starcheck/src/lib/Ska/Parse_CM_File.pm +++ b/starcheck/src/lib/Ska/Parse_CM_File.pm @@ -106,9 +106,6 @@ sub dither { # 2002262.094827395 | DSDITH AODSDITH # 2002262.095427395 | ENDITH AOENDITH - # Check that the dither history does not run into the load. This is the only thing - # that is checked for the dither history file, the values are not used. - # Starcheck v2.0: skip it? # Read the last 1000 bytes in the $dh_file (guaranteed to be enough). my $dith_hist_fh = IO::File->new($dh_file, "r") or croak "Can't open $dh_file: $!"; $dith_hist_fh->seek(-1000, 2); @@ -124,13 +121,12 @@ sub dither { if (not defined $dither_error){ # If the most recent/last entry in the dither file has a timestamp newer than - # the first entry in the load, return string error and undef dither history. + # the first entry in the load, update the dither_error var. if ($dh_date ge $bs_arr->[0]->{date}){ $dither_error = "Dither history runs into load\n"; } # Confirm that last state matches kadi continuity ENAB/DISA. - # Otherwise, return string error and undef dither history. if ($kadi_dither->{'dither'} ne $dith_enab_cmd_map{$dh_state}){ $dither_error = "Dither status in kadi commands does not match DITHER history\n" . sprintf("kadi '%s' ; History '%s' \n", From bed6674d5cad5d930acd038b5c0c08c715a33237 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 11:37:24 -0500 Subject: [PATCH 22/30] If we have two dither errors, keep them. --- starcheck/src/lib/Ska/Parse_CM_File.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcheck/src/lib/Ska/Parse_CM_File.pm b/starcheck/src/lib/Ska/Parse_CM_File.pm index 5117152b..2d2e80db 100644 --- a/starcheck/src/lib/Ska/Parse_CM_File.pm +++ b/starcheck/src/lib/Ska/Parse_CM_File.pm @@ -128,7 +128,7 @@ sub dither { # Confirm that last state matches kadi continuity ENAB/DISA. if ($kadi_dither->{'dither'} ne $dith_enab_cmd_map{$dh_state}){ - $dither_error = "Dither status in kadi commands does not match DITHER history\n" + $dither_error .= "Dither status in kadi commands does not match DITHER history\n" . sprintf("kadi '%s' ; History '%s' \n", $kadi_dither->{'dither'}, $dith_enab_cmd_map{$dh_state}); From 5bc68f0b5425f95b0a5a1dbaabcc25fef125abd8 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 11:48:56 -0500 Subject: [PATCH 23/30] Remove unused Dumper import --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 8ca7e93c..72f2c713 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -28,7 +28,6 @@ use File::Basename; use POSIX qw(floor); use English; use IO::All; -use Data::Dumper qw(Dumper); use RDB; From 10118937c620ae6892ced6ea13b3c3483b145ebf Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 12:17:30 -0500 Subject: [PATCH 24/30] Increase server timeout to 180s --- starcheck/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcheck/server.py b/starcheck/server.py index 780337ad..cf30a9f6 100644 --- a/starcheck/server.py +++ b/starcheck/server.py @@ -12,7 +12,7 @@ class PythonServer(socketserver.TCPServer): - timeout = 2 + timeout = 180 def handle_timeout(self) -> None: print(f"SERVER: starcheck python server timeout after {self.timeout}s idle", From 2d351311b50de5b1763a4e995e65a2942099dd05 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 12:36:58 -0500 Subject: [PATCH 25/30] Change check on kadi log level to be gt --- starcheck/src/starcheck.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index c6cfea6a..9bac63bb 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -119,7 +119,7 @@ # kadi log levels are a little different and INFO (corresponding to the default # verbose=1) is too chatty for the default. Instead allow only verbose=0 # (CRITICAL) or verbose=2 (DEBUG). -my $kadi_verbose = $par{verbose} eq '2' ? '2' : '0'; +my $kadi_verbose = $par{verbose} gt 1 ? '2' : '0'; call_python("utils.config_logging", [$STARCHECK, $kadi_verbose, "kadi"]); call_python("utils.set_kadi_scenario_default"); From c5ea583b238f8236a9faf95f555a2fe71daa9501 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 12:41:53 -0500 Subject: [PATCH 26/30] Use Dumper instead of Dump Use Dumper instead of Dump because at least we know where it came from. --- starcheck/src/starcheck.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 9bac63bb..5c174b70 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -12,6 +12,7 @@ use strict; use warnings; +use Data::Dumper; use Getopt::Long; use IO::File; use IO::All; @@ -1164,7 +1165,7 @@ END my $server_calls = call_python("get_server_calls"); # print the server_calls hash sorted by value in descending order print("Python server calls:"); - print Dump($server_calls); + print Dumper($server_calls); } print("Shutting down python starcheck server with pid=$pid\n"); kill 9, $pid; # must it be 9 (SIGKILL)? From d719afdb6d9b8cc0972f4b503b097611bd196a7a Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 13:21:59 -0500 Subject: [PATCH 27/30] Configure output to work with -verbose --- starcheck/server.py | 34 ++++++++++++++++------- starcheck/src/lib/Ska/Starcheck/Python.pm | 17 ++++++++---- starcheck/src/starcheck.pl | 14 ++++++++-- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/starcheck/server.py b/starcheck/server.py index cf30a9f6..45e49f2b 100644 --- a/starcheck/server.py +++ b/starcheck/server.py @@ -1,10 +1,15 @@ import collections import importlib import json +import logging import socketserver import sys import traceback +from ska_helpers.logging import basic_logger + +logger = basic_logger(__name__, level="INFO") + HOST = "localhost" KEY = None @@ -33,18 +38,18 @@ class MyTCPHandler(socketserver.StreamRequestHandler): def handle(self): # self.request is the TCP socket connected to the client data = self.rfile.readline() - # print(f"SERVER receive: {data.decode('utf-8')}") + logger.debug(f"SERVER receive: {data.decode('utf-8')}") # Decode self.data from JSON cmd = json.loads(data) if cmd["key"] != KEY: - print(f"SERVER: bad key {cmd['key']!r}") + logger.ERROR(f"SERVER: bad key {cmd['key']!r}") return - # print(f"SERVER receive func: {cmd['func']}") - # print(f"SERVER receive args: {cmd['args']}") - # print(f"SERVER receive kwargs: {cmd['kwargs']}") + logger.debug(f"SERVER receive func: {cmd['func']}") + logger.debug(f"SERVER receive args: {cmd['args']}") + logger.debug(f"SERVER receive kwargs: {cmd['kwargs']}") exc = None @@ -69,20 +74,29 @@ def handle(self): exc = traceback.format_exc() resp = json.dumps({"result": result, "exception": exc}) - # print(f"SERVER send: {resp}") + logger.debug(f"SERVER send: {resp}") self.request.sendall(resp.encode("utf-8")) def main(): global KEY - + # Read a line from STDIN port = int(sys.stdin.readline().strip()) KEY = sys.stdin.readline().strip() - - print("SERVER: starting on port", port) - + loglevel = sys.stdin.readline().strip() + + logmap = {'0': logging.CRITICAL, + '1': logging.WARNING, + '2': logging.INFO} + if loglevel in logmap: + logger.setLevel(logmap[loglevel]) + if int(loglevel) > 2: + logger.setLevel(logging.DEBUG) + + logger.info(f"SERVER: starting on port {port}") + # Create the server, binding to localhost on supplied port with PythonServer((HOST, port), MyTCPHandler) as server: # Activate the server; this will keep running until you diff --git a/starcheck/src/lib/Ska/Starcheck/Python.pm b/starcheck/src/lib/Ska/Starcheck/Python.pm index c7ec3b0e..a83363f0 100644 --- a/starcheck/src/lib/Ska/Starcheck/Python.pm +++ b/starcheck/src/lib/Ska/Starcheck/Python.pm @@ -22,15 +22,18 @@ $Data::Dumper::Terse = 1; my $HOST = "localhost"; my $PORT = 40000; my $KEY = "fff"; +my $VERBOSE = 1; sub set_port { $PORT = shift; - print "CLIENT: Setting port to $PORT\n"; } sub set_key { $KEY = shift; - print "CLIENT: Setting key to $KEY\n"; +} + +sub set_debug { + $VERBOSE = shift; } sub call_python { @@ -47,8 +50,10 @@ sub call_python { "key" => $KEY, }; my $command_json = encode_json $command; - # print "CLIENT: Sending command $command_json\n"; + if ($VERBOSE gt 2){ + print STDERR "CLIENT: Sending command $command_json\n"; + } my $handle; my $iter = 0; while ($iter++ < 10) { @@ -68,8 +73,10 @@ sub call_python { $handle->close(); my $data = decode_json $response; - # print "CLIENT: Got response: $response\n"; - # print Dumper($data); + if ($VERBOSE gt 2){ + print STDERR "CLIENT: Got response: $response\n"; + print STDERR Dumper($data); + } if (defined $data->{exception}) { my $msg = "\nPython exception:\n"; $msg .= "command = " . Dumper($command) . "\n"; diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 5c174b70..053255cd 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -91,13 +91,19 @@ # Configure the Python interface Ska::Starcheck::Python::set_port($server_port); Ska::Starcheck::Python::set_key($server_key); +Ska::Starcheck::Python::set_debug($par{verbose}); +if ($par{verbose} gt 1){ + print STDERR "CLIENT: starcheck.server started on port $server_port\n"; + print STDERR "CLIENT: starcheck.server key $server_key\n"; +} # Start a server that can call functions in the starcheck package my $pid = open(SERVER, "| python -m starcheck.server"); SERVER->autoflush(1); -# Send the port and key to the server +# Send the port, key, and verbosity to the server print SERVER "$server_port\n"; print SERVER "$server_key\n"; +print SERVER "$par{verbose}\n"; # DEBUG, limit number of obsids. # Set to undef for no limit (though option defaults to 0) @@ -1167,8 +1173,10 @@ END print("Python server calls:"); print Dumper($server_calls); } - print("Shutting down python starcheck server with pid=$pid\n"); - kill 9, $pid; # must it be 9 (SIGKILL)? + if ($par{verbose} gt 1){ + print("Shutting down python starcheck server with pid=$pid\n"); + } + kill 9, $pid; # must it be 9 (SIGKILL)? my $gone_pid = waitpid $pid, 0; # then check that it's gone } }; From dec5dbb2fa244b1800282bdf297155279a593008 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 13:24:54 -0500 Subject: [PATCH 28/30] Add help for verbosity --- starcheck/src/starcheck.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 053255cd..55670271 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -1209,6 +1209,12 @@ =head1 OPTIONS Output reports will be .html, .txt. Star plots will be /stars_.png. The default is = 'STARCHECK'. +=item B<-verbose > + +Output verbosity. The default is 1. verbose=2 includes kadi logger INFO statements, +starcheck.server startup and shutdown information. verbose > 2 includes starcheck.server +full debug output. + =item B<-vehicle> Use vehicle-only products and the vehicle-only ACA checklist to perform From 5be93b463e17cfee9bf22559f5275f006e43ab0a Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 16:13:31 -0500 Subject: [PATCH 29/30] Fix guide summ bug introduced in #389 --- starcheck/src/starcheck.pl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 55670271..66b5f059 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -440,8 +440,7 @@ $obs{$oflsid}->add_guide_summ($oflsid, \%guidesumm); } else { - my $obsid = $obs{$oflsid}->{obsid}; - my $cat = Ska::Starcheck::Obsid::find_command($obs{$obsid}, "MP_STARCAT"); + my $cat = Ska::Starcheck::Obsid::find_command($obs{$oflsid}, "MP_STARCAT"); if (defined $cat){ push @{$obs{$oflsid}->{warn}}, sprintf("No Guide Star Summary for obsid $obsid ($oflsid). \n"); } From 65ae9aa91bb9fc787095ce1189eb24d9ff82c790 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 5 Jan 2023 20:22:32 -0500 Subject: [PATCH 30/30] Remove defunct decode()s --- starcheck/pcad_att_check.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/starcheck/pcad_att_check.py b/starcheck/pcad_att_check.py index f733d370..75b1dfbb 100755 --- a/starcheck/pcad_att_check.py +++ b/starcheck/pcad_att_check.py @@ -9,10 +9,6 @@ def check_characteristics_date(ofls_characteristics_file, ref_date=None): - # de_bytetring these inputs from Perl -> Python 3 - ofls_characteristics_file = ofls_characteristics_file.decode() - if ref_date is not None: - ref_date = ref_date.decode() match = re.search(r'CHARACTERIS_(\d\d)([A-Z]{3})(\d\d)', ofls_characteristics_file) if not match: return False