Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace Inline::Python with a light Python function server #402

Merged
merged 30 commits into from
Jan 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
e14e616
Apparently working w/ time conversions
taldcroft Dec 30, 2022
c0b7b7b
Vectorize date2time for backstop
taldcroft Dec 30, 2022
c552e4c
Apply black/isort to utils.py
taldcroft Dec 30, 2022
d6450f4
Running to completion
taldcroft Dec 30, 2022
a795acc
Remove final INLINE refs (env vars)
taldcroft Dec 30, 2022
dce71d8
Minor cleanup
taldcroft Dec 30, 2022
58c11e3
Get free port and secure comm w/ key
taldcroft Dec 31, 2022
b7aeb60
Add functionality to collect call statistics
taldcroft Dec 31, 2022
a4f57d9
Reduce some Python calls and cut unused processing
taldcroft Dec 31, 2022
38beb17
No limit on number of obsids
taldcroft Dec 31, 2022
4ada1e6
Vectorize yagzag_to_pixels call
taldcroft Jan 1, 2023
d6087bd
Don't call pixel_to_yagzag for bad pixels
taldcroft Jan 1, 2023
c95b910
More performance and code tidy
taldcroft Jan 1, 2023
79c351f
Start server in the main starcheck process (don't fork)
taldcroft Jan 5, 2023
86c39d6
Add a server timeout (probably not needed?)
taldcroft Jan 5, 2023
69105ce
More verbose server timeout text
jeanconn Jan 5, 2023
dc8a6b6
Put GetOptions earlier in the code
jeanconn Jan 5, 2023
3cc1cf4
Only show server calls if verbose > 1
jeanconn Jan 5, 2023
7f73242
Put usage() right after GetOptions
jeanconn Jan 5, 2023
9a54ead
Set max_obsids by cmdline option
jeanconn Jan 5, 2023
05a1002
Update dither parsing comments
jeanconn Jan 5, 2023
bed6674
If we have two dither errors, keep them.
jeanconn Jan 5, 2023
5bc68f0
Remove unused Dumper import
jeanconn Jan 5, 2023
1011893
Increase server timeout to 180s
jeanconn Jan 5, 2023
2d35131
Change check on kadi log level to be gt
jeanconn Jan 5, 2023
c5ea583
Use Dumper instead of Dump
jeanconn Jan 5, 2023
d719afd
Configure output to work with -verbose
jeanconn Jan 5, 2023
dec5dbb
Add help for verbosity
jeanconn Jan 5, 2023
5be93b4
Fix guide summ bug introduced in #389
jeanconn Jan 5, 2023
65ae9aa
Remove defunct decode()s
jeanconn Jan 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions sandbox_starcheck
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,6 @@ then
exit 1
fi

PERL_INLINE_DIRECTORY=`mktemp -d -t starcheck_inline.XXXXXX` || exit 1
export PERL_INLINE_DIRECTORY
perl -I ${DIR}/starcheck/src/lib ${DIR}/starcheck/src/starcheck.pl "$@"
SC_STATUS=$?
if [[ -d $PERL_INLINE_DIRECTORY && $PERL_INLINE_DIRECTORY =~ .*starcheck_inline.* ]];
then
rm -r $PERL_INLINE_DIRECTORY
fi
exit $SC_STATUS
4 changes: 0 additions & 4 deletions starcheck/pcad_att_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@


def check_characteristics_date(ofls_characteristics_file, ref_date=None):
# de_bytetring these inputs from Perl -> Python 3
ofls_characteristics_file = ofls_characteristics_file.decode()
if ref_date is not None:
ref_date = ref_date.decode()
match = re.search(r'CHARACTERIS_(\d\d)([A-Z]{3})(\d\d)', ofls_characteristics_file)
if not match:
return False
Expand Down
109 changes: 109 additions & 0 deletions starcheck/server.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import collections
import importlib
import json
import logging
import socketserver
import sys
import traceback

from ska_helpers.logging import basic_logger

logger = basic_logger(__name__, level="INFO")

HOST = "localhost"
KEY = None

func_calls = collections.Counter()


class PythonServer(socketserver.TCPServer):
timeout = 180

def handle_timeout(self) -> None:
print(f"SERVER: starcheck python server timeout after {self.timeout}s idle",
file=sys.stderr)
# self.shutdown() # DOES NOT WORK, just hangs
sys.exit(1)


class MyTCPHandler(socketserver.StreamRequestHandler):
"""
The request handler class for our server.

It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""

def handle(self):
# self.request is the TCP socket connected to the client
data = self.rfile.readline()
logger.debug(f"SERVER receive: {data.decode('utf-8')}")

# Decode self.data from JSON
cmd = json.loads(data)

if cmd["key"] != KEY:
logger.ERROR(f"SERVER: bad key {cmd['key']!r}")
return

logger.debug(f"SERVER receive func: {cmd['func']}")
logger.debug(f"SERVER receive args: {cmd['args']}")
logger.debug(f"SERVER receive kwargs: {cmd['kwargs']}")

exc = None

# For security reasons, only allow functions in the public API of starcheck module
if cmd["func"] == "get_server_calls":
# Sort func calls by the items in the counter
result = dict(func_calls)
else:
func_calls[cmd["func"]] += 1
parts = cmd["func"].split(".")
package = ".".join(["starcheck"] + parts[:-1])
func = parts[-1]
module = importlib.import_module(package)
func = getattr(module, func)
args = cmd["args"]
kwargs = cmd["kwargs"]

try:
result = func(*args, **kwargs)
except Exception:
result = None
exc = traceback.format_exc()

resp = json.dumps({"result": result, "exception": exc})
logger.debug(f"SERVER send: {resp}")

self.request.sendall(resp.encode("utf-8"))


def main():
global KEY

# Read a line from STDIN
port = int(sys.stdin.readline().strip())
KEY = sys.stdin.readline().strip()
loglevel = sys.stdin.readline().strip()

logmap = {'0': logging.CRITICAL,
jeanconn marked this conversation as resolved.
Show resolved Hide resolved
'1': logging.WARNING,
'2': logging.INFO}
if loglevel in logmap:
logger.setLevel(logmap[loglevel])
if int(loglevel) > 2:
logger.setLevel(logging.DEBUG)

logger.info(f"SERVER: starting on port {port}")

# Create the server, binding to localhost on supplied port
with PythonServer((HOST, port), MyTCPHandler) as server:
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
while True:
server.handle_request()


if __name__ == "__main__":
main()
Loading