Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch rpc_console to ipython #31490

Merged
merged 5 commits into from
Jan 17, 2024
Merged

Switch rpc_console to ipython #31490

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cdb3138
Some formatting
andy31415 Jan 17, 2024
ee1e432
completely replace the console
andy31415 Jan 17, 2024
9373a7c
Remove some logfiles deps
andy31415 Jan 17, 2024
5eea4a9
minor format update
andy31415 Jan 17, 2024
0c5237d
Remove unused imports
andy31415 Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions examples/common/pigweed/rpc_console/py/BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ pw_python_package("chip_rpc") {
"$dir_pw_log_tokenized/py",
"$dir_pw_protobuf_compiler/py",
"$dir_pw_rpc/py",
"$dir_pw_system/py",
"$dir_pw_tokenizer/py",
"${chip_root}/examples/common/pigweed:attributes_service.python",
"${chip_root}/examples/common/pigweed:button_service.python",
Expand Down
309 changes: 53 additions & 256 deletions examples/common/pigweed/rpc_console/py/chip_rpc/console.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,35 +29,22 @@

rpcs - used to invoke RPCs
device - the serial device used for communication
client - the pw_rpc.Client
client - the HDLC rpc client
protos - protocol buffer messages indexed by proto package

An example RPC command:
rpcs.chip.rpc.DeviceCommon.GetDeviceInfo()
rpcs.chip.rpc.Device.GetDeviceInfo()
device.rpcs.chip.rpc.Device.GetDeviceInfo()
"""

import argparse
import logging
import re
import sys
import threading
from collections import namedtuple
from concurrent.futures import ThreadPoolExecutor
from inspect import cleandoc
from typing import Any, BinaryIO, Callable, Collection
from pathlib import Path
from typing import Any, Collection

import pw_cli.log
from chip_rpc.plugins.device_toolbar import DeviceToolbar
from chip_rpc.plugins.helper_scripts import HelperScripts
from pw_console import PwConsoleEmbed, socket_client
from pw_console.__main__ import create_temp_log_file
from pw_console.pyserial_wrapper import SerialWithLogging
from pw_hdlc.rpc import HdlcRpcClient, SelectableReader, SerialReader, default_channels
from pw_rpc import callback_client
from pw_rpc.console_tools.console import ClientInfo, flattened_rpc_completions
import pw_system.console
from pw_hdlc import rpc
from pw_tokenizer import tokens
from pw_tokenizer.database import LoadTokenDatabases
from pw_tokenizer.detokenize import Detokenizer

# Protos
# isort: off
Expand All @@ -72,24 +59,6 @@
from thread_service import thread_service_pb2
from wifi_service import wifi_service_pb2

_LOG = logging.getLogger(__name__)
_DEVICE_LOG = logging.getLogger('rpc_device')

PW_RPC_MAX_PACKET_SIZE = 256
SOCKET_SERVER = 'localhost'
SOCKET_PORT = 33000

PROTOS = [attributes_service_pb2,
button_service_pb2,
descriptor_service_pb2,
device_service_pb2,
echo_pb2,
lighting_service_pb2,
locking_service_pb2,
ot_cli_service_pb2,
thread_service_pb2,
wifi_service_pb2]


def _parse_args():
"""Parses and returns the command line arguments."""
Expand All @@ -116,7 +85,7 @@ def _parse_args():
parser.add_argument("--token-databases",
metavar='elf_or_token_database',
nargs="+",
action=LoadTokenDatabases,
type=Path,
help="Path to tokenizer database csv file(s).")
group.add_argument('-s',
'--socket-addr',
Expand All @@ -126,227 +95,55 @@ def _parse_args():
return parser.parse_args()


def _start_ipython_raw_terminal() -> None:
"""Starts an interactive IPython terminal with preset variables. This raw
terminal does not use HDLC and provides no RPC functionality, this is
just a serial log viewer."""
local_variables = dict(
LOG=_DEVICE_LOG,
)

welcome_message = cleandoc("""
Welcome to the CHIP Console!

This has been started in raw serial mode,
and all RPC functionality is disabled.

Press F1 for help.
""")

interactive_console = PwConsoleEmbed(
global_vars=local_variables,
local_vars=None,
loggers={
'Device Logs': [_DEVICE_LOG],
'Host Logs': [logging.getLogger()],
'Serial Debug': [logging.getLogger('pw_console.serial_debug_logger')],
},
repl_startup_message=welcome_message,
help_text=__doc__,
app_title="CHIP Console",
)

interactive_console.hide_windows('Host Logs')
interactive_console.hide_windows('Serial Debug')
interactive_console.hide_windows('Python Repl')

# Setup Python logger propagation
interactive_console.setup_python_logging()
# Don't send device logs to the root logger.
_DEVICE_LOG.propagate = False
interactive_console.embed()


def _start_ipython_hdlc_terminal(client: HdlcRpcClient) -> None:
"""Starts an interactive IPython terminal with preset variables."""
local_variables = dict(
client=client,
channel_client=client.client.channel(1),
rpcs=client.client.channel(1).rpcs,
scripts=HelperScripts(client.client.channel(1).rpcs),
protos=client.protos.packages,
# Include the active pane logger for creating logs in the repl.
LOG=_DEVICE_LOG,
)

client_info = ClientInfo('channel_client',
client.client.channel(1).rpcs, client.client)
completions = flattened_rpc_completions([client_info])

welcome_message = cleandoc("""
Welcome to the CHIP RPC Console!

Press F1 for help.
Example commands:

rpcs.chip.rpc.Device.GetDeviceInfo()

LOG.warning('Message appears console log window.')
""")

interactive_console = PwConsoleEmbed(
global_vars=local_variables,
local_vars=None,
loggers={
'Device Logs': [_DEVICE_LOG],
'Host Logs': [logging.getLogger()],
'Serial Debug': [logging.getLogger('pw_console.serial_debug_logger')],
},
repl_startup_message=welcome_message,
help_text=__doc__,
app_title="CHIP Console",
)

interactive_console.add_sentence_completer(completions)
interactive_console.add_bottom_toolbar(
DeviceToolbar(client.client.channel(1).rpcs))

interactive_console.hide_windows('Host Logs')
interactive_console.hide_windows('Serial Debug')

# Setup Python logger propagation
interactive_console.setup_python_logging()
# Don't send device logs to the root logger.
_DEVICE_LOG.propagate = False
with client:
interactive_console.embed()


def write_to_output(data: bytes,
unused_output: BinaryIO = sys.stdout.buffer,
detokenizer=None):
log_line = data
RegexStruct = namedtuple('RegexStruct', 'platform type regex match_num')
LEVEL_MAPPING = {"I": logging.INFO, "W": logging.WARNING, "P": logging.INFO,
"E": logging.ERROR, "F": logging.FATAL, "V": logging.DEBUG, "D": logging.DEBUG,
"<inf>": logging.INFO, "<dbg>": logging.DEBUG, "<err>": logging.ERROR,
"<info >": logging.INFO, "<warn >": logging.WARNING,
"<error >": logging.ERROR, "<detail>": logging.DEBUG,
"ERR": logging.ERROR, "DBG": logging.DEBUG, "INF": logging.INFO}

ESP_CHIP_REGEX = r"(?P<level>[IWEFV]) \((?P<time>\d+)\) (?P<mod>chip\[[a-zA-Z]+\]):\s(?P<msg>.*)"
ESP_APP_REGEX = r"(?P<level>[IWEFVD]) \((?P<time>\d+)\) (?P<mod>[a-z\-_A-Z]+):\s(?P<msg>.*)"

EFR_CHIP_REGEX = r"(?P<level><detail>|<info >|<error >|<warn >)\s(?P<mod>\[[a-zA-Z\-]+\])\s(?P<msg>.*)"
EFR_APP_REGEX = r"<efr32 >\s(?P<msg>.*)"

NRF_CHIP_REGEX = r"\[(?P<time>\d+)\] (?P<level><inf>|<dbg>|<err>) chip.*: \[(?P<mod>[a-z\-A-Z]+)\](?P<msg>.*)"
NRF_APP_REGEX = r"\[(?P<time>\d+)\] (?P<level><inf>|<dbg>|<err>) (?P<msg>.*)"

NXP_CHIP_REGEX = r"\[(?P<time>\d+)\]\[(?P<level>[EPDF])\]\[(?P<mod>[a-z\-A-Z]+)\](?P<msg>.*)"
NXP_APP_REGEX = r"\[(?P<time>\d+)\]\[(?P<mod>[a-z\-A-Z]+)\](?P<msg>.*)"

LINUX_REGEX = r".*(?P<level>INF|DBG|ERR).*\s+\[(?P<time>[0-9]+\.?[0-9]*)\]\[(?P<pid>\d+)\:(?P<tid>\d+)\] CHIP:(?P<mod>[a-z\-A-Z]+)\: (?P<msg>.*)"

LogRegexes = [RegexStruct("ESP", "CHIP", re.compile(ESP_CHIP_REGEX), 4),
RegexStruct("ESP", "APP", re.compile(ESP_APP_REGEX), 4),
RegexStruct("EFR", "CHIP", re.compile(EFR_CHIP_REGEX), 3),
RegexStruct("EFR", "APP", re.compile(EFR_APP_REGEX), 1),
RegexStruct("NRF", "CHIP", re.compile(NRF_CHIP_REGEX), 4),
RegexStruct("NRF", "APP", re.compile(NRF_APP_REGEX), 3),
RegexStruct("NXP", "CHIP", re.compile(NXP_CHIP_REGEX), 4),
RegexStruct("NXP", "APP", re.compile(NXP_APP_REGEX), 3),
RegexStruct("LINUX", "CHIP", re.compile(LINUX_REGEX), 6)
]
for line in log_line.decode(errors="surrogateescape").splitlines():
fields = {'level': logging.INFO, "time": "",
"mod": "", "type": "", "msg": line}
for log_regex in LogRegexes:
match = log_regex.regex.search(line)
if match and len(match.groups()) == log_regex.match_num:
fields['type'] = log_regex.type
fields.update(match.groupdict())
if "level" in match.groupdict():
fields["level"] = LEVEL_MAPPING[fields["level"]]
if detokenizer:
_LOG.warn(fields["msg"])
if len(fields["msg"]) % 2:
# TODO the msg likely wrapped, trim for now
fields["msg"] = fields["msg"][:-1]
fields["msg"] = detokenizer.detokenize(
bytes.fromhex(fields["msg"]))
break

_DEVICE_LOG.log(fields["level"], fields["msg"], extra={'extra_metadata_fields': {
"timestamp": fields["time"], "type": fields["type"], "mod": fields["mod"]}})


def _read_raw_serial(read: Callable[[], bytes], output):
"""Continuously read and pass to output."""
with ThreadPoolExecutor() as _:
while True:
try:
data = read()
except Exception: # pylint: disable=broad-except
continue
if data:
output(data)


def console(device: str, baudrate: int,
token_databases: Collection[tokens.Database],
socket_addr: str, output: Any, raw_serial: bool) -> int:
"""Starts an interactive RPC console for HDLC."""
# argparse.FileType doesn't correctly handle '-' for binary files.
if output is sys.stdout:
output = sys.stdout.buffer

logfile = create_temp_log_file()
pw_cli.log.install(logging.INFO, True, False, logfile)

serial_impl = SerialWithLogging

if socket_addr is None:
serial_device = serial_impl(device, baudrate, timeout=0.1)
reader = SerialReader(serial_device, 8192)
write = serial_device.write
else:
try:
socket_device = socket_client.SocketClient(socket_addr)
reader = SelectableReader(socket_device)
write = socket_device.write
except ValueError:
_LOG.exception('Failed to initialize socket at %s', socket_addr)
return 1

callback_client_impl = callback_client.Impl(
default_unary_timeout_s=5.0,
default_stream_timeout_s=None,
def show_console(device: str, baudrate: int,
token_databases: Collection[Path],
socket_addr: str, output: Any, raw_serial: bool) -> int:

# TODO: this shows a default console with little customization
# Ideally we should at least customize the default messages
#
#
# For now example of how to run commands:
#
# device.rpcs.chip.rpc.Device.GetDeviceInfo()
#

pw_system.console.console(
device=device,
baudrate=baudrate,
socket_addr=socket_addr,
output=output,
hdlc_encoding=not raw_serial,
token_databases=token_databases,
logfile="",
device_logfile="",
channel_id=rpc.DEFAULT_CHANNEL_ID,

# Defaults beyond the original console
proto_globs=[],
ticks_per_second=None,
host_logfile="",
json_logfile="",
rpc_logging=False,
# the pt-python based console seems to break on python 3.1 with
# "set_wakeup_fd only works in main thread of the main interpreter"
use_ipython=True,
compiled_protos=[
attributes_service_pb2,
button_service_pb2,
descriptor_service_pb2,
device_service_pb2,
echo_pb2,
lighting_service_pb2,
locking_service_pb2,
ot_cli_service_pb2,
thread_service_pb2,
wifi_service_pb2]
)

detokenizer = Detokenizer(tokens.Database.merged(*token_databases),
show_errors=False) if token_databases else None

if raw_serial:
threading.Thread(target=_read_raw_serial,
daemon=True,
args=(reader,
lambda data: write_to_output(
data, output, detokenizer),
)).start()
_start_ipython_raw_terminal()
else:
_start_ipython_hdlc_terminal(
HdlcRpcClient(reader, PROTOS, default_channels(write),
lambda data: write_to_output(
data, output, detokenizer),
client_impl=callback_client_impl)
)
return 0


def main() -> int:
return console(**vars(_parse_args()))
return show_console(**vars(_parse_args()))


if __name__ == '__main__':
Expand Down
Loading