From 4094cf567472cabfc3ddb5d830e8971d5f2e037b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simen=20S=2E=20R=C3=B8stad?= Date: Wed, 22 Jan 2025 14:37:00 +0100 Subject: [PATCH] modules: memfault: Add automatic sending of coredumps over LTE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automatic sending of coredumps over LTE: - Add new file memfault_lte_coredump.c that uses LTE cereg and PDN to determine if the device is connected to the network. If connected, the layer will trigger sending of a stored coredump. Library is implemented with retry logic and backoff. - Add overlay file to SLM that enables Memfault features - Update Memfault sample to use the new coredump feature. - Add missing features to Memfault sample - Enable assertions by default in SLM, there is no reason why it should be enabled as long as there is space. Signed-off-by: Simen S. Røstad --- .../serial_lte_modem/doc/slm_description.rst | 2 + .../serial_lte_modem/overlay-memfault.conf | 28 + applications/serial_lte_modem/prj.conf | 3 +- applications/serial_lte_modem/sample.yaml | 21 + .../serial_lte_modem/src/slm_at_commands.c | 15 +- doc/nrf/libraries/debug/memfault_ncs.rst | 12 + .../releases/release-notes-changelog.rst | 5 +- modules/memfault-firmware-sdk/CMakeLists.txt | 4 + modules/memfault-firmware-sdk/Kconfig | 27 + .../memfault_lte_coredump.c | 498 ++++++++++++++++++ .../memfault/boards/nrf9151dk_nrf9151_ns.conf | 3 + .../memfault/boards/nrf9160dk_nrf9160_ns.conf | 3 + .../memfault/boards/nrf9161dk_nrf9161_ns.conf | 3 + .../memfault/boards/thingy91_nrf9160_ns.conf | 3 + .../memfault/boards/thingy91x_nrf9151_ns.conf | 4 + samples/debug/memfault/prj.conf | 7 +- samples/debug/memfault/src/main.c | 5 + 17 files changed, 633 insertions(+), 10 deletions(-) create mode 100644 applications/serial_lte_modem/overlay-memfault.conf create mode 100644 modules/memfault-firmware-sdk/memfault_lte_coredump.c diff --git a/applications/serial_lte_modem/doc/slm_description.rst b/applications/serial_lte_modem/doc/slm_description.rst index 1f1ff3c352ec..8d827843d8e0 100644 --- a/applications/serial_lte_modem/doc/slm_description.rst +++ b/applications/serial_lte_modem/doc/slm_description.rst @@ -336,6 +336,8 @@ The following configuration files are provided: It can be customized to fit your configuration (UART, baud rate, and so on). By default, it sets the baud rate of the PPP UART to 1 000 000. +* :file:`overlay-memfault.conf` - Configuration file that enables `Memfault`_, for more information about the Memfault integration in NCS see :ref:`mod_memfault`. + * :file:`overlay-zephyr-modem.conf`, :file:`overlay-zephyr-modem-external-mcu.conf`, :file:`overlay-zephyr-modem-nrf9160dk-nrf52840.conf`, :file:`overlay-external-mcu.overlay`, and :file:`overlay-zephyr-modem-nrf9160dk-nrf52840.overlay` - These configuration files are used when compiling SLM to turn an nRF91 Series SiP into a Zephyr-compatible standalone modem. See :ref:`slm_as_zephyr_modem` for more information. diff --git a/applications/serial_lte_modem/overlay-memfault.conf b/applications/serial_lte_modem/overlay-memfault.conf new file mode 100644 index 000000000000..71c70ae4a664 --- /dev/null +++ b/applications/serial_lte_modem/overlay-memfault.conf @@ -0,0 +1,28 @@ +# +# Copyright (c) 2025 Nordic Semiconductor +# +# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause +# + +CONFIG_MEMFAULT=y +CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD=y +CONFIG_MODEM_KEY_MGMT=y +CONFIG_MEMFAULT_LOGGING_ENABLE=y +CONFIG_MEMFAULT_HTTP_ENABLE=y +CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_STACK_METRICS=y +CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP=y +CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y +CONFIG_MEMFAULT_LOGGING_RAM_SIZE=4096 +CONFIG_MEMFAULT_HEAP_STATS=y +CONFIG_MEMFAULT_HTTP_DEDICATED_WORKQUEUE_STACK_SIZE=1560 +CONFIG_MEMFAULT_COREDUMP_FULL_THREAD_STACKS=y +CONFIG_MEMFAULT_EVENT_STORAGE_SIZE=2048 +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_PDN=y +CONFIG_LTE_LINK_CONTROL=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y + +# Memfault depends on POSIX, disable unneeded POSIX features +CONFIG_POSIX_FILE_SYSTEM=n diff --git a/applications/serial_lte_modem/prj.conf b/applications/serial_lte_modem/prj.conf index 222c1d3acd9e..fc9ac692ad6a 100644 --- a/applications/serial_lte_modem/prj.conf +++ b/applications/serial_lte_modem/prj.conf @@ -5,6 +5,8 @@ # # General config CONFIG_LOG=y +CONFIG_ASSERT=y +CONFIG_ASSERT_VERBOSE=n CONFIG_LOG_DEFAULT_LEVEL=3 CONFIG_STACK_SENTINEL=y CONFIG_PICOLIBC_IO_FLOAT=y @@ -118,7 +120,6 @@ CONFIG_SLM_CUSTOMER_VERSION="" CONFIG_SLM_EXTERNAL_XTAL=n # debug options -#CONFIG_ASSERT=y #CONFIG_LOG_BUFFER_SIZE=16384 #CONFIG_SLM_LOG_LEVEL_DBG=y #CONFIG_LOG_PRINTK=n diff --git a/applications/serial_lte_modem/sample.yaml b/applications/serial_lte_modem/sample.yaml index c7da81025ba6..5fef7f1c1169 100644 --- a/applications/serial_lte_modem/sample.yaml +++ b/applications/serial_lte_modem/sample.yaml @@ -21,6 +21,27 @@ tests: tags: - ci_build - sysbuild + applications.serial_lte_modem.memfault: + sysbuild: true + build_only: true + extra_args: EXTRA_CONF_FILE=overlay-memfault.conf + platform_allow: + - nrf9160dk/nrf9160/ns + - nrf9161dk/nrf9161/ns + - nrf9151dk/nrf9151/ns + - nrf9131ek/nrf9131/ns + - thingy91/nrf9160/ns + - thingy91x/nrf9151/ns + integration_platforms: + - nrf9160dk/nrf9160/ns + - nrf9161dk/nrf9161/ns + - nrf9151dk/nrf9151/ns + - nrf9131ek/nrf9131/ns + - thingy91/nrf9160/ns + - thingy91x/nrf9151/ns + tags: + - ci_build + - sysbuild applications.serial_lte_modem.native_tls: sysbuild: true build_only: true diff --git a/applications/serial_lte_modem/src/slm_at_commands.c b/applications/serial_lte_modem/src/slm_at_commands.c index f5c42d0b08b8..d2155d0cbbb8 100644 --- a/applications/serial_lte_modem/src/slm_at_commands.c +++ b/applications/serial_lte_modem/src/slm_at_commands.c @@ -81,7 +81,7 @@ enum sleep_modes { static struct { struct k_work_delayable work; uint32_t mode; -} sleep; +} sleep_control; #endif bool verify_datamode_control(uint16_t time_limit, uint16_t *time_limit_min); @@ -130,13 +130,13 @@ static int handle_at_slmver(enum at_parser_cmd_type cmd_type, struct at_parser * static void go_sleep_wk(struct k_work *) { - if (sleep.mode == SLEEP_MODE_IDLE) { + if (sleep_control.mode == SLEEP_MODE_IDLE) { if (slm_at_host_power_off() == 0) { slm_enter_idle(); } else { LOG_ERR("failed to power off UART"); } - } else if (sleep.mode == SLEEP_MODE_DEEP) { + } else if (sleep_control.mode == SLEEP_MODE_DEEP) { slm_enter_sleep(); } } @@ -148,12 +148,13 @@ static int handle_at_sleep(enum at_parser_cmd_type cmd_type, struct at_parser *p int ret = -EINVAL; if (cmd_type == AT_PARSER_CMD_TYPE_SET) { - ret = at_parser_num_get(parser, 1, &sleep.mode); + ret = at_parser_num_get(parser, 1, &sleep_control.mode); if (ret) { return -EINVAL; } - if (sleep.mode == SLEEP_MODE_DEEP || sleep.mode == SLEEP_MODE_IDLE) { - k_work_reschedule(&sleep.work, SLM_UART_RESPONSE_DELAY); + if (sleep_control.mode == SLEEP_MODE_DEEP || + sleep_control.mode == SLEEP_MODE_IDLE) { + k_work_reschedule(&sleep_control.work, SLM_UART_RESPONSE_DELAY); } else { ret = -EINVAL; } @@ -376,7 +377,7 @@ int slm_at_init(void) int err; #if POWER_PIN_IS_ENABLED - k_work_init_delayable(&sleep.work, go_sleep_wk); + k_work_init_delayable(&sleep_control.work, go_sleep_wk); #endif err = slm_at_tcp_proxy_init(); diff --git a/doc/nrf/libraries/debug/memfault_ncs.rst b/doc/nrf/libraries/debug/memfault_ncs.rst index 05410fbf0468..2b8a6b71aec9 100644 --- a/doc/nrf/libraries/debug/memfault_ncs.rst +++ b/doc/nrf/libraries/debug/memfault_ncs.rst @@ -58,6 +58,17 @@ To have these configuration files in the include path, add the following to the .. memfault_config_files_end +Automatic Sending of Coredumps to Memfault +========================================== + +To post a stored coredump from a previous crash to Memfault upon network connection, set the option :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` to ``y``. +The option is only supported for nRF91 targets. + +The library has built in connection awareness and will try to post the coredump a maximum of :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX` times with an interval of :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS` seconds between each attempt. +If unsuccessful within the number of attempts, the will give up. If at any point network is lost during the retry process, the library will wait for the device to reconnect before resetting the rety process. + +This feature is useful when you want to post the coredump as soon as possible after a crash and its not desireable to wait for the next periodic upload set by :kconfig:option:`CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD_INTERVAL_SECS`. +Alternatively, you can manually trigger the coredump upload by calling the function :c:func:`memfault_zephyr_port_post_data`. :c:function:`memfault_coredump_has_valid_coredump` can be used to check if a coredump is available. Configuration options in Memfault SDK ===================================== @@ -93,6 +104,7 @@ Configuration options in |NCS| The Kconfig options for Memfault that are defined in |NCS| provide some additional features compared to the options that are already implemented in Memfault SDK: +* :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` * :kconfig:option:`CONFIG_MEMFAULT_NCS_PROJECT_KEY` * :kconfig:option:`CONFIG_MEMFAULT_NCS_PROVISION_CERTIFICATES` * :kconfig:option:`CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP` diff --git a/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst b/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst index 8b9df4478333..bcbe36862c49 100644 --- a/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst +++ b/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst @@ -228,6 +228,7 @@ Serial LTE modem ---------------- * Updated the application to use the :ref:`lib_downloader` library instead of the deprecated :ref:`lib_download_client` library. +* Added overlay :file:`overlay-memfault.conf` to enable Memfault. For more information, see the documentation for Memfault integration in NCS, :ref:`mod_memfault`. Thingy:53: Matter weather station --------------------------------- @@ -615,7 +616,9 @@ Edge Impulse integration Memfault integration -------------------- -|no_changes_yet_note| +* Added a new feature to autoamtically post coredumps to Memfault when network connectivity is available. + This feature can be enabled by setting the :kconfig:option:`MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` Kconfig option to ``y``. + Only supported for nRF91 Series targets. AVSystem integration -------------------- diff --git a/modules/memfault-firmware-sdk/CMakeLists.txt b/modules/memfault-firmware-sdk/CMakeLists.txt index b0d972ef529a..79ed87400217 100644 --- a/modules/memfault-firmware-sdk/CMakeLists.txt +++ b/modules/memfault-firmware-sdk/CMakeLists.txt @@ -31,6 +31,10 @@ zephyr_library_sources_ifdef( CONFIG_MEMFAULT_NCS_BT_METRICS memfault_bt_metrics.c) +zephyr_library_sources_ifdef( + CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED + memfault_lte_coredump.c) + zephyr_library_sources_ifdef( CONFIG_MEMFAULT_NCS_ETB_CAPTURE memfault_etb_trace_capture.c) diff --git a/modules/memfault-firmware-sdk/Kconfig b/modules/memfault-firmware-sdk/Kconfig index 3b8137b8b480..53d6ebaf4c72 100644 --- a/modules/memfault-firmware-sdk/Kconfig +++ b/modules/memfault-firmware-sdk/Kconfig @@ -191,6 +191,33 @@ config MEMFAULT_NCS_IMPLEMENT_METRICS_COLLECTION Implement the Memfault 'memfault_metrics_heartbeat_collect_data()' function for the selected metrics. Disable this to override the implementation. +config MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED + bool "Post coredump on network connected" + depends on PDN + depends on LTE_LINK_CONTROL + select SMF + select SMF_ANCESTOR_SUPPORT + select SMF_INITIAL_TRANSITION + help + Post coredump to Memfault when the device is connected to LTE network. + This option is only supported for nRF91 targets. + +config MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX + int "Max number of coredump post retries" + default 3 + help + Maximum number of retries to post a coredump to Memfault. + +config MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS + int "Retry interval in seconds" + default 30 + help + Interval in seconds between coredump post retries. + +config MEMFAULT_NCS_POST_COREDUMP_THREAD_STACK_SIZE + int "Post coredump thread size" + default 512 + config MEMFAULT_NCS_ETB_CAPTURE bool "Enable ETB trace capture" depends on ETB_TRACE diff --git a/modules/memfault-firmware-sdk/memfault_lte_coredump.c b/modules/memfault-firmware-sdk/memfault_lte_coredump.c new file mode 100644 index 000000000000..8d7e174e2fb3 --- /dev/null +++ b/modules/memfault-firmware-sdk/memfault_lte_coredump.c @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2025 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +LOG_MODULE_DECLARE(memfault_ncs, CONFIG_MEMFAULT_NCS_LOG_LEVEL); + +NRF_MODEM_LIB_ON_INIT(memfault_ncs_lte_coredump_init_hook, on_modem_lib_init, NULL); +NRF_MODEM_LIB_ON_SHUTDOWN(memfault_ncs_lte_coredump_shutdown_hook, on_modem_lib_shutdown, NULL); + +#define MSG_QUEUE_ENTRY_COUNT 4 +#define MSG_QUEUE_BYTE_ALIGNMENT 4 + +enum library_state { + STATE_RUNNING, + STATE_WAITING_FOR_NRF_MODEM_LIB_INIT, + STATE_WAITING_FOR_NETWORK_CONNECTION, + STATE_NETWORK_CONNECTED, + STATE_COREDUMP_SEND_ATTEMPT, + STATE_COREDUMP_SEND_BACKOFF, + STATE_FINISHED, + STATE_ERROR +}; + +enum library_event { + EVENT_NRF_MODEM_LIB_INITED, + EVENT_NRF_MODEM_LIB_SHUTDOWN, + EVENT_LTE_REGISTERED, + EVENT_LTE_DEREGISTERED, + EVENT_PDN_ACTIVATED, + EVENT_PDN_DEACTIVATED, + EVENT_NETWORK_CONNECTED, + EVENT_NETWORK_DISCONNECTED, + EVENT_BACKOFF_TIMER_EXPIRED, + EVENT_COREDUMP_SEND_FAIL, + EVENT_COREDUMP_SEND_SUCCESS, + EVENT_ERROR +}; + +K_MSGQ_DEFINE(mflt_lte_coredump_queue, + sizeof(enum library_event), + MSG_QUEUE_ENTRY_COUNT, + MSG_QUEUE_BYTE_ALIGNMENT); + +struct fsm_state_object { + /* This must be first */ + struct smf_ctx ctx; + + enum library_event event; + + uint8_t retry_count; + + struct k_work_delayable backoff_work; + + bool lte_registered; + bool pdn_active; +}; + +/* Forward declarations: Local functions */ +static void pdn_event_handler(uint8_t cid, enum pdn_event event, int reason); +static void lte_handler(const struct lte_lc_evt *const evt); +static void on_modem_lib_init(int ret, void *ctx); +static void on_modem_lib_shutdown(void *ctx); +static void event_send(enum library_event event); +static void schedule_send_backoff(struct fsm_state_object *state_object); + +/* Forward declarations: State Machine Functions */ +static void state_running_entry(void *o); +static void state_running_run(void *o); +static void state_running_exit(void *o); +static void state_waiting_for_nrf_modem_lib_init_entry(void *o); +static void state_waiting_for_nrf_modem_lib_init_run(void *o); +static void state_waiting_for_network_connection_run(void *o); +static void state_network_connected_entry(void *o); +static void state_network_connected_run(void *o); +static void state_coredump_send_attempt_entry(void *o); +static void state_coredump_send_attempt_run(void *o); +static void state_coredump_send_backoff_entry(void *o); +static void state_coredump_send_backoff_run(void *o); +static void state_finished_entry(void *o); +static void state_error_entry(void *o); + +static const struct smf_state states[] = { + [STATE_WAITING_FOR_NRF_MODEM_LIB_INIT] = + SMF_CREATE_STATE(state_waiting_for_nrf_modem_lib_init_entry, + state_waiting_for_nrf_modem_lib_init_run, + NULL, + NULL, + NULL), + [STATE_RUNNING] = + SMF_CREATE_STATE(state_running_entry, + state_running_run, + state_running_exit, + NULL, + &states[STATE_WAITING_FOR_NETWORK_CONNECTION]), + [STATE_WAITING_FOR_NETWORK_CONNECTION] = + SMF_CREATE_STATE(NULL, + state_waiting_for_network_connection_run, + NULL, + &states[STATE_RUNNING], + NULL), + [STATE_NETWORK_CONNECTED] = + SMF_CREATE_STATE(state_network_connected_entry, + state_network_connected_run, + NULL, + &states[STATE_RUNNING], + &states[STATE_COREDUMP_SEND_ATTEMPT]), + [STATE_COREDUMP_SEND_ATTEMPT] = + SMF_CREATE_STATE(state_coredump_send_attempt_entry, + state_coredump_send_attempt_run, + NULL, + &states[STATE_NETWORK_CONNECTED], + NULL), + [STATE_COREDUMP_SEND_BACKOFF] = + SMF_CREATE_STATE(state_coredump_send_backoff_entry, + state_coredump_send_backoff_run, + NULL, + &states[STATE_NETWORK_CONNECTED], + NULL), + [STATE_FINISHED] = + SMF_CREATE_STATE(state_finished_entry, + NULL, + NULL, + NULL, + NULL), + [STATE_ERROR] = + SMF_CREATE_STATE(state_error_entry, + NULL, + NULL, + NULL, + NULL) +}; + +static void timer_work_fn(struct k_work *work) +{ + event_send(EVENT_BACKOFF_TIMER_EXPIRED); +} + +static struct fsm_state_object state_object = { 0 }; + +/* nRF Modem Library Handlers */ + +static void on_modem_lib_init(int ret, void *ctx) +{ + event_send(EVENT_NRF_MODEM_LIB_INITED); +} + +static void on_modem_lib_shutdown(void *ctx) +{ + event_send(EVENT_NRF_MODEM_LIB_SHUTDOWN); +} + +/* Callback Handlers for Link Controller and PDN Libraries */ + +static void lte_handler(const struct lte_lc_evt *const evt) +{ + if (evt->type == LTE_LC_EVT_NW_REG_STATUS) { + switch (evt->nw_reg_status) { + case LTE_LC_NW_REG_REGISTERED_HOME: + __fallthrough; + case LTE_LC_NW_REG_REGISTERED_ROAMING: + event_send(EVENT_LTE_REGISTERED); + break; + case LTE_LC_NW_REG_UNKNOWN: + __fallthrough; + case LTE_LC_NW_REG_UICC_FAIL: + __fallthrough; + case LTE_LC_NW_REG_NOT_REGISTERED: + __fallthrough; + case LTE_LC_NW_REG_REGISTRATION_DENIED: + event_send(EVENT_LTE_DEREGISTERED); + default: + /* Don't care */ + break; + } + } +} + +static void pdn_event_handler(uint8_t cid, enum pdn_event event, int reason) +{ + switch (event) { + case PDN_EVENT_ACTIVATED: + event_send(EVENT_PDN_ACTIVATED); + break; + case PDN_EVENT_NETWORK_DETACH: + __fallthrough; + case PDN_EVENT_DEACTIVATED: + event_send(EVENT_PDN_DEACTIVATED); + break; + default: + /* Don't care */ + break; + } +} + +/* SMF State Handlers */ + +static void state_waiting_for_nrf_modem_lib_init_entry(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_waiting_for_nrf_modem_lib_init_entry"); + + state_object->lte_registered = false; + state_object->pdn_active = false; + state_object->retry_count = 0; +} + +static void state_waiting_for_nrf_modem_lib_init_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_waiting_for_nrf_modem_lib_init_run"); + + if (state_object->event == EVENT_NRF_MODEM_LIB_INITED) { + smf_set_state(SMF_CTX(state_object), &states[STATE_RUNNING]); + } +} + +static void state_running_entry(void *o) +{ + int err; + + ARG_UNUSED(o); + + LOG_DBG("state_running_entry"); + + /* Subscribe to +CEREG notifications, level 5 */ + err = nrf_modem_at_printf("AT+CEREG=5"); + if (err) { + LOG_ERR("nrf_modem_at_printf, error: %d", err); + event_send(EVENT_ERROR); + return; + } + + lte_lc_register_handler(lte_handler); + + err = pdn_default_ctx_cb_reg(pdn_event_handler); + if (err) { + LOG_ERR("pdn_default_ctx_cb_reg, error: %d", err); + event_send(EVENT_ERROR); + return; + } +} + +static void state_running_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_running_run"); + + if (state_object->event == EVENT_NRF_MODEM_LIB_SHUTDOWN) { + smf_set_state(SMF_CTX(state_object), &states[STATE_WAITING_FOR_NRF_MODEM_LIB_INIT]); + } else if (state_object->event == EVENT_ERROR) { + smf_set_state(SMF_CTX(state_object), &states[STATE_ERROR]); + } +} + +static void state_running_exit(void *o) +{ + int err; + + struct fsm_state_object *state_object = o; + + LOG_DBG("state_running_exit"); + + err = lte_lc_deregister_handler(lte_handler); + if (err) { + LOG_ERR("lte_lc_deregister_handler, error: %d", err); + event_send(EVENT_ERROR); + return; + } + + err = pdn_default_ctx_cb_dereg(pdn_event_handler); + if (err) { + LOG_ERR("pdn_default_ctx_cb_dereg, error: %d", err); + event_send(EVENT_ERROR); + return; + } + + (void)k_work_cancel_delayable(&state_object->backoff_work); +} + +static void state_waiting_for_network_connection_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_waiting_for_network_connection_run"); + + switch (state_object->event) { + case EVENT_LTE_REGISTERED: + state_object->lte_registered = true; + break; + case EVENT_LTE_DEREGISTERED: + state_object->lte_registered = false; + break; + case EVENT_PDN_ACTIVATED: + state_object->pdn_active = true; + break; + case EVENT_PDN_DEACTIVATED: + state_object->pdn_active = false; + break; + default: + /* Don't care */ + break; + } + + if (state_object->lte_registered && state_object->pdn_active) { + smf_set_state(SMF_CTX(state_object), &states[STATE_NETWORK_CONNECTED]); + } +} + +static void state_network_connected_entry(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_network_connected_entry"); + + state_object->retry_count = 0; +} + +static void state_network_connected_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_network_connected_run"); + + if (state_object->event == EVENT_LTE_DEREGISTERED) { + state_object->lte_registered = false; + + smf_set_state(SMF_CTX(state_object), &states[STATE_WAITING_FOR_NETWORK_CONNECTION]); + } else if (state_object->event == EVENT_PDN_DEACTIVATED) { + state_object->pdn_active = false; + + smf_set_state(SMF_CTX(state_object), &states[STATE_WAITING_FOR_NETWORK_CONNECTION]); + } +} + +static void state_coredump_send_attempt_entry(void *o) +{ + int err; + + ARG_UNUSED(o); + + LOG_DBG("state_coredump_send_attempt_entry"); + LOG_DBG("Triggering heartbeat"); + LOG_DBG("Attempting to send coredump"); + + memfault_metrics_heartbeat_debug_trigger(); + + err = memfault_zephyr_port_post_data(); + if (err) { + LOG_DBG("Failed to post data to Memfault"); + event_send(EVENT_COREDUMP_SEND_FAIL); + } else { + LOG_DBG("Succeeded posting data to Memfault"); + event_send(EVENT_COREDUMP_SEND_SUCCESS); + } +} + +static void state_coredump_send_attempt_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_coredump_send_attempt_run"); + + if (state_object->event == EVENT_COREDUMP_SEND_FAIL) { + smf_set_state(SMF_CTX(state_object), &states[STATE_COREDUMP_SEND_BACKOFF]); + } else if (state_object->event == EVENT_COREDUMP_SEND_SUCCESS) { + smf_set_state(SMF_CTX(state_object), &states[STATE_FINISHED]); + } +} + +static void state_coredump_send_backoff_entry(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_coredump_send_backoff_entry"); + + schedule_send_backoff(state_object); +} + +static void state_coredump_send_backoff_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_coredump_send_backoff_run"); + + if (state_object->event == EVENT_BACKOFF_TIMER_EXPIRED) { + smf_set_state(SMF_CTX(state_object), &states[STATE_COREDUMP_SEND_ATTEMPT]); + } +} + +static void state_finished_entry(void *o) +{ + ARG_UNUSED(o); + + LOG_DBG("state_finished_entry"); +} + +static void state_error_entry(void *o) +{ + ARG_UNUSED(o); + + LOG_DBG("state_error_entry"); + + __ASSERT_NO_MSG(false); +} + +/* Local Convenience Functions */ + +static void event_send(enum library_event event) +{ + int ret; + enum library_event event_new = event; + + ret = k_msgq_put(&mflt_lte_coredump_queue, &event_new, K_NO_WAIT); + if (ret) { + LOG_ERR("k_msgq_put, error: %d", ret); + __ASSERT_NO_MSG(false); + return; + } +} + +static void schedule_send_backoff(struct fsm_state_object *state_object) +{ + state_object->retry_count++; + + if (state_object->retry_count >= CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX) { + event_send(STATE_FINISHED); + return; + } + + LOG_DBG("Retry number: %d", state_object->retry_count); + LOG_DBG("Scheduling backoff, next attempt in %d seconds", + CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS); + + (void)k_work_schedule(&state_object->backoff_work, + K_SECONDS(CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS)); +} + +/* Library thread */ + +static void mflt_lte_coredump_fn(void) +{ + int err; + + k_work_init_delayable(&state_object.backoff_work, timer_work_fn); + + bool has_coredump = memfault_coredump_has_valid_coredump(NULL); + + if (has_coredump) { + LOG_DBG("Coredump found"); + smf_set_initial(SMF_CTX(&state_object), + &states[STATE_WAITING_FOR_NRF_MODEM_LIB_INIT]); + } else { + LOG_DBG("No coredump found"); + smf_set_initial(SMF_CTX(&state_object), + &states[STATE_FINISHED]); + + return; + } + + while (1) { + err = k_msgq_get(&mflt_lte_coredump_queue, &state_object.event, K_FOREVER); + if (err) { + LOG_ERR("k_msgq_get, error: %d", err); + __ASSERT_NO_MSG(false); + return; + } + + err = smf_run_state(SMF_CTX(&state_object)); + if (err) { + LOG_ERR("smf_run_state, error: %d", err); + __ASSERT_NO_MSG(false); + return; + } + } +} + +K_THREAD_DEFINE(mflt_lte_coredump_thread_id, 4096, + mflt_lte_coredump_fn, NULL, NULL, NULL, + K_LOWEST_APPLICATION_THREAD_PRIO, 0, 0); diff --git a/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf b/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf index b7395a7e1d90..1b0e2ceca52f 100644 --- a/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf +++ b/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y CONFIG_MODEM_INFO=y diff --git a/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf b/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf index 8b9b2695cf42..53c1b66a3b7f 100644 --- a/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf +++ b/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y CONFIG_MODEM_INFO=y # Certificate management diff --git a/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf b/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf index 8b9b2695cf42..53c1b66a3b7f 100644 --- a/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf +++ b/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y CONFIG_MODEM_INFO=y # Certificate management diff --git a/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf b/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf index 4c6cad97c8bf..195b4b6e80c0 100644 --- a/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf +++ b/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y # Certificate management CONFIG_MEMFAULT_ROOT_CERT_STORAGE_NRF9160_MODEM=y diff --git a/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf b/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf index b7395a7e1d90..96fc888348b4 100644 --- a/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf +++ b/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf @@ -10,6 +10,10 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y + CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y CONFIG_MODEM_INFO=y diff --git a/samples/debug/memfault/prj.conf b/samples/debug/memfault/prj.conf index 33c199a1441b..e8ba4d94ca36 100644 --- a/samples/debug/memfault/prj.conf +++ b/samples/debug/memfault/prj.conf @@ -10,7 +10,7 @@ CONFIG_ASSERT=y # Logging CONFIG_LOG=y -CONFIG_LOG_MODE_IMMEDIATE=y +CONFIG_LOG_BUFFER_SIZE=4096 # For more verbose and detailed log output, set the log level to # CONFIG_MEMFAULT_SAMPLE_LOG_LEVEL_DBG=y instead. @@ -31,12 +31,17 @@ CONFIG_MEMFAULT_NCS_PROJECT_KEY="" CONFIG_MEMFAULT_SHELL=y CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD=y CONFIG_MEMFAULT_LOGGING_ENABLE=y +CONFIG_MEMFAULT_LOGGING_RAM_SIZE=4096 CONFIG_MEMFAULT_LOG_LEVEL_DBG=y +CONFIG_MEMFAULT_NCS_STACK_METRICS=y +CONFIG_MEMFAULT_COREDUMP_FULL_THREAD_STACKS=y +CONFIG_MEMFAULT_HEAP_STATS=y # Add to pick up support for sending Memfault data over HTTP CONFIG_MEMFAULT_HTTP_ENABLE=y # Store coredump to flash +CONFIG_MEMFAULT_NCS_LOG_LEVEL_DBG=y CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP=y CONFIG_MEMFAULT_COREDUMP_COLLECT_BSS_REGIONS=y diff --git a/samples/debug/memfault/src/main.c b/samples/debug/memfault/src/main.c index 0011da377c8d..6560f9f320fc 100644 --- a/samples/debug/memfault/src/main.c +++ b/samples/debug/memfault/src/main.c @@ -92,6 +92,11 @@ static void on_connect(void) LOG_INF("Time to connect: %d ms", time_to_lte_connection); #endif /* IS_ENABLED(MEMFAULT_NCS_LTE_METRICS) */ + if (IS_ENABLED(CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED)) { + /* Coredump sending handled internally */ + return; + } + LOG_INF("Sending already captured data to Memfault"); /* Trigger collection of heartbeat data. */