Terminal: change command-line parser (#2247)

Change the underlying command line handling:
- switch to a custom parser, inspired by redis / sds
- update terminalRegisterCommand signature, pass only bare minimum
- clean-up `help` & `commands`. update settings `set`, `get` and `del`
- allow our custom test suite to run command-line tests
- clean-up Stream IO to allow us to print large things into debug stream (for example, `eeprom.dump`)
- send parsing errors to the debug log

As a proof of concept, introduce `TERMINAL_MQTT_SUPPORT` and `TERMINAL_WEB_API_SUPPORT`
- MQTT subscribes to the `<root>/cmd/set` and sends response to the `<root>/cmd`. We can't output too much, as we don't have any large-send API.
- Web API listens to the `/api/cmd?apikey=...&line=...` (or PUT, params inside the body). This one is intended as a possible replacement of the `API_SUPPORT`. Internals introduce a 'task' around the AsyncWebServerRequest object that will simulate what WiFiClient does and push data into it continuously, switching between CONT and SYS.

Both are experimental. We only accept a single command and not every command is updated to use Print `ctx.output` object. We are also somewhat limited by the Print / Stream overall, perhaps I am overestimating the usefulness of Arduino compatibility to such an extent :)
Web API handler can also sometimes show only part of the result, whenever the command tries to yield() by itself waiting for something. Perhaps we would need to create a custom request handler for that specific use-case.

code/espurna/api.cpp

@@ -8,75 +8,34 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #include "api.h"
 
+// -----------------------------------------------------------------------------
+
 #if API_SUPPORT
 
 #include <vector>
 
 #include "system.h"
 #include "web.h"
 #include "rpc.h"
-#include "ws.h"
 
 struct web_api_t {
-    char * key;
-    api_get_callback_f getFn = NULL;
-    api_put_callback_f putFn = NULL;
+    explicit web_api_t(const String& key, api_get_callback_f getFn, api_put_callback_f putFn) :
+        key(key),
+        getFn(getFn),
+        putFn(putFn)
+    {}
+    web_api_t() = delete;
+
+    const String key;
+    api_get_callback_f getFn;
+    api_put_callback_f putFn;
 };
 std::vector<web_api_t> _apis;
 
-// -----------------------------------------------------------------------------
-
-bool _apiEnabled() {
-    return getSetting("apiEnabled", 1 == API_ENABLED);
-}
-
-bool _apiRestFul() {
-    return getSetting("apiRestFul", 1 == API_RESTFUL);
-}
-
-String _apiKey() {
-    return getSetting("apiKey", API_KEY);
-}
-
-bool _apiWebSocketOnKeyCheck(const char * key, JsonVariant& value) {
-    return (strncmp(key, "api", 3) == 0);
-}
-
-void _apiWebSocketOnConnected(JsonObject& root) {
-    root["apiEnabled"] = _apiEnabled();
-    root["apiKey"] = _apiKey();
-    root["apiRestFul"] = _apiRestFul();
-    root["apiRealTime"] = getSetting("apiRealTime", 1 == API_REAL_TIME_VALUES);
-}
-
-void _apiConfigure() {
-    // Nothing to do
-}
-
 // -----------------------------------------------------------------------------
 // API
 // -----------------------------------------------------------------------------
 
-bool _authAPI(AsyncWebServerRequest *request) {
-
-    const auto key = _apiKey();
-    if (!key.length() || !_apiEnabled()) {
-        DEBUG_MSG_P(PSTR("[WEBSERVER] HTTP API is not enabled\n"));
-        request->send(403);
-        return false;
-    }
-
-    AsyncWebParameter* keyParam = request->getParam("apikey", (request->method() == HTTP_PUT));
-    if (!keyParam || !keyParam->value().equals(key)) {
-        DEBUG_MSG_P(PSTR("[WEBSERVER] Wrong / missing apikey parameter\n"));
-        request->send(403);
-        return false;
-    }
-
-    return true;
-
-}
-
 bool _asJson(AsyncWebServerRequest *request) {
     bool asJson = false;
     if (request->hasHeader("Accept")) {
@@ -102,19 +61,18 @@ void _onAPIsText(AsyncWebServerRequest *request) {
     request->send(response);
 }
 
-constexpr const size_t API_JSON_BUFFER_SIZE = 1024;
+constexpr size_t ApiJsonBufferSize = 1024;
 
 void _onAPIsJson(AsyncWebServerRequest *request) {
 
-
-    DynamicJsonBuffer jsonBuffer(API_JSON_BUFFER_SIZE);
+    DynamicJsonBuffer jsonBuffer(ApiJsonBufferSize);
     JsonObject& root = jsonBuffer.createObject();
 
     constexpr const int BUFFER_SIZE = 48;
 
     for (unsigned int i=0; i < _apis.size(); i++) {
         char buffer[BUFFER_SIZE] = {0};
-        int res = snprintf(buffer, sizeof(buffer), "/api/%s", _apis[i].key);
+        int res = snprintf(buffer, sizeof(buffer), "/api/%s", _apis[i].key.c_str());
         if ((res < 0) || (res > (BUFFER_SIZE - 1))) {
             request->send(500);
             return;
@@ -130,7 +88,7 @@ void _onAPIsJson(AsyncWebServerRequest *request) {
 void _onAPIs(AsyncWebServerRequest *request) {
 
     webLog(request);
-    if (!_authAPI(request)) return;
+    if (!apiAuthenticate(request)) return;
 
     bool asJson = _asJson(request);
 
@@ -146,7 +104,7 @@ void _onAPIs(AsyncWebServerRequest *request) {
 void _onRPC(AsyncWebServerRequest *request) {
 
     webLog(request);
-    if (!_authAPI(request)) return;
+    if (!apiAuthenticate(request)) return;
 
     //bool asJson = _asJson(request);
     int response = 404;
@@ -187,19 +145,18 @@ bool _apiRequestCallback(AsyncWebServerRequest *request) {
     // Not API request
     if (!url.startsWith("/api/")) return false;
 
-    for (unsigned char i=0; i < _apis.size(); i++) {
+    for (auto& api : _apis) {
 
-        // Search API url
-        web_api_t api = _apis[i];
+        // Search API url for the exact match
         if (!url.endsWith(api.key)) continue;
 
         // Log and check credentials
         webLog(request);
-        if (!_authAPI(request)) return false;
+        if (!apiAuthenticate(request)) return false;
 
         // Check if its a PUT
         if (api.putFn != NULL) {
-            if (!_apiRestFul() || (request->method() == HTTP_PUT)) {
+            if (!apiRestFul() || (request->method() == HTTP_PUT)) {
                 if (request->hasParam("value", request->method() == HTTP_PUT)) {
                     AsyncWebParameter* p = request->getParam("value", request->method() == HTTP_PUT);
                     (api.putFn)((p->value()).c_str());
@@ -224,9 +181,9 @@ bool _apiRequestCallback(AsyncWebServerRequest *request) {
         if (_asJson(request)) {
             char buffer[64];
             if (isNumber(value)) {
-                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": %s }"), api.key, value);
+                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": %s }"), api.key.c_str(), value);
             } else {
-                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": \"%s\" }"), api.key, value);
+                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": \"%s\" }"), api.key.c_str(), value);
             }
             request->send(200, "application/json", buffer);
         } else {
@@ -243,25 +200,13 @@ bool _apiRequestCallback(AsyncWebServerRequest *request) {
 
 // -----------------------------------------------------------------------------
 
-void apiRegister(const char * key, api_get_callback_f getFn, api_put_callback_f putFn) {
-
-    // Store it
-    web_api_t api;
-    api.key = strdup(key);
-    api.getFn = getFn;
-    api.putFn = putFn;
-    _apis.push_back(api);
-
+void apiRegister(const String& key, api_get_callback_f getFn, api_put_callback_f putFn) {
+    _apis.emplace_back(key, std::move(getFn), std::move(putFn));
 }
 
 void apiSetup() {
-    _apiConfigure();
-    wsRegister()
-        .onVisible([](JsonObject& root) { root["apiVisible"] = 1; })
-        .onConnected(_apiWebSocketOnConnected)
-        .onKeyCheck(_apiWebSocketOnKeyCheck);
     webRequestRegister(_apiRequestCallback);
-    espurnaRegisterReload(_apiConfigure);
 }
 
 #endif // API_SUPPORT
+

code/espurna/api.h

@@ -11,18 +11,28 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include "espurna.h"
 #include "web.h"
 
-#include <functional>
+#if WEB_SUPPORT
+
+bool apiAuthenticate(AsyncWebServerRequest*);
+bool apiEnabled();
+bool apiRestFul();
+String apiKey();
+
+#endif // WEB_SUPPORT == 1
 
 #if WEB_SUPPORT && API_SUPPORT
 
+#include <functional>
+
 #include <ESPAsyncTCP.h>
 #include <ArduinoJson.h>
 
 using api_get_callback_f = std::function<void(char * buffer, size_t size)>;
 using api_put_callback_f = std::function<void(const char * payload)> ;
 
-void apiRegister(const char * key, api_get_callback_f getFn, api_put_callback_f putFn = nullptr);
+void apiRegister(const String& key, api_get_callback_f getFn, api_put_callback_f putFn = nullptr);
 
+void apiCommonSetup();
 void apiSetup();
 
 #endif // API_SUPPORT == 1

code/espurna/api_common.cpp

@@ -0,0 +1,79 @@
+/*
+
+Part of the API MODULE
+
+Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#include "espurna.h"
+
+#include "api.h"
+
+#include "ws.h"
+#include "web.h"
+
+// -----------------------------------------------------------------------------
+
+#if WEB_SUPPORT
+
+namespace {
+
+bool _apiWebSocketOnKeyCheck(const char * key, JsonVariant& value) {
+    return (strncmp(key, "api", 3) == 0);
+}
+
+void _apiWebSocketOnConnected(JsonObject& root) {
+    root["apiEnabled"] = apiEnabled();
+    root["apiKey"] = apiKey();
+    root["apiRestFul"] = apiRestFul();
+    root["apiRealTime"] = getSetting("apiRealTime", 1 == API_REAL_TIME_VALUES);
+}
+
+}
+
+// -----------------------------------------------------------------------------
+// Public API
+// -----------------------------------------------------------------------------
+
+bool apiEnabled() {
+    return getSetting("apiEnabled", 1 == API_ENABLED);
+}
+
+bool apiRestFul() {
+    return getSetting("apiRestFul", 1 == API_RESTFUL);
+}
+
+String apiKey() {
+    return getSetting("apiKey", API_KEY);
+}
+
+bool apiAuthenticate(AsyncWebServerRequest *request) {
+
+    const auto key = apiKey();
+    if (!apiEnabled() || !key.length()) {
+        DEBUG_MSG_P(PSTR("[WEBSERVER] HTTP API is not enabled\n"));
+        request->send(403);
+        return false;
+    }
+
+    AsyncWebParameter* keyParam = request->getParam("apikey", (request->method() == HTTP_PUT));
+    if (!keyParam || !keyParam->value().equals(key)) {
+        DEBUG_MSG_P(PSTR("[WEBSERVER] Wrong / missing apikey parameter\n"));
+        request->send(403);
+        return false;
+    }
+
+    return true;
+
+}
+
+void apiCommonSetup() {
+    wsRegister()
+        .onVisible([](JsonObject& root) { root["apiVisible"] = 1; })
+        .onConnected(_apiWebSocketOnConnected)
+        .onKeyCheck(_apiWebSocketOnKeyCheck);
+}
+
+#endif // WEB_SUPPORT == 1

code/espurna/config/dependencies.h

@@ -121,6 +121,20 @@
 #define RELAY_SUPPORT               1           // Most of the time we require it
 #endif
 
+#if TERMINAL_WEB_API_SUPPORT
+#undef TERMINAL_SUPPORT
+#define TERMINAL_SUPPORT            1           // Need terminal command line parser and commands
+#undef WEB_SUPPORT
+#define WEB_SUPPORT                 1           // Registered as web server request handler
+#endif
+
+#if TERMINAL_MQTT_SUPPORT
+#undef TERMINAL_SUPPORT
+#define TERMINAL_SUPPORT            1           // Need terminal command line parser and commands
+#undef MQTT_SUPPORT
+#define MQTT_SUPPORT                1           // Subscribe and publish things
+#endif
+
 //------------------------------------------------------------------------------
 // Hint about ESPAsyncTCP options and our internal one
 // TODO: clean-up SSL_ENABLED and USE_SSL settings for 1.15.0

code/espurna/config/general.h

@@ -175,7 +175,23 @@
 #define TERMINAL_SUPPORT         1              // Enable terminal commands (0.97Kb)
 #endif
 
-#define TERMINAL_BUFFER_SIZE     128            // Max size for commands commands
+#ifndef TERMINAL_SHARED_BUFFER_SIZE
+#define TERMINAL_SHARED_BUFFER_SIZE     128     // Maximum size for command line, shared by the WebUI, Telnet and Serial
+#endif
+
+#ifndef TERMINAL_MQTT_SUPPORT
+#define TERMINAL_MQTT_SUPPORT       0       // MQTT Terminal support built in
+                                            // Depends on MQTT_SUPPORT and TERMINAL_SUPPORT commands being available
+#endif
+
+#ifndef TERMINAL_WEB_API_SUPPORT
+#define TERMINAL_WEB_API_SUPPORT    0       // Web server API Terminal support built in
+                                            // Depends on WEB_SUPPORT and TERMINAL_SUPPORT commands being available
+#endif
+
+#ifndef TERMINAL_WEB_API_PATH
+#define TERMINAL_WEB_API_PATH       "/api/cmd"
+#endif
 
 //------------------------------------------------------------------------------
 // SYSTEM CHECK
@@ -768,7 +784,6 @@
 #define API_REAL_TIME_VALUES        0           // Show filtered/median values by default (0 => median, 1 => real time)
 #endif
 
-
 // -----------------------------------------------------------------------------
 // MDNS / LLMNR / NETBIOS / SSDP
 // -----------------------------------------------------------------------------
@@ -1163,6 +1178,7 @@
 #define MQTT_TOPIC_OTA              "ota"
 #define MQTT_TOPIC_TELNET_REVERSE   "telnet_reverse"
 #define MQTT_TOPIC_CURTAIN          "curtain"
+#define MQTT_TOPIC_CMD              "cmd"
 
 // Light module
 #define MQTT_TOPIC_CHANNEL          "channel"

code/espurna/crash.cpp

@@ -142,7 +142,7 @@ void crashDump() {
 void crashSetup() {
 
     #if TERMINAL_SUPPORT
-        terminalRegisterCommand(F("CRASH"), [](Embedis* e) {
+        terminalRegisterCommand(F("CRASH"), [](const terminal::CommandContext&) {
             crashDump();
             crashClear();
             terminalOK();

code/espurna/debug.cpp

@@ -63,6 +63,11 @@ void _debugSend(const char * format, va_list args) {
 
 }
 
+void debugSendRaw(const char* line, bool timestamp) {
+    if (!_debug_enabled) return;
+    _debugSendInternal(line, timestamp);
+}
+
 void debugSend(const char* format, ...) {
 
     if (!_debug_enabled) return;
@@ -263,7 +268,7 @@ void debugSetup() {
 
     #if DEBUG_LOG_BUFFER_SUPPORT
 
-        terminalRegisterCommand(F("DEBUG.BUFFER"), [](Embedis* e) {
+        terminalRegisterCommand(F("DEBUG.BUFFER"), [](const terminal::CommandContext&) {
             _debug_log_buffer_enabled = false;
             if (!_debug_log_buffer.size()) {
                 DEBUG_MSG_P(PSTR("[DEBUG] Buffer is empty\n"));