From d62234ac65be842fbf84951f45870efcbcc882a9 Mon Sep 17 00:00:00 2001 From: Moritz Lerch Date: Tue, 2 Jan 2024 23:40:21 +0100 Subject: [PATCH 01/30] webapp: add missing button spacing --- webapp/src/views/DeviceAdminView.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/views/DeviceAdminView.vue b/webapp/src/views/DeviceAdminView.vue index 1f350993a..1be7a72a0 100644 --- a/webapp/src/views/DeviceAdminView.vue +++ b/webapp/src/views/DeviceAdminView.vue @@ -39,7 +39,7 @@
-
+
{{ doc.name }}
From f5c69060f564a17220bc81dbf2fa000c3a578395 Mon Sep 17 00:00:00 2001 From: Fribur Date: Thu, 4 Jan 2024 16:20:32 -0500 Subject: [PATCH 02/30] re-factoring of HttpPowerMeter Added ability to deal with local host names (mDNS), remove use of FirebasedJson to save ~20kB build size, some changes to PowerLimiter to avoid setting new inverter power limits when not needed (=current limit as reported by inverter is within hysteresis) --- include/HttpPowerMeter.h | 26 ++-- src/HttpPowerMeter.cpp | 248 +++++++++++++++++++------------------- src/PowerLimiter.cpp | 30 +++-- src/WebApi_powermeter.cpp | 38 +++--- 4 files changed, 181 insertions(+), 161 deletions(-) diff --git a/include/HttpPowerMeter.h b/include/HttpPowerMeter.h index aff05e64e..16ee85851 100644 --- a/include/HttpPowerMeter.h +++ b/include/HttpPowerMeter.h @@ -10,17 +10,23 @@ class HttpPowerMeterClass { void init(); bool updateValues(); float getPower(int8_t phase); - bool httpRequest(const char* url, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, - char* response, size_t responseSize, char* error, size_t errorSize); - float getFloatValueByJsonPath(const char* jsonString, const char* jsonPath, float &value); + char httpPowerMeterError[256]; + bool queryPhase(int phase, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password, + const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath); + void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri); -private: - void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri); - void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue); - HTTPClient httpClient; - float power[POWERMETER_MAX_PHASES]; - String sha256(const String& data); - +private: + float power[POWERMETER_MAX_PHASES]; + HTTPClient httpClient; + String httpResponse; + bool httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& urlUri, Auth authType, const char* username, + const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath); + String extractParam(String& authReq, const String& param, const char delimit); + void getcNonce(char* cNounce); + String getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter); + bool tryGetFloatValueForPhase(int phase, int httpCode, const char* jsonPath); + void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue); + String sha256(const String& data); }; extern HttpPowerMeterClass HttpPowerMeter; diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index 09da47a53..b0ab68207 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -3,11 +3,12 @@ #include "HttpPowerMeter.h" #include "MessageOutput.h" #include -#include +#include //saves 20kB to not use FirebaseJson as ArduinoJson is used already elsewhere (e.g. in WebApi_powermeter) #include #include #include #include +#include void HttpPowerMeterClass::init() { @@ -20,13 +21,14 @@ float HttpPowerMeterClass::getPower(int8_t phase) bool HttpPowerMeterClass::updateValues() { - const CONFIG_T& config = Configuration.get(); - - char response[2000], - errorMessage[256]; + const CONFIG_T& config = Configuration.get(); for (uint8_t i = 0; i < POWERMETER_MAX_PHASES; i++) { POWERMETER_HTTP_PHASE_CONFIG_T phaseConfig = config.PowerMeter.Http_Phase[i]; + String urlProtocol; + String urlHostname; + String urlUri; + extractUrlComponents(phaseConfig.Url, urlProtocol, urlHostname, urlUri); if (!phaseConfig.Enabled) { power[i] = 0.0; @@ -34,33 +36,42 @@ bool HttpPowerMeterClass::updateValues() } if (i == 0 || config.PowerMeter.HttpIndividualRequests) { - if (httpRequest(phaseConfig.Url, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout, - response, sizeof(response), errorMessage, sizeof(errorMessage))) { - if (!getFloatValueByJsonPath(response, phaseConfig.JsonPath, power[i])) { - MessageOutput.printf("[HttpPowerMeter] Couldn't find a value with Json query \"%s\"\r\n", phaseConfig.JsonPath); - return false; - } - } else { - MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d failed. Error: %s\r\n", - i + 1, errorMessage); + if (!queryPhase(i, urlProtocol, urlHostname, urlUri, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout, + phaseConfig.JsonPath)) { + MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d failed.\r\n", i + 1); + MessageOutput.printf("%s\r\n", httpPowerMeterError); return false; } } } - return true; } -bool HttpPowerMeterClass::httpRequest(const char* url, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, - char* response, size_t responseSize, char* error, size_t errorSize) +bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password, + const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath) { - String urlProtocol; - String urlHostname; - String urlUri; - extractUrlComponents(url, urlProtocol, urlHostname, urlUri); + //hostByName in WiFiGeneric fails to resolve local names. issue described in + //https://github.com/espressif/arduino-esp32/issues/3822 + //and in depth analyzed in https://github.com/espressif/esp-idf/issues/2507#issuecomment-761836300 + //in conclusion: we cannot rely on httpClient.begin(*wifiClient, url) to resolve IP adresses. + //have to do it manually here. Feels Hacky... + IPAddress ipaddr((uint32_t)0); + //first check if the urlHostname is already an IP adress + if (!ipaddr.fromString(urlHostname)) + { + //no it is not, so try to resolve the IP adress + const bool mdnsEnabled = Configuration.get().Mdns.Enabled; + if (!mdnsEnabled) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Enable mDNS in Network Settings")); + return false; + } - response[0] = '\0'; - error[0] = '\0'; + ipaddr = MDNS.queryHost(urlHostname); + if (ipaddr == INADDR_NONE){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s"), urlHostname.c_str()); + return false; + } + } // secureWifiClient MUST be created before HTTPClient // see discussion: https://github.com/helgeerbe/OpenDTU-OnBattery/issues/381 @@ -73,14 +84,21 @@ bool HttpPowerMeterClass::httpRequest(const char* url, Auth authType, const char } else { wifiClient = std::make_unique(); } - - - if (!httpClient.begin(*wifiClient, url)) { - snprintf_P(error, errorSize, "httpClient.begin(%s) failed", url); - return false; + return httpRequest(phase, *wifiClient, urlProtocol, ipaddr.toString(), uri, authType, username, password, httpHeader, httpValue, timeout, jsonPath); +} +bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, + const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath) +{ + int port = 80; + if (urlProtocol == "https") { + port = 443; + } + if(!httpClient.begin(wifiClient, urlHostname, port, uri)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), urlProtocol.c_str(), urlHostname.c_str()); + return false; } - prepareRequest(timeout, httpHeader, httpValue); - + + prepareRequest(timeout, httpHeader, httpValue); if (authType == Auth::digest) { const char *headers[1] = {"WWW-Authenticate"}; httpClient.collectHeaders(headers, 1); @@ -92,111 +110,98 @@ bool HttpPowerMeterClass::httpRequest(const char* url, Auth authType, const char auth.concat(base64::encode(authString)); httpClient.addHeader("Authorization", auth); } - int httpCode = httpClient.GET(); + if (httpCode == HTTP_CODE_UNAUTHORIZED && authType == Auth::digest) { // Handle authentication challenge - char realm[256]; // Buffer to store the realm received from the server - char nonce[256]; // Buffer to store the nonce received from the server if (httpClient.hasHeader("WWW-Authenticate")) { - String authHeader = httpClient.header("WWW-Authenticate"); - if (authHeader.indexOf("Digest") != -1) { - int realmIndex = authHeader.indexOf("realm=\""); - int nonceIndex = authHeader.indexOf("nonce=\""); - if (realmIndex != -1 && nonceIndex != -1) { - int realmEndIndex = authHeader.indexOf("\"", realmIndex + 7); - int nonceEndIndex = authHeader.indexOf("\"", nonceIndex + 7); - if (realmEndIndex != -1 && nonceEndIndex != -1) { - authHeader.substring(realmIndex + 7, realmEndIndex).toCharArray(realm, sizeof(realm)); - authHeader.substring(nonceIndex + 7, nonceEndIndex).toCharArray(nonce, sizeof(nonce)); - } - } - String cnonce = String(random(1000)); // Generate client nonce - String str = username; - str += ":"; - str += realm; - str += ":"; - str += password; - String ha1 = sha256(str); - str = "GET:"; - str += urlUri; - String ha2 = sha256(str); - str = ha1; - str += ":"; - str += nonce; - str += ":00000001:"; - str += cnonce; - str += ":auth:"; - str += ha2; - String response = sha256(str); - - String authorization = "Digest username=\""; - authorization += username; - authorization += "\", realm=\""; - authorization += realm; - authorization += "\", nonce=\""; - authorization += nonce; - authorization += "\", uri=\""; - authorization += urlUri; - authorization += "\", cnonce=\""; - authorization += cnonce; - authorization += "\", nc=00000001, qop=auth, response=\""; - authorization += response; - authorization += "\", algorithm=SHA-256"; - httpClient.end(); - if (!httpClient.begin(*wifiClient, url)) { - snprintf_P(error, errorSize, "httpClient.begin(%s) for digest auth failed", url); - return false; - } - prepareRequest(timeout, httpHeader, httpValue); - httpClient.addHeader("Authorization", authorization); - httpCode = httpClient.GET(); + String authReq = httpClient.header("WWW-Authenticate"); + String authorization = getDigestAuth(authReq, String(username), String(password), "GET", String(uri), 1); + httpClient.end(); + if(!httpClient.begin(wifiClient, urlHostname, port, uri)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s using digest auth"), urlProtocol.c_str(), urlHostname.c_str()); + return false; } - } - } - - if (httpCode == HTTP_CODE_OK) { - String responseBody = httpClient.getString(); - if (responseBody.length() > (responseSize - 1)) { - snprintf_P(error, errorSize, "Response too large! Response length: %d Body start: %s", - httpClient.getSize(), responseBody.c_str()); - } else { - snprintf(response, responseSize, responseBody.c_str()); + prepareRequest(timeout, httpHeader, httpValue); + httpClient.addHeader("Authorization", authorization); + httpCode = httpClient.GET(); } - } else if (httpCode <= 0) { - snprintf_P(error, errorSize, "Error(%s): %s", url, httpClient.errorToString(httpCode).c_str()); - } else if (httpCode != HTTP_CODE_OK) { - snprintf_P(error, errorSize, "Bad HTTP code: %d", httpCode); } - + bool result = tryGetFloatValueForPhase(phase, httpCode, jsonPath); httpClient.end(); - - if (error[0] != '\0') { - return false; - } - - return true; + return result; } -float HttpPowerMeterClass::getFloatValueByJsonPath(const char* jsonString, const char* jsonPath, float& value) -{ - FirebaseJson firebaseJson; - firebaseJson.setJsonData(jsonString); - - FirebaseJsonData firebaseJsonResult; - if (!firebaseJson.get(firebaseJsonResult, jsonPath)) { - return false; - } - - value = firebaseJsonResult.to(); +String HttpPowerMeterClass::extractParam(String& authReq, const String& param, const char delimit) { + int _begin = authReq.indexOf(param); + if (_begin == -1) { return ""; } + return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length())); +} +void HttpPowerMeterClass::getcNonce(char* cNounce) { + static const char alphanum[] = "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + auto len=sizeof(cNounce); - firebaseJson.clear(); + for (int i = 0; i < len; ++i) { cNounce[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; } - return true; } - - void HttpPowerMeterClass::extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri) { +String HttpPowerMeterClass::getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter) { + // extracting required parameters for RFC 2069 simpler Digest + String realm = extractParam(authReq, "realm=\"", '"'); + String nonce = extractParam(authReq, "nonce=\"", '"'); + char cNonce[8]; + getcNonce(cNonce); + + char nc[9]; + snprintf(nc, sizeof(nc), "%08x", counter); + + // parameters for the Digest + // sha256 of the user:realm:user + char h1Prep[sizeof(username)+sizeof(realm)+sizeof(password)+2]; + snprintf(h1Prep, sizeof(h1Prep), "%s:%s:%s", username.c_str(),realm.c_str(), password.c_str()); + String ha1 = sha256(h1Prep); + + //sha256 of method:uri + char h2Prep[sizeof(method) + sizeof(uri) + 1]; + snprintf(h2Prep, sizeof(h2Prep), "%s:%s", method.c_str(),uri.c_str()); + String ha2 = sha256(h2Prep); + + //md5 of h1:nonce:nc:cNonce:auth:h2 + char responsePrep[sizeof(ha1)+sizeof(nc)+sizeof(cNonce)+4+sizeof(ha2) + 5]; + snprintf(responsePrep, sizeof(responsePrep), "%s:%s:%s:%s:auth:%s", ha1.c_str(),nonce.c_str(), nc, cNonce,ha2.c_str()); + String response = sha256(responsePrep); + + //Final authorization String; + char authorization[17 + sizeof(username) + 10 + sizeof(realm) + 10 + sizeof(nonce) + 8 + sizeof(uri) + 34 + sizeof(nc) + 10 + sizeof(cNonce) + 13 + sizeof(response)]; + snprintf(authorization, sizeof(authorization), "Digest username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", algorithm=SHA-256, qop=auth, nc=%s, cnonce=\"%s\", response=\"%s\"", username.c_str(), realm.c_str(), nonce.c_str(), uri.c_str(), nc, cNonce, response.c_str()); + + return authorization; +} +bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, int httpCode, const char* jsonPath) +{ + bool success = false; + if (httpCode == HTTP_CODE_OK) { + httpResponse = httpClient.getString(); //very unfortunate that we cannot parse WifiClient stream directly + StaticJsonDocument<2048> json; //however creating these allocations on stack should be fine to avoid heap fragmentation + deserializeJson(json, httpResponse); + if(!json.containsKey(jsonPath)) + { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("[HttpPowerMeter] Couldn't find a value for phase %i with Json query \"%s\""), phase, jsonPath); + }else { + power[phase] = json[jsonPath].as(); + //MessageOutput.printf("Power for Phase %i: %5.2fW\r\n", phase, power[phase]); + success = true; + } + } else if (httpCode <= 0) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("HTTP Error %s"), httpClient.errorToString(httpCode).c_str()); + } else if (httpCode != HTTP_CODE_OK) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Bad HTTP code: %d"), httpCode); + } + return success; +} +void HttpPowerMeterClass::extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri) { // Find protocol delimiter int protocolEndIndex = url.indexOf(":"); if (protocolEndIndex != -1) { @@ -247,7 +252,6 @@ String HttpPowerMeterClass::sha256(const String& data) { return hashStr; } - void HttpPowerMeterClass::prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue) { httpClient.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); httpClient.setUserAgent("OpenDTU-OnBattery"); diff --git a/src/PowerLimiter.cpp b/src/PowerLimiter.cpp index 86cb2751b..450555618 100644 --- a/src/PowerLimiter.cpp +++ b/src/PowerLimiter.cpp @@ -187,7 +187,8 @@ void PowerLimiterClass::loop() // a calculated power limit will always be limited to the reported // device's max power. that upper limit is only known after the first // DevInfoSimpleCommand succeeded. - if (_inverter->DevInfo()->getMaxPower() <= 0) { + auto maxPower = _inverter->DevInfo()->getMaxPower(); + if (maxPower <= 0) { return announceStatus(Status::InverterDevInfoPending); } @@ -199,12 +200,13 @@ void PowerLimiterClass::loop() // the normal mode of operation requires a valid // power meter reading to calculate a power limit if (!config.PowerMeter.Enabled) { - shutdown(Status::PowerMeterDisabled); + //instead of shutting down completelty, how about setting alternativly to a save "low production" mode? + //Could be usefull when PowerMeter fails but we know for sure house consumption will never fall below a certain limit (say 200W) + shutdown(Status::PowerMeterDisabled); return; } if (millis() - PowerMeter.getLastPowerMeterUpdate() > (30 * 1000)) { - shutdown(Status::PowerMeterTimeout); return; } @@ -222,6 +224,7 @@ void PowerLimiterClass::loop() if (_inverter->Statistics()->getLastUpdate() <= settlingEnd) { return announceStatus(Status::InverterStatsPending); } + if (PowerMeter.getLastPowerMeterUpdate() <= settlingEnd) { return announceStatus(Status::PowerMeterPending); @@ -545,8 +548,8 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr inver dcTotalChnls, dcProdChnls); effPowerLimit = round(effPowerLimit * static_cast(dcTotalChnls) / dcProdChnls); } - - effPowerLimit = std::min(effPowerLimit, inverter->DevInfo()->getMaxPower()); + auto maxPower = inverter->DevInfo()->getMaxPower(); + effPowerLimit = std::min(effPowerLimit, maxPower); // Check if the new value is within the limits of the hysteresis auto diff = std::abs(effPowerLimit - _lastRequestedPowerLimit); @@ -556,16 +559,23 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr inver // staleness in case a power limit update was not received by the inverter. auto ageMillis = millis() - _lastPowerLimitMillis; - if (diff < hysteresis && ageMillis < 60 * 1000) { + //instead pushing limit to inverter every 60 seconds no matter what, + //why not query instead the currenty configured limit...and do nothing if not needed + int currentLimit = round(inverter->SystemConfigPara()->getLimitPercent() * maxPower / 100); + auto currentDiff = std::abs(effPowerLimit - currentLimit ); + + if (diff < hysteresis && currentDiff < hysteresis ){ + //if (diff < hysteresis && ageMillis < 60 * 1000) { + //MessageOutput.printf("Keep limit: %d W, current limit %d W\r\n", effPowerLimit, currentLimit); if (_verboseLogging) { - MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, last limit: %d W, diff: %d W, hysteresis: %d W, age: %ld ms\r\n", - newPowerLimit, _lastRequestedPowerLimit, diff, hysteresis, ageMillis); + MessageOutput.printf("[DPL::setNewPowerLimit] Keep current limit. (new calculated: %d W, last limit: %d W, diff: %d W, hysteresis: %d W, age: %ld ms)\r\n", + effPowerLimit, _lastRequestedPowerLimit, diff, hysteresis, ageMillis); } return false; } - + //if we end up here, it we will set new limit if (_verboseLogging) { - MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, (re-)sending limit: %d W\r\n", + MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, sending limit: %d W\r\n", newPowerLimit, effPowerLimit); } diff --git a/src/WebApi_powermeter.cpp b/src/WebApi_powermeter.cpp index 383af5b41..83b2eda5c 100644 --- a/src/WebApi_powermeter.cpp +++ b/src/WebApi_powermeter.cpp @@ -203,10 +203,11 @@ void WebApiPowerMeterClass::onAdminPost(AsyncWebServerRequest* request) response->setLength(); request->send(response); - yield(); - delay(1000); - yield(); - ESP.restart(); + // why reboot..WebApi_powerlimiter is also not rebooting + // yield(); + // delay(1000); + // yield(); + // ESP.restart(); } void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request) @@ -254,25 +255,24 @@ void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request) return; } - char powerMeterResponse[2000], - errorMessage[256]; - char response[200]; - if (HttpPowerMeter.httpRequest(root[F("url")].as().c_str(), + char response[256]; + + String urlProtocol; + String urlHostname; + String urlUri; + + HttpPowerMeter.extractUrlComponents(root[F("url")].as().c_str(), urlProtocol, urlHostname, urlUri); + + int phase = 0;//"absuing" index 0 of the float power[3] in HttpPowerMeter to store the result + if (HttpPowerMeter.queryPhase(phase, urlProtocol, urlHostname, urlUri, root[F("auth_type")].as(), root[F("username")].as().c_str(), root[F("password")].as().c_str(), root[F("header_key")].as().c_str(), root[F("header_value")].as().c_str(), root[F("timeout")].as(), - powerMeterResponse, sizeof(powerMeterResponse), errorMessage, sizeof(errorMessage))) { - float power; - - if (HttpPowerMeter.getFloatValueByJsonPath(powerMeterResponse, - root[F("json_path")].as().c_str(), power)) { - retMsg[F("type")] = F("success"); - snprintf_P(response, sizeof(response), "Success! Power: %5.2fW", power); - } else { - snprintf_P(response, sizeof(response), "Error: Could not find value for JSON path!"); - } + root[F("json_path")].as().c_str())) { + retMsg[F("type")] = F("success"); + snprintf_P(response, sizeof(response), "Success! Power: %5.2fW", HttpPowerMeter.getPower(phase + 1)); } else { - snprintf_P(response, sizeof(response), errorMessage); + snprintf_P(response, sizeof(response), "%s", HttpPowerMeter.httpPowerMeterError); } retMsg[F("message")] = F(response); From d5eba2392c6e8b6a8781c747e994ef6074e7d3ff Mon Sep 17 00:00:00 2001 From: Fribur Date: Thu, 4 Jan 2024 16:32:42 -0500 Subject: [PATCH 03/30] fixed long/float parsing bug --- src/HttpPowerMeter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index b0ab68207..8bd1fa14b 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -190,7 +190,7 @@ bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, int httpCode, cons { snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("[HttpPowerMeter] Couldn't find a value for phase %i with Json query \"%s\""), phase, jsonPath); }else { - power[phase] = json[jsonPath].as(); + power[phase] = json[jsonPath].as(); //MessageOutput.printf("Power for Phase %i: %5.2fW\r\n", phase, power[phase]); success = true; } From bc38ce344fc416a8a91868549e54369f11864191 Mon Sep 17 00:00:00 2001 From: Fribur Date: Thu, 4 Jan 2024 18:22:58 -0500 Subject: [PATCH 04/30] remove FirebaseJson from platfromio.ini, fix unintended change in PowerLimiter --- platformio.ini | 1 - src/PowerLimiter.cpp | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio.ini b/platformio.ini index d7f10e766..11377d008 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,6 @@ lib_deps = https://github.com/arkhipenko/TaskScheduler#testing https://github.com/coryjfowler/MCP_CAN_lib plerup/EspSoftwareSerial@^8.0.1 - mobizt/FirebaseJson @ ^3.0.6 rweather/Crypto@^0.4.0 extra_scripts = diff --git a/src/PowerLimiter.cpp b/src/PowerLimiter.cpp index 450555618..56d732570 100644 --- a/src/PowerLimiter.cpp +++ b/src/PowerLimiter.cpp @@ -199,14 +199,15 @@ void PowerLimiterClass::loop() // the normal mode of operation requires a valid // power meter reading to calculate a power limit - if (!config.PowerMeter.Enabled) { - //instead of shutting down completelty, how about setting alternativly to a save "low production" mode? - //Could be usefull when PowerMeter fails but we know for sure house consumption will never fall below a certain limit (say 200W) + if (!config.PowerMeter.Enabled) { shutdown(Status::PowerMeterDisabled); return; } + //instead of shutting down on PowerMeterTimeout, how about setting alternativly to a safe "low production" mode? + //Could be usefull when PowerMeter fails but we know for sure house consumption will never fall below a certain limit (say 200W) if (millis() - PowerMeter.getLastPowerMeterUpdate() > (30 * 1000)) { + shutdown(Status::PowerMeterTimeout); return; } From 9ed5a7881899bbf2fb714610e26e4385959720d9 Mon Sep 17 00:00:00 2001 From: Fribur Date: Fri, 5 Jan 2024 10:13:16 -0500 Subject: [PATCH 05/30] Reverted changes to PowerLimiter, adapted DNS and mDNS handling in HttpPowerMeter For non IP address URLs, HttpPowerMeter now first tries DNS for resolution as done in WifiClient::connect, and only if that fails tries mDNS. For the latter to work mDNS needs to be enabled in settings. Log in console if mDNS is disabled. String building for Digest authorization still tries to avoid "+" for reasons outlined in https://cpp4arduino.com/2020/02/07/how-to-format-strings-without-the-string-class.html This should also be saver than just concatenating user input strings in preventing format string attacks. https://owasp.org/www-community/attacks/Format_string_attack --- src/HttpPowerMeter.cpp | 136 ++++++++++++++++++++------------------ src/PowerLimiter.cpp | 31 +++------ src/WebApi_powermeter.cpp | 10 +-- 3 files changed, 88 insertions(+), 89 deletions(-) diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index 8bd1fa14b..f58ba0cfb 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -3,7 +3,7 @@ #include "HttpPowerMeter.h" #include "MessageOutput.h" #include -#include //saves 20kB to not use FirebaseJson as ArduinoJson is used already elsewhere (e.g. in WebApi_powermeter) +#include #include #include #include @@ -59,18 +59,23 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const //first check if the urlHostname is already an IP adress if (!ipaddr.fromString(urlHostname)) { - //no it is not, so try to resolve the IP adress - const bool mdnsEnabled = Configuration.get().Mdns.Enabled; - if (!mdnsEnabled) { - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Enable mDNS in Network Settings")); - return false; - } + //urlHostname is not an IP address so try to resolve the IP adress + //first, try DNS + if(!WiFiGenericClass::hostByName(urlHostname.c_str(), ipaddr)) + { + //DNS failed, so now try mDNS + const bool mdnsEnabled = Configuration.get().Mdns.Enabled; + if (!mdnsEnabled) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS, try to enable mDNS in Network Settings"), urlHostname); + return false; + } - ipaddr = MDNS.queryHost(urlHostname); - if (ipaddr == INADDR_NONE){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s"), urlHostname.c_str()); - return false; - } + ipaddr = MDNS.queryHost(urlHostname); + if (ipaddr == INADDR_NONE){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS and mDNS"), urlHostname.c_str()); + return false; + } + } } // secureWifiClient MUST be created before HTTPClient @@ -84,8 +89,10 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const } else { wifiClient = std::make_unique(); } + return httpRequest(phase, *wifiClient, urlProtocol, ipaddr.toString(), uri, authType, username, password, httpHeader, httpValue, timeout, jsonPath); } + bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath) { @@ -134,51 +141,53 @@ bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const S } String HttpPowerMeterClass::extractParam(String& authReq, const String& param, const char delimit) { - int _begin = authReq.indexOf(param); - if (_begin == -1) { return ""; } - return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length())); + int _begin = authReq.indexOf(param); + if (_begin == -1) { return ""; } + return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length())); } + void HttpPowerMeterClass::getcNonce(char* cNounce) { - static const char alphanum[] = "0123456789" + static const char alphanum[] = "0123456789" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; - auto len=sizeof(cNounce); + auto len=sizeof(cNounce); - for (int i = 0; i < len; ++i) { cNounce[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; } + for (int i = 0; i < len; ++i) { cNounce[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; } } + String HttpPowerMeterClass::getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter) { - // extracting required parameters for RFC 2069 simpler Digest - String realm = extractParam(authReq, "realm=\"", '"'); - String nonce = extractParam(authReq, "nonce=\"", '"'); - char cNonce[8]; - getcNonce(cNonce); - - char nc[9]; - snprintf(nc, sizeof(nc), "%08x", counter); - - // parameters for the Digest - // sha256 of the user:realm:user - char h1Prep[sizeof(username)+sizeof(realm)+sizeof(password)+2]; - snprintf(h1Prep, sizeof(h1Prep), "%s:%s:%s", username.c_str(),realm.c_str(), password.c_str()); - String ha1 = sha256(h1Prep); - - //sha256 of method:uri - char h2Prep[sizeof(method) + sizeof(uri) + 1]; - snprintf(h2Prep, sizeof(h2Prep), "%s:%s", method.c_str(),uri.c_str()); - String ha2 = sha256(h2Prep); - - //md5 of h1:nonce:nc:cNonce:auth:h2 - char responsePrep[sizeof(ha1)+sizeof(nc)+sizeof(cNonce)+4+sizeof(ha2) + 5]; - snprintf(responsePrep, sizeof(responsePrep), "%s:%s:%s:%s:auth:%s", ha1.c_str(),nonce.c_str(), nc, cNonce,ha2.c_str()); - String response = sha256(responsePrep); - - //Final authorization String; - char authorization[17 + sizeof(username) + 10 + sizeof(realm) + 10 + sizeof(nonce) + 8 + sizeof(uri) + 34 + sizeof(nc) + 10 + sizeof(cNonce) + 13 + sizeof(response)]; - snprintf(authorization, sizeof(authorization), "Digest username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", algorithm=SHA-256, qop=auth, nc=%s, cnonce=\"%s\", response=\"%s\"", username.c_str(), realm.c_str(), nonce.c_str(), uri.c_str(), nc, cNonce, response.c_str()); - - return authorization; + // extracting required parameters for RFC 2617 Digest + String realm = extractParam(authReq, "realm=\"", '"'); + String nonce = extractParam(authReq, "nonce=\"", '"'); + char cNonce[8]; + getcNonce(cNonce); + + char nc[9]; + snprintf(nc, sizeof(nc), "%08x", counter); + + // sha256 of the user:realm:user + char h1Prep[1024];//can username+password be longer than 255 chars each? + snprintf(h1Prep, sizeof(h1Prep), "%s:%s:%s", username.c_str(),realm.c_str(), password.c_str()); + String ha1 = sha256(h1Prep); + + //sha256 of method:uri + char h2Prep[1024];//can uri be longer? + snprintf(h2Prep, sizeof(h2Prep), "%s:%s", method.c_str(),uri.c_str()); + String ha2 = sha256(h2Prep); + + //sha256 of h1:nonce:nc:cNonce:auth:h2 + char responsePrep[2048];//can nounce and cNounce be longer? + snprintf(responsePrep, sizeof(responsePrep), "%s:%s:%s:%s:auth:%s", ha1.c_str(),nonce.c_str(), nc, cNonce,ha2.c_str()); + String response = sha256(responsePrep); + + //Final authorization String; + char authorization[2048];//can username+password be longer than 255 chars each? can uri be longer? + snprintf(authorization, sizeof(authorization), "Digest username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", algorithm=SHA-256, qop=auth, nc=%s, cnonce=\"%s\", response=\"%s\"", username.c_str(), realm.c_str(), nonce.c_str(), uri.c_str(), nc, cNonce, response.c_str()); + + return authorization; } + bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, int httpCode, const char* jsonPath) { bool success = false; @@ -201,6 +210,7 @@ bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, int httpCode, cons } return success; } + void HttpPowerMeterClass::extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri) { // Find protocol delimiter int protocolEndIndex = url.indexOf(":"); @@ -234,23 +244,23 @@ void HttpPowerMeterClass::extractUrlComponents(const String& url, String& protoc #define HASH_SIZE 32 String HttpPowerMeterClass::sha256(const String& data) { - SHA256 sha256; - uint8_t hash[HASH_SIZE]; - - sha256.reset(); - sha256.update(data.c_str(), data.length()); - sha256.finalize(hash, HASH_SIZE); - - String hashStr = ""; - for (int i = 0; i < HASH_SIZE; i++) { - String hex = String(hash[i], HEX); - if (hex.length() == 1) { - hashStr += "0"; + SHA256 sha256; + uint8_t hash[HASH_SIZE]; + + sha256.reset(); + sha256.update(data.c_str(), data.length()); + sha256.finalize(hash, HASH_SIZE); + + String hashStr = ""; + for (int i = 0; i < HASH_SIZE; i++) { + String hex = String(hash[i], HEX); + if (hex.length() == 1) { + hashStr += "0"; + } + hashStr += hex; } - hashStr += hex; - } - return hashStr; + return hashStr; } void HttpPowerMeterClass::prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue) { httpClient.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); diff --git a/src/PowerLimiter.cpp b/src/PowerLimiter.cpp index 56d732570..86cb2751b 100644 --- a/src/PowerLimiter.cpp +++ b/src/PowerLimiter.cpp @@ -187,8 +187,7 @@ void PowerLimiterClass::loop() // a calculated power limit will always be limited to the reported // device's max power. that upper limit is only known after the first // DevInfoSimpleCommand succeeded. - auto maxPower = _inverter->DevInfo()->getMaxPower(); - if (maxPower <= 0) { + if (_inverter->DevInfo()->getMaxPower() <= 0) { return announceStatus(Status::InverterDevInfoPending); } @@ -199,13 +198,11 @@ void PowerLimiterClass::loop() // the normal mode of operation requires a valid // power meter reading to calculate a power limit - if (!config.PowerMeter.Enabled) { - shutdown(Status::PowerMeterDisabled); + if (!config.PowerMeter.Enabled) { + shutdown(Status::PowerMeterDisabled); return; } - //instead of shutting down on PowerMeterTimeout, how about setting alternativly to a safe "low production" mode? - //Could be usefull when PowerMeter fails but we know for sure house consumption will never fall below a certain limit (say 200W) if (millis() - PowerMeter.getLastPowerMeterUpdate() > (30 * 1000)) { shutdown(Status::PowerMeterTimeout); return; @@ -225,7 +222,6 @@ void PowerLimiterClass::loop() if (_inverter->Statistics()->getLastUpdate() <= settlingEnd) { return announceStatus(Status::InverterStatsPending); } - if (PowerMeter.getLastPowerMeterUpdate() <= settlingEnd) { return announceStatus(Status::PowerMeterPending); @@ -549,8 +545,8 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr inver dcTotalChnls, dcProdChnls); effPowerLimit = round(effPowerLimit * static_cast(dcTotalChnls) / dcProdChnls); } - auto maxPower = inverter->DevInfo()->getMaxPower(); - effPowerLimit = std::min(effPowerLimit, maxPower); + + effPowerLimit = std::min(effPowerLimit, inverter->DevInfo()->getMaxPower()); // Check if the new value is within the limits of the hysteresis auto diff = std::abs(effPowerLimit - _lastRequestedPowerLimit); @@ -560,23 +556,16 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr inver // staleness in case a power limit update was not received by the inverter. auto ageMillis = millis() - _lastPowerLimitMillis; - //instead pushing limit to inverter every 60 seconds no matter what, - //why not query instead the currenty configured limit...and do nothing if not needed - int currentLimit = round(inverter->SystemConfigPara()->getLimitPercent() * maxPower / 100); - auto currentDiff = std::abs(effPowerLimit - currentLimit ); - - if (diff < hysteresis && currentDiff < hysteresis ){ - //if (diff < hysteresis && ageMillis < 60 * 1000) { - //MessageOutput.printf("Keep limit: %d W, current limit %d W\r\n", effPowerLimit, currentLimit); + if (diff < hysteresis && ageMillis < 60 * 1000) { if (_verboseLogging) { - MessageOutput.printf("[DPL::setNewPowerLimit] Keep current limit. (new calculated: %d W, last limit: %d W, diff: %d W, hysteresis: %d W, age: %ld ms)\r\n", - effPowerLimit, _lastRequestedPowerLimit, diff, hysteresis, ageMillis); + MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, last limit: %d W, diff: %d W, hysteresis: %d W, age: %ld ms\r\n", + newPowerLimit, _lastRequestedPowerLimit, diff, hysteresis, ageMillis); } return false; } - //if we end up here, it we will set new limit + if (_verboseLogging) { - MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, sending limit: %d W\r\n", + MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, (re-)sending limit: %d W\r\n", newPowerLimit, effPowerLimit); } diff --git a/src/WebApi_powermeter.cpp b/src/WebApi_powermeter.cpp index 83b2eda5c..d5958668f 100644 --- a/src/WebApi_powermeter.cpp +++ b/src/WebApi_powermeter.cpp @@ -203,11 +203,11 @@ void WebApiPowerMeterClass::onAdminPost(AsyncWebServerRequest* request) response->setLength(); request->send(response); - // why reboot..WebApi_powerlimiter is also not rebooting - // yield(); - // delay(1000); - // yield(); - // ESP.restart(); + // reboot requiered as per https://github.com/helgeerbe/OpenDTU-OnBattery/issues/565#issuecomment-1872552559 + yield(); + delay(1000); + yield(); + ESP.restart(); } void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request) From 7b5d31efcadfc8f4bb0a3b3cad9315ae6f3e9bd2 Mon Sep 17 00:00:00 2001 From: Thomas Basler Date: Fri, 5 Jan 2024 17:26:02 +0100 Subject: [PATCH 06/30] Added .editorconfig --- .editorconfig | 9 +++++++++ .vscode/extensions.json | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..559f9bfd2 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +# http://editorconfig.org +root = true + +[*] +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true diff --git a/.vscode/extensions.json b/.vscode/extensions.json index d14c6bfee..0d84eb653 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -5,7 +5,8 @@ "DavidAnson.vscode-markdownlint", "Vue.volar", "Vue.vscode-typescript-vue-plugin", - "platformio.platformio-ide" + "platformio.platformio-ide", + "EditorConfig.EditorConfig" ], "unwantedRecommendations": [ "ms-vscode.cpptools-extension-pack" From 85d0f2a8fbb41d9625a3502b1c91f38dbbde9559 Mon Sep 17 00:00:00 2001 From: Fribur Date: Fri, 5 Jan 2024 14:36:19 -0500 Subject: [PATCH 07/30] HttpPowerMeterClass: change order of resolving hostname OpenDTU console gets spammed with "WifiGeneric::hostByName() error when first trying to resolve the hostname via DNS. So reverse order: first try mDNS, if that fails try DNS. Also ensure that https bool is passed correctly to HTTPClient::begin(). Lastly, concatenate strings for building Digest authorization using "+" and not via snprintf. --- include/HttpPowerMeter.h | 9 ++-- src/HttpPowerMeter.cpp | 103 ++++++++++++++++++++------------------ src/WebApi_powermeter.cpp | 8 +-- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/include/HttpPowerMeter.h b/include/HttpPowerMeter.h index 16ee85851..8530f4cda 100644 --- a/include/HttpPowerMeter.h +++ b/include/HttpPowerMeter.h @@ -11,18 +11,19 @@ class HttpPowerMeterClass { bool updateValues(); float getPower(int8_t phase); char httpPowerMeterError[256]; - bool queryPhase(int phase, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password, + bool queryPhase(int phase, const String& url, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath); - void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri); + private: float power[POWERMETER_MAX_PHASES]; HTTPClient httpClient; String httpResponse; - bool httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& urlUri, Auth authType, const char* username, + bool httpRequest(int phase, WiFiClient &wifiClient, const String& urlHostname, const String& uri, bool https, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath); + void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri); String extractParam(String& authReq, const String& param, const char delimit); - void getcNonce(char* cNounce); + String getcNonce(const int len); String getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter); bool tryGetFloatValueForPhase(int phase, int httpCode, const char* jsonPath); void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue); diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index f58ba0cfb..c232ce8a1 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -25,10 +25,6 @@ bool HttpPowerMeterClass::updateValues() for (uint8_t i = 0; i < POWERMETER_MAX_PHASES; i++) { POWERMETER_HTTP_PHASE_CONFIG_T phaseConfig = config.PowerMeter.Http_Phase[i]; - String urlProtocol; - String urlHostname; - String urlUri; - extractUrlComponents(phaseConfig.Url, urlProtocol, urlHostname, urlUri); if (!phaseConfig.Enabled) { power[i] = 0.0; @@ -36,7 +32,7 @@ bool HttpPowerMeterClass::updateValues() } if (i == 0 || config.PowerMeter.HttpIndividualRequests) { - if (!queryPhase(i, urlProtocol, urlHostname, urlUri, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout, + if (!queryPhase(i, phaseConfig.Url, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout, phaseConfig.JsonPath)) { MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d failed.\r\n", i + 1); MessageOutput.printf("%s\r\n", httpPowerMeterError); @@ -47,7 +43,7 @@ bool HttpPowerMeterClass::updateValues() return true; } -bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password, +bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath) { //hostByName in WiFiGeneric fails to resolve local names. issue described in @@ -55,34 +51,39 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const //and in depth analyzed in https://github.com/espressif/esp-idf/issues/2507#issuecomment-761836300 //in conclusion: we cannot rely on httpClient.begin(*wifiClient, url) to resolve IP adresses. //have to do it manually here. Feels Hacky... + String protocol; + String host; + String uri; + extractUrlComponents(url, protocol, host, uri); + IPAddress ipaddr((uint32_t)0); //first check if the urlHostname is already an IP adress - if (!ipaddr.fromString(urlHostname)) + if (!ipaddr.fromString(host)) { //urlHostname is not an IP address so try to resolve the IP adress - //first, try DNS - if(!WiFiGenericClass::hostByName(urlHostname.c_str(), ipaddr)) + //first try locally via mDNS, then via DNS (WiFiGeneric::hostByName() will spam the console if done the otherway around) + const bool mdnsEnabled = Configuration.get().Mdns.Enabled; + if (!mdnsEnabled) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS, try to enable mDNS in Network Settings"), url.c_str()); + } + else { - //DNS failed, so now try mDNS - const bool mdnsEnabled = Configuration.get().Mdns.Enabled; - if (!mdnsEnabled) { - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS, try to enable mDNS in Network Settings"), urlHostname); - return false; - } - - ipaddr = MDNS.queryHost(urlHostname); + ipaddr = MDNS.queryHost(host); if (ipaddr == INADDR_NONE){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS and mDNS"), urlHostname.c_str()); - return false; + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via mDNS"), url.c_str()); + if(!WiFiGenericClass::hostByName(host.c_str(), ipaddr)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS"), url.c_str()); + } } - } + } } // secureWifiClient MUST be created before HTTPClient // see discussion: https://github.com/helgeerbe/OpenDTU-OnBattery/issues/381 std::unique_ptr wifiClient; - if (urlProtocol == "https") { + bool https = protocol == "https"; + if (https) { auto secureWifiClient = std::make_unique(); secureWifiClient->setInsecure(); wifiClient = std::move(secureWifiClient); @@ -90,18 +91,15 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const wifiClient = std::make_unique(); } - return httpRequest(phase, *wifiClient, urlProtocol, ipaddr.toString(), uri, authType, username, password, httpHeader, httpValue, timeout, jsonPath); + return httpRequest(phase, *wifiClient, ipaddr.toString(), uri, https, authType, username, password, httpHeader, httpValue, timeout, jsonPath); } -bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, +bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& urlHostname, const String& uri, bool https, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath) { - int port = 80; - if (urlProtocol == "https") { - port = 443; - } - if(!httpClient.begin(wifiClient, urlHostname, port, uri)){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), urlProtocol.c_str(), urlHostname.c_str()); + int port = (https ? 443 : 80); + if(!httpClient.begin(wifiClient, urlHostname, port, uri, https)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), (https ? "https" : "http"), urlHostname.c_str()); return false; } @@ -125,8 +123,8 @@ bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const S String authReq = httpClient.header("WWW-Authenticate"); String authorization = getDigestAuth(authReq, String(username), String(password), "GET", String(uri), 1); httpClient.end(); - if(!httpClient.begin(wifiClient, urlHostname, port, uri)){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s using digest auth"), urlProtocol.c_str(), urlHostname.c_str()); + if(!httpClient.begin(wifiClient, urlHostname, port, uri, https)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s using digest auth"), (https ? "https" : "http"), urlHostname.c_str()); return false; } @@ -146,44 +144,51 @@ String HttpPowerMeterClass::extractParam(String& authReq, const String& param, c return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length())); } -void HttpPowerMeterClass::getcNonce(char* cNounce) { +String HttpPowerMeterClass::getcNonce(const int len) { static const char alphanum[] = "0123456789" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; - auto len=sizeof(cNounce); + String s = ""; - for (int i = 0; i < len; ++i) { cNounce[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; } + for (int i = 0; i < len; ++i) { s += alphanum[rand() % (sizeof(alphanum) - 1)]; } + return s; } String HttpPowerMeterClass::getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter) { // extracting required parameters for RFC 2617 Digest String realm = extractParam(authReq, "realm=\"", '"'); String nonce = extractParam(authReq, "nonce=\"", '"'); - char cNonce[8]; - getcNonce(cNonce); + String cNonce = getcNonce(8); char nc[9]; snprintf(nc, sizeof(nc), "%08x", counter); - // sha256 of the user:realm:user - char h1Prep[1024];//can username+password be longer than 255 chars each? - snprintf(h1Prep, sizeof(h1Prep), "%s:%s:%s", username.c_str(),realm.c_str(), password.c_str()); - String ha1 = sha256(h1Prep); + //sha256 of the user:realm:password + String ha1 = sha256(username + ":" + realm + ":" + password); //sha256 of method:uri - char h2Prep[1024];//can uri be longer? - snprintf(h2Prep, sizeof(h2Prep), "%s:%s", method.c_str(),uri.c_str()); - String ha2 = sha256(h2Prep); + String ha2 = sha256(method + ":" + uri); - //sha256 of h1:nonce:nc:cNonce:auth:h2 - char responsePrep[2048];//can nounce and cNounce be longer? - snprintf(responsePrep, sizeof(responsePrep), "%s:%s:%s:%s:auth:%s", ha1.c_str(),nonce.c_str(), nc, cNonce,ha2.c_str()); - String response = sha256(responsePrep); + //sha256 of h1:nonce:nc:cNonce:auth:h2 + String response = sha256(ha1 + ":" + nonce + ":" + String(nc) + ":" + cNonce + ":" + "auth" + ":" + ha2); //Final authorization String; - char authorization[2048];//can username+password be longer than 255 chars each? can uri be longer? - snprintf(authorization, sizeof(authorization), "Digest username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", algorithm=SHA-256, qop=auth, nc=%s, cnonce=\"%s\", response=\"%s\"", username.c_str(), realm.c_str(), nonce.c_str(), uri.c_str(), nc, cNonce, response.c_str()); + String authorization = "Digest username=\""; + authorization += username; + authorization += "\", realm=\""; + authorization += realm; + authorization += "\", nonce=\""; + authorization += nonce; + authorization += "\", uri=\""; + authorization += uri; + authorization += "\", cnonce=\""; + authorization += cNonce; + authorization += "\", nc="; + authorization += String(nc); + authorization += ", qop=auth, response=\""; + authorization += response; + authorization += "\", algorithm=SHA-256"; return authorization; } diff --git a/src/WebApi_powermeter.cpp b/src/WebApi_powermeter.cpp index d5958668f..8e8a4e652 100644 --- a/src/WebApi_powermeter.cpp +++ b/src/WebApi_powermeter.cpp @@ -258,14 +258,8 @@ void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request) char response[256]; - String urlProtocol; - String urlHostname; - String urlUri; - - HttpPowerMeter.extractUrlComponents(root[F("url")].as().c_str(), urlProtocol, urlHostname, urlUri); - int phase = 0;//"absuing" index 0 of the float power[3] in HttpPowerMeter to store the result - if (HttpPowerMeter.queryPhase(phase, urlProtocol, urlHostname, urlUri, + if (HttpPowerMeter.queryPhase(phase, root[F("url")].as().c_str(), root[F("auth_type")].as(), root[F("username")].as().c_str(), root[F("password")].as().c_str(), root[F("header_key")].as().c_str(), root[F("header_value")].as().c_str(), root[F("timeout")].as(), root[F("json_path")].as().c_str())) { From e09ffcbb53ede72616a9752346aa5495e58922dd Mon Sep 17 00:00:00 2001 From: Fribur Date: Fri, 5 Jan 2024 14:39:32 -0500 Subject: [PATCH 08/30] shorter parameter names --- include/HttpPowerMeter.h | 4 ++-- src/HttpPowerMeter.cpp | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/HttpPowerMeter.h b/include/HttpPowerMeter.h index 8530f4cda..95cc1781d 100644 --- a/include/HttpPowerMeter.h +++ b/include/HttpPowerMeter.h @@ -19,9 +19,9 @@ class HttpPowerMeterClass { float power[POWERMETER_MAX_PHASES]; HTTPClient httpClient; String httpResponse; - bool httpRequest(int phase, WiFiClient &wifiClient, const String& urlHostname, const String& uri, bool https, Auth authType, const char* username, + bool httpRequest(int phase, WiFiClient &wifiClient, const String& host, const String& uri, bool https, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath); - void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri); + void extractUrlComponents(const String& url, String& protocol, String& host, String& uri); String extractParam(String& authReq, const String& param, const char delimit); String getcNonce(const int len); String getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter); diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index c232ce8a1..4775c6f8f 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -64,15 +64,15 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType //first try locally via mDNS, then via DNS (WiFiGeneric::hostByName() will spam the console if done the otherway around) const bool mdnsEnabled = Configuration.get().Mdns.Enabled; if (!mdnsEnabled) { - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS, try to enable mDNS in Network Settings"), url.c_str()); + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via DNS, try to enable mDNS in Network Settings"), host.c_str()); } else { ipaddr = MDNS.queryHost(host); if (ipaddr == INADDR_NONE){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via mDNS"), url.c_str()); + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via mDNS"), host.c_str()); if(!WiFiGenericClass::hostByName(host.c_str(), ipaddr)){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s via DNS"), url.c_str()); + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via DNS"), host.c_str()); } } } @@ -94,12 +94,12 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType return httpRequest(phase, *wifiClient, ipaddr.toString(), uri, https, authType, username, password, httpHeader, httpValue, timeout, jsonPath); } -bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& urlHostname, const String& uri, bool https, Auth authType, const char* username, +bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& host, const String& uri, bool https, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath) { int port = (https ? 443 : 80); - if(!httpClient.begin(wifiClient, urlHostname, port, uri, https)){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), (https ? "https" : "http"), urlHostname.c_str()); + if(!httpClient.begin(wifiClient, host, port, uri, https)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), (https ? "https" : "http"), host.c_str()); return false; } @@ -123,8 +123,8 @@ bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const S String authReq = httpClient.header("WWW-Authenticate"); String authorization = getDigestAuth(authReq, String(username), String(password), "GET", String(uri), 1); httpClient.end(); - if(!httpClient.begin(wifiClient, urlHostname, port, uri, https)){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s using digest auth"), (https ? "https" : "http"), urlHostname.c_str()); + if(!httpClient.begin(wifiClient, host, port, uri, https)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s using digest auth"), (https ? "https" : "http"), host.c_str()); return false; } From 92de3e9f8700f66e11cb6a4e892c02aa9922f148 Mon Sep 17 00:00:00 2001 From: Fribur Date: Fri, 5 Jan 2024 20:54:53 -0500 Subject: [PATCH 09/30] fixed a bug where under one condition DNS was not tried for resolving host IP --- src/HttpPowerMeter.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index 4775c6f8f..ade8d3802 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -57,20 +57,25 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType extractUrlComponents(url, protocol, host, uri); IPAddress ipaddr((uint32_t)0); - //first check if the urlHostname is already an IP adress + //first check if "host" is already an IP adress if (!ipaddr.fromString(host)) { - //urlHostname is not an IP address so try to resolve the IP adress - //first try locally via mDNS, then via DNS (WiFiGeneric::hostByName() will spam the console if done the otherway around) + //"host"" is not an IP address so try to resolve the IP adress + //first try locally via mDNS, then via DNS. WiFiGeneric::hostByName() will spam the console if done the otherway around. const bool mdnsEnabled = Configuration.get().Mdns.Enabled; if (!mdnsEnabled) { - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via DNS, try to enable mDNS in Network Settings"), host.c_str()); + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via DNS, try to enable mDNS in Network Settings"), host.c_str()); + //ensure we try resolving via DNS even if mDNS is disabled + if(!WiFiGenericClass::hostByName(host.c_str(), ipaddr)){ + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via DNS"), host.c_str()); + } } else { ipaddr = MDNS.queryHost(host); if (ipaddr == INADDR_NONE){ - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via mDNS"), host.c_str()); + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via mDNS"), host.c_str()); + //when we cannot find local server via mDNS, try resolving via DNS if(!WiFiGenericClass::hostByName(host.c_str(), ipaddr)){ snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving host %s via DNS"), host.c_str()); } From 024ee26705badbfca9f8d1f9b6a4f7eb5b9da17b Mon Sep 17 00:00:00 2001 From: Thomas Basler Date: Sat, 6 Jan 2024 20:03:52 +0100 Subject: [PATCH 10/30] Feature: Added pull to refresh and websocket indicator --- webapp/package.json | 2 + webapp/src/components/BasePage.vue | 84 +++++++++++++++++++++++++++++- webapp/src/locales/de.json | 5 +- webapp/src/locales/en.json | 5 +- webapp/src/locales/fr.json | 5 +- webapp/src/views/HomeView.vue | 56 +++++++++++++++----- webapp/yarn.lock | 10 ++++ 7 files changed, 148 insertions(+), 19 deletions(-) diff --git a/webapp/package.json b/webapp/package.json index 8b36555a9..a785674ed 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -28,6 +28,7 @@ "@tsconfig/node18": "^18.2.2", "@types/bootstrap": "^5.2.10", "@types/node": "^20.10.6", + "@types/pulltorefreshjs": "^0.1.7", "@types/sortablejs": "^1.15.7", "@types/spark-md5": "^3.0.4", "@vitejs/plugin-vue": "^5.0.2", @@ -36,6 +37,7 @@ "eslint": "^8.56.0", "eslint-plugin-vue": "^9.19.2", "npm-run-all": "^4.1.5", + "pulltorefreshjs": "^0.1.22", "sass": "^1.69.7", "terser": "^5.26.0", "typescript": "^5.3.3", diff --git a/webapp/src/components/BasePage.vue b/webapp/src/components/BasePage.vue index 8bb463b53..10b924a89 100644 --- a/webapp/src/components/BasePage.vue +++ b/webapp/src/components/BasePage.vue @@ -4,7 +4,12 @@
-

{{ title }}

+

{{ title }} + +