From 46c55602229f7550f9995c35690ab1a643565610 Mon Sep 17 00:00:00 2001 From: proddy Date: Mon, 25 May 2026 12:32:53 +0200 Subject: [PATCH] add TLS support for all boards --- Makefile | 1 - boards/c3_mini_4M.json | 1 - boards/s2_4M_P.json | 1 - boards/s_16M.json | 1 - boards/s_4M.json | 1 - boards/seeed_xiao_esp32c6.json | 1 - .../src/Transport/ClientSecureSync.cpp | 4 - .../src/Transport/ClientSecureSync.h | 3 - lib/espMqttClient/src/espMqttClient.cpp | 10 - lib/espMqttClient/src/espMqttClient.h | 4 - lib_standalone/ESP_SSLClient.h | 72 +++++ lib_standalone/WiFiClient.h | 49 ++++ lib_standalone/lwip/sockets.h | 10 + platformio.ini | 2 - src/ESP32React/MqttSettingsService.cpp | 20 +- src/ESP32React/NetworkStatus.cpp | 4 - src/core/shuntingYard.cpp | 218 +++++++------- src/core/shuntingYard.h | 2 + src/core/system.cpp | 266 +++++++++++++++--- src/web/WebSettingsService.cpp | 22 +- 20 files changed, 485 insertions(+), 207 deletions(-) create mode 100644 lib_standalone/ESP_SSLClient.h create mode 100644 lib_standalone/WiFiClient.h create mode 100644 lib_standalone/lwip/sockets.h diff --git a/Makefile b/Makefile index 40ce7f991..01ce71667 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,6 @@ CXX_STANDARD := -std=gnu++20 #---------------------------------------------------------------------- DEFINES += -DARDUINOJSON_ENABLE -DARDUINOJSON_ENABLE_ARDUINO_STRING -DARDUINOJSON_USE_DOUBLE=0 DEFINES += -DEMSESP_STANDALONE -DEMSESP_TEST -DEMSESP_DEBUG -DEMC_RX_BUFFER_SIZE=1500 -DEFINES += -DNO_TLS_SUPPORT DEFINES += $(ARGS) DEFAULTS = -DEMSESP_DEFAULT_LOCALE=\"en\" -DEMSESP_DEFAULT_BOARD_PROFILE=\"S32S3\" diff --git a/boards/c3_mini_4M.json b/boards/c3_mini_4M.json index 96b9906df..7f19d1662 100644 --- a/boards/c3_mini_4M.json +++ b/boards/c3_mini_4M.json @@ -5,7 +5,6 @@ }, "core": "esp32", "extra_flags": [ - "-DNO_TLS_SUPPORT", "-DARDUINO_LOLIN_C3_MINI", "-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1" diff --git a/boards/s2_4M_P.json b/boards/s2_4M_P.json index 0df1c4b64..8baa60e01 100644 --- a/boards/s2_4M_P.json +++ b/boards/s2_4M_P.json @@ -6,7 +6,6 @@ "core": "esp32", "extra_flags": [ "-DBOARD_HAS_PSRAM", - "-DNO_TLS_SUPPORT", "-DARDUINO_USB_CDC_ON_BOOT=1", "-DARDUINO_USB_MODE=0" ], diff --git a/boards/s_16M.json b/boards/s_16M.json index b346a31cc..664d2bd61 100644 --- a/boards/s_16M.json +++ b/boards/s_16M.json @@ -1,7 +1,6 @@ { "build": { "core": "esp32", - "extra_flags": "-DNO_TLS_SUPPORT", "f_cpu": "240000000L", "f_flash": "40000000L", "flash_mode": "dio", diff --git a/boards/s_4M.json b/boards/s_4M.json index 8658bd3c9..8978e2625 100644 --- a/boards/s_4M.json +++ b/boards/s_4M.json @@ -1,7 +1,6 @@ { "build": { "core": "esp32", - "extra_flags": "-DNO_TLS_SUPPORT", "f_cpu": "240000000L", "f_flash": "40000000L", "flash_mode": "dio", diff --git a/boards/seeed_xiao_esp32c6.json b/boards/seeed_xiao_esp32c6.json index a3de41d41..224f0e215 100644 --- a/boards/seeed_xiao_esp32c6.json +++ b/boards/seeed_xiao_esp32c6.json @@ -2,7 +2,6 @@ "build": { "core": "esp32", "extra_flags": [ - "-DNO_TLS_SUPPORT", "-DARDUINO_XIAO_ESP32C6", "-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1" diff --git a/lib/espMqttClient/src/Transport/ClientSecureSync.cpp b/lib/espMqttClient/src/Transport/ClientSecureSync.cpp index aa9e30f13..a61090f67 100644 --- a/lib/espMqttClient/src/Transport/ClientSecureSync.cpp +++ b/lib/espMqttClient/src/Transport/ClientSecureSync.cpp @@ -6,8 +6,6 @@ For a copy, see or the LICENSE file. */ -#ifndef NO_TLS_SUPPORT - #include "ClientSecureSync.h" #include #include "../Config.h" @@ -66,5 +64,3 @@ bool ClientSecureSync::disconnected() { } } // namespace espMqttClientInternals - -#endif \ No newline at end of file diff --git a/lib/espMqttClient/src/Transport/ClientSecureSync.h b/lib/espMqttClient/src/Transport/ClientSecureSync.h index 84c51f761..c949eec54 100644 --- a/lib/espMqttClient/src/Transport/ClientSecureSync.h +++ b/lib/espMqttClient/src/Transport/ClientSecureSync.h @@ -8,7 +8,6 @@ the LICENSE file. #pragma once -#ifndef NO_TLS_SUPPORT // #include "esp_tls.h" #include @@ -34,5 +33,3 @@ class ClientSecureSync : public Transport { }; } // namespace espMqttClientInternals - -#endif \ No newline at end of file diff --git a/lib/espMqttClient/src/espMqttClient.cpp b/lib/espMqttClient/src/espMqttClient.cpp index 6649487cc..c2c12118f 100644 --- a/lib/espMqttClient/src/espMqttClient.cpp +++ b/lib/espMqttClient/src/espMqttClient.cpp @@ -34,36 +34,26 @@ espMqttClientSecure::espMqttClientSecure(uint8_t priority, uint8_t core) } espMqttClientSecure & espMqttClientSecure::setInsecure() { -#ifndef NO_TLS_SUPPORT _client.client.setInsecure(); -#endif return *this; } espMqttClientSecure & espMqttClientSecure::setCACert(const char * rootCA) { -#ifndef NO_TLS_SUPPORT _client.client.setCACert(rootCA); -#endif return *this; } espMqttClientSecure & espMqttClientSecure::setCertificate(const char * clientCa) { -#ifndef NO_TLS_SUPPORT _client.client.setCertificate(clientCa); -#endif return *this; } espMqttClientSecure & espMqttClientSecure::setPrivateKey(const char * privateKey) { -#ifndef NO_TLS_SUPPORT _client.client.setPrivateKey(privateKey); -#endif return *this; } espMqttClientSecure & espMqttClientSecure::setPreSharedKey(const char * pskIdent, const char * psKey) { -#ifndef NO_TLS_SUPPORT -#endif return *this; } diff --git a/lib/espMqttClient/src/espMqttClient.h b/lib/espMqttClient/src/espMqttClient.h index b31a8ea64..41d2c516b 100644 --- a/lib/espMqttClient/src/espMqttClient.h +++ b/lib/espMqttClient/src/espMqttClient.h @@ -65,11 +65,7 @@ class espMqttClientSecure : public MqttClientSetup { espMqttClientSecure & setPreSharedKey(const char * pskIdent, const char * psKey); protected: -#ifndef NO_TLS_SUPPORT espMqttClientInternals::ClientSecureSync _client; -#else - espMqttClientInternals::ClientSync _client; -#endif }; #endif diff --git a/lib_standalone/ESP_SSLClient.h b/lib_standalone/ESP_SSLClient.h new file mode 100644 index 000000000..131f10230 --- /dev/null +++ b/lib_standalone/ESP_SSLClient.h @@ -0,0 +1,72 @@ +// standalone stub for ESP_SSLClient (BearSSL) - no-op TLS on host build +// Inherits from Stream so it gets print/println via Print, matching the real API. + +#ifndef ESP_SSLClient_h +#define ESP_SSLClient_h + +#include +#include + +#include "Arduino.h" +#include "Stream.h" +#include "WiFiClient.h" +#include + +class ESP_SSLClient : public Stream { + public: + ESP_SSLClient() = default; + + void setInsecure() { + } + void setCACert(const char *) { + } + void setCertificate(const char *) { + } + void setPrivateKey(const char *) { + } + void setBufferSizes(int, int) { + } + void setSessionTimeout(int) { + } + void setClient(WiFiClient *) { + } + void setClient(WiFiClient *, bool) { + } + + int connect(IPAddress, uint16_t) { + return 0; + } + int connect(const char *, uint16_t) { + return 0; + } + bool connected() { + return false; + } + + // Stream / Print overrides + int available() override { + return 0; + } + int read() override { + return -1; + } + int read(uint8_t *, size_t) { + return 0; + } + int peek() override { + return -1; + } + size_t write(uint8_t) override { + return 0; + } + size_t write(const uint8_t *, size_t) override { + return 0; + } + void flush() override { + } + + void stop() { + } +}; + +#endif // ESP_SSLClient_h diff --git a/lib_standalone/WiFiClient.h b/lib_standalone/WiFiClient.h new file mode 100644 index 000000000..ab43e928e --- /dev/null +++ b/lib_standalone/WiFiClient.h @@ -0,0 +1,49 @@ +// standalone stub for WiFiClient (no-op networking on host build) + +#ifndef WiFiClient_h +#define WiFiClient_h + +#include +#include + +#include "Arduino.h" +#include + +class WiFiClient { + public: + WiFiClient() = default; + + int connect(IPAddress, uint16_t) { + return 0; + } + int connect(const char *, uint16_t) { + return 0; + } + bool connected() { + return false; + } + int available() { + return 0; + } + int read() { + return -1; + } + int read(uint8_t *, size_t) { + return 0; + } + size_t write(uint8_t) { + return 0; + } + size_t write(const uint8_t *, size_t) { + return 0; + } + void stop() { + } + + // ESP32 socket option passthrough (e.g. TCP_NODELAY) + int setSocketOption(int, int, const void *, size_t) { + return 0; + } +}; + +#endif // WiFiClient_h diff --git a/lib_standalone/lwip/sockets.h b/lib_standalone/lwip/sockets.h new file mode 100644 index 000000000..6bae137d2 --- /dev/null +++ b/lib_standalone/lwip/sockets.h @@ -0,0 +1,10 @@ +// standalone stub for +// pulls in POSIX equivalents on the host build for things like IPPROTO_TCP / TCP_NODELAY. + +#ifndef LWIP_SOCKETS_STUB_H_ +#define LWIP_SOCKETS_STUB_H_ + +#include +#include + +#endif // LWIP_SOCKETS_STUB_H_ diff --git a/platformio.ini b/platformio.ini index d526fbd8c..eacaa0944 100644 --- a/platformio.ini +++ b/platformio.ini @@ -169,7 +169,6 @@ build_flags = build_src_flags = -DEMSESP_STANDALONE -DEMSESP_TEST -DARDUINOJSON_ENABLE_ARDUINO_STRING=1 - -DNO_TLS_SUPPORT -std=gnu++20 -Og -ggdb -Wall -Wextra -Wno-unused-parameter -Wno-sign-compare -Wno-missing-braces @@ -208,7 +207,6 @@ build_src_flags = -DEMSESP_STANDALONE -DEMSESP_TEST -DEMSESP_UNITY -DARDUINOJSON_ENABLE_ARDUINO_STRING=1 - -DNO_TLS_SUPPORT -DEMSESP_DEFAULT_LOCALE=\"en\" -DEMSESP_DEFAULT_BOARD_PROFILE=\"S32\" -std=gnu++20 -Og -ggdb -Wall -Wextra diff --git a/src/ESP32React/MqttSettingsService.cpp b/src/ESP32React/MqttSettingsService.cpp index c371a5438..550b4343a 100644 --- a/src/ESP32React/MqttSettingsService.cpp +++ b/src/ESP32React/MqttSettingsService.cpp @@ -41,7 +41,6 @@ void MqttSettingsService::startClient() { delete _mqttClient; _mqttClient = nullptr; } -#ifndef NO_TLS_SUPPORT if (_state.enableTLS) { isSecure = true; if (emsesp::EMSESP::system_.PSram() == 0) { @@ -62,7 +61,6 @@ void MqttSettingsService::startClient() { }); return; } -#endif isSecure = false; if (emsesp::EMSESP::system_.PSram() == 0) { _mqttClient = new espMqttClient(espMqttClientTypes::UseInternalTask::NO); @@ -164,12 +162,10 @@ bool MqttSettingsService::configureMqtt() { } _reconfigureMqtt = false; -#ifndef NO_TLS_SUPPORT if (_state.enableTLS) { if (_state.rootCA == "insecure") { #if defined(EMSESP_DEBUG) emsesp::EMSESP::logger().debug("Start insecure MQTT"); -#endif static_cast(_mqttClient)->setInsecure(); } else { #if defined(EMSESP_DEBUG) @@ -205,10 +201,8 @@ bool MqttSettingsService::configureMqtt() { } void MqttSettings::read(MqttSettings & settings, JsonObject root) { -#ifndef NO_TLS_SUPPORT - root["enableTLS"] = settings.enableTLS; - root["rootCA"] = settings.rootCA; -#endif + root["enableTLS"] = settings.enableTLS; + root["rootCA"] = settings.rootCA; root["enabled"] = settings.enabled; root["host"] = settings.host; root["port"] = settings.port; @@ -244,12 +238,8 @@ StateUpdateResult MqttSettings::update(JsonObject root, MqttSettings & settings) MqttSettings newSettings; bool changed = false; -#ifndef NO_TLS_SUPPORT - newSettings.enableTLS = root["enableTLS"]; - newSettings.rootCA = root["rootCA"] | ""; -#else - newSettings.enableTLS = false; -#endif + newSettings.enableTLS = root["enableTLS"]; + newSettings.rootCA = root["rootCA"] | ""; newSettings.enabled = root["enabled"] | FACTORY_MQTT_ENABLED; newSettings.host = root["host"] | FACTORY_MQTT_HOST; newSettings.port = static_cast(root["port"] | FACTORY_MQTT_PORT); @@ -375,7 +365,6 @@ StateUpdateResult MqttSettings::update(JsonObject root, MqttSettings & settings) emsesp::EMSESP::mqtt_.set_publish_time_heartbeat(newSettings.publish_time_heartbeat); } -#ifndef NO_TLS_SUPPORT // strip down to certificate only newSettings.rootCA.replace("\r", ""); newSettings.rootCA.replace("\n", ""); @@ -388,7 +377,6 @@ StateUpdateResult MqttSettings::update(JsonObject root, MqttSettings & settings) if (newSettings.enableTLS != settings.enableTLS || newSettings.rootCA != settings.rootCA) { changed = true; } -#endif // save the new settings settings = newSettings; diff --git a/src/ESP32React/NetworkStatus.cpp b/src/ESP32React/NetworkStatus.cpp index 8f99cfa74..75469892b 100644 --- a/src/ESP32React/NetworkStatus.cpp +++ b/src/ESP32React/NetworkStatus.cpp @@ -2,10 +2,6 @@ #include -#ifdef NO_TLS_SUPPORT -#include "lwip/dns.h" -#endif - NetworkStatus::NetworkStatus(AsyncWebServer * server, SecurityManager * securityManager) { securityManager->addEndpoint(server, NETWORK_STATUS_SERVICE_PATH, AuthenticationPredicates::IS_AUTHENTICATED, [this](AsyncWebServerRequest * request) { networkStatus(request); diff --git a/src/core/shuntingYard.cpp b/src/core/shuntingYard.cpp index da3e6df10..1db178e4f 100644 --- a/src/core/shuntingYard.cpp +++ b/src/core/shuntingYard.cpp @@ -22,6 +22,9 @@ #include "shuntingYard.h" +#include +#include + namespace emsesp { // find tokens - optimized to reduce string allocations @@ -683,6 +686,106 @@ std::string calculate(const std::string & expr) { return result; } +// perform an HTTP/HTTPS request; returns the HTTP status code (0 on failure or unsupported scheme) +// for HTTPS the response headers are stripped, so `result` always contains only the body +int http_request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result) { + int httpResult = 0; + const bool is_post = value.length() || Helpers::toLower(method) == "post"; + const auto lower_url = Helpers::toLower(url.c_str()); + + if (lower_url.starts_with("https://")) { + WiFiClient * basic_client = new WiFiClient; + ESP_SSLClient * ssl_client = new ESP_SSLClient; + ssl_client->setInsecure(); // with root CA we should set here: ssl_client->setCACert(rootCACert); + // NOTE: 1 KB RX buffer is fine for small JSON-style endpoints used by the scheduler/shunting-yard, + // but it is NOT enough for servers that send full-size TLS records (>1 KB), e.g. GitHub release + // assets / large CDN responses. Such servers do not negotiate max_fragment_length, so the body + // can't be decoded and reads return 0. If this path is ever used to fetch large or CDN-hosted + // payloads, bump the RX buffer to 16384 (see uploadFirmwareURL in core/system.cpp for reference). + ssl_client->setBufferSizes(1024, 1024); + ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds) + url.replace(0, 8, ""); + std::string host = url; + auto index = url.find_first_of('/'); + if (index != std::string::npos) { + host = url.substr(0, index); + url.replace(0, index, ""); + } + ssl_client->setClient(basic_client); + if (ssl_client->connect(host.c_str(), 443)) { + bool content_set = false; + ssl_client->print(is_post ? "POST " : "GET "); + ssl_client->print(url.c_str()); + ssl_client->println(" HTTP/1.1"); + ssl_client->print("Host: "); + ssl_client->println(host.c_str()); + for (JsonPairConst p : headers) { + content_set |= (Helpers::toLower(p.key().c_str()) == "content-type"); + ssl_client->print(p.key().c_str()); + ssl_client->print(": "); + ssl_client->println(p.value().as().c_str()); + } + if (is_post) { + if (!content_set) { + ssl_client->print("Content-Type: "); + ssl_client->println(value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain); + } + ssl_client->print("Content-Length: "); + ssl_client->println(value.length()); + ssl_client->println("Connection: close"); + ssl_client->print("\r\n"); + ssl_client->print(value.c_str()); + } else { + ssl_client->println("Connection: close"); + } + auto ms = millis(); + while (ssl_client->connected() && !ssl_client->available() && millis() - ms < 3000) { + delay(0); + } + while (ssl_client->available()) { + result += (char)ssl_client->read(); + } + ssl_client->stop(); + index = result.find_first_of(' '); + if (index != std::string::npos) { + httpResult = stoi(result.substr(index + 1, 3)); + } + index = result.find("\r\n\r\n"); + if (index != std::string::npos) { + result.replace(0, index + 4, ""); + } + } else { + EMSESP::logger().warning("HTTPS connection failed"); + } + delete ssl_client; + delete basic_client; + } else if (lower_url.starts_with("http://")) { + HTTPClient * http = new HTTPClient; + if (http->begin(url.c_str())) { + bool content_set = false; + for (JsonPairConst p : headers) { + http->addHeader(p.key().c_str(), p.value().as().c_str()); + content_set |= (Helpers::toLower(p.key().c_str()) == "content-type"); + } + if (is_post) { + if (!content_set) { + http->addHeader(asyncsrv::T_Content_Type, value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain); + } + httpResult = http->POST(value.c_str()); + } else { + httpResult = http->GET(); + } + if (httpResult > 0) { + result = http->getString().c_str(); + } + } + http->end(); + delete http; + } + + return httpResult; +} + // check for multiple instances of ? : std::string compute(const std::string & expr) { std::string expr_new = expr; @@ -723,119 +826,10 @@ std::string compute(const std::string & expr) { keys_s = p.key().c_str(); } } - bool content_set = false; - std::string value = doc[value_s] | ""; - std::string method = doc[method_s] | "GET"; - if (value.length()) { - method = "POST"; - } + std::string value = doc[value_s] | ""; + std::string method = doc[method_s] | "GET"; std::string result; - int httpResult = 0; -#ifndef NO_TLS_SUPPORT - if (Helpers::toLower(url.c_str()).starts_with("https://")) { - WiFiClient * basic_client = new WiFiClient; - ESP_SSLClient * ssl_client = new ESP_SSLClient; - ssl_client->setInsecure(); // with root CA we should set here: ssl_client->setCACert(rootCACert); - ssl_client->setBufferSizes(1024, 1024); - ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds) - url.replace(0, 8, ""); - std::string host = url; - auto index = url.find_first_of('/'); - if (index != std::string::npos) { - host = url.substr(0, index); - url.replace(0, index, ""); - } - /* - index = host.find_first_of('@'); - std::string auth; - if (index != std::string::npos) { - auth = base64::encode(host.substr(0, index)); - host.replace(0, index, ""); - } - */ - ssl_client->setClient(basic_client); - if (ssl_client->connect(host.c_str(), 443)) { - if (value.length() || Helpers::toLower(method) == "post") { - ssl_client->print("POST "); - ssl_client->print(url.c_str()); - ssl_client->println(" HTTP/1.1"); - ssl_client->print("Host: "); - ssl_client->println(host.c_str()); - for (JsonPair p : doc[header_s].as()) { - content_set |= (emsesp::Helpers::toLower(p.key().c_str()) == "content-type"); - ssl_client->print(p.key().c_str()); - ssl_client->print(": "); - ssl_client->println(p.value().as().c_str()); - } - if (!content_set) { - ssl_client->print("Content-Type: "); - if (value.starts_with('{')) { - ssl_client->println(asyncsrv::T_application_json); - } else { - ssl_client->println(asyncsrv::T_text_plain); - } - } - ssl_client->print("Content-Length: "); - ssl_client->println(value.length()); - ssl_client->println("Connection: close"); - ssl_client->print("\r\n"); - ssl_client->print(value.c_str()); - } else { - ssl_client->print("GET "); - ssl_client->print(url.c_str()); - ssl_client->println(" HTTP/1.1"); - ssl_client->print("Host: "); - ssl_client->println(host.c_str()); - for (JsonPair p : doc[header_s].as()) { - ssl_client->print(p.key().c_str()); - ssl_client->print(": "); - ssl_client->println(p.value().as().c_str()); - } - ssl_client->println("Connection: close"); - } - auto ms = millis(); - while (!ssl_client->available() && millis() - ms < 3000) { - delay(0); - } - while (ssl_client->available()) { - result += (char)ssl_client->read(); - } - ssl_client->stop(); - index = result.find_first_of(' '); - if (index != std::string::npos) { - httpResult = stoi(result.substr(index + 1, 3)); - } - index = result.find("\r\n\r\n"); - if (index != std::string::npos) { - result.replace(0, index + 4, ""); - } - } - delete ssl_client; - delete basic_client; - } else -#endif - if (Helpers::toLower(url.c_str()).starts_with("http://")) { - HTTPClient * http = new HTTPClient; - if (http->begin(url.c_str())) { - for (JsonPair p : doc[header_s].as()) { - http->addHeader(p.key().c_str(), p.value().as().c_str()); - content_set |= (emsesp::Helpers::toLower(p.key().c_str()) == "content-type"); - } - if (value.length() || Helpers::toLower(method) == "post") { - if (!content_set) { - http->addHeader("Content-Type", value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain); - } - httpResult = http->POST(value.c_str()); - } else { - httpResult = http->GET(); // normal GET - } - if (httpResult > 0) { - result = http->getString().c_str(); - } - } - http->end(); - delete http; - } + int httpResult = http_request(url, method, value, doc[header_s].as(), result); if (httpResult == 200) { std::string key = doc[key_s] | ""; JsonDocument keys_doc; // JsonDocument to hold "keys" after doc is parsed with HTTP body diff --git a/src/core/shuntingYard.h b/src/core/shuntingYard.h index ce9f4af74..712ab2295 100644 --- a/src/core/shuntingYard.h +++ b/src/core/shuntingYard.h @@ -85,6 +85,8 @@ std::string calculate(const std::string & expr); // check for multiple instances of ? : std::string compute(const std::string & expr); +int http_request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result); + #endif } // namespace emsesp \ No newline at end of file diff --git a/src/core/system.cpp b/src/core/system.cpp index 1fca50f8a..fd48ecd4c 100644 --- a/src/core/system.cpp +++ b/src/core/system.cpp @@ -38,7 +38,7 @@ #include "../test/test.h" #endif -#ifndef NO_TLS_SUPPORT +#ifndef EMSESP_STANDALONE #define ENABLE_SMTP #define USE_ESP_SSLCLIENT #define READYCLIENT_SSL_CLIENT ESP_SSLClient @@ -138,7 +138,7 @@ bool System::command_sendmail(const char * value, const int8_t id) { bool success = false; -#ifndef NO_TLS_SUPPORT +#ifndef EMSESP_STANDALONE WiFiClient * basic_client = new WiFiClient; ESP_SSLClient * ssl_client = new ESP_SSLClient; ReadyClient * r_client = new ReadyClient(*ssl_client); @@ -2956,65 +2956,265 @@ bool System::uploadFirmwareURL(const char * url) { Shell::loop_all(); // flush log buffers so latest messages are shown in console - // Configure temporary client - HTTPClient http; - http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); // important for GitHub 302's - http.setTimeout(8000); - http.useHTTP10(true); // use HTTP/1.0 for update since the update handler does not support any transfer Encoding - http.begin(saved_url); + // detect scheme (case-insensitive) + String scheme = saved_url.substring(0, 8); + scheme.toLowerCase(); + const bool is_https = scheme.startsWith("https://"); - // start a connection, returns -1 if fails - int httpCode = http.GET(); - if (httpCode != HTTP_CODE_OK) { - LOG_ERROR("Firmware upload failed - HTTP code %d", httpCode); - http.end(); - return false; // error + HTTPClient http; + WiFiClient basic_client; + ESP_SSLClient ssl_client; + + Stream * stream = nullptr; + int firmware_size = 0; + + if (is_https) { + ssl_client.setInsecure(); // no CA validation, matches the rest of the project + // BearSSL needs a receive buffer large enough to hold one full TLS record. + // GitHub's release-assets CDN sends standard up-to-16 KB records and does NOT + // negotiate max_fragment_length, so a small (e.g. 1 KB) RX buffer makes the + // body unreadable (headers still fit one small record, hence Content-Length + // looks fine, but the first body record cannot be decoded). 16384 + overhead + // is the safe value the library itself uses by default; we go a bit smaller + // to be friendlier to 4 MB / no-PSRAM boards while still big enough for any + // record the CDN actually sends in practice. + ssl_client.setBufferSizes(16384, 1024); + ssl_client.setSessionTimeout(120); + basic_client.setTimeout(15000); // socket-level read timeout + ssl_client.setTimeout(15000); // Stream::readBytes timeout used by Update + ssl_client.setClient(&basic_client); + + String url_remain = saved_url.substring(8); // strip "https://" + int redirect_count = 0; + + while (true) { + // split url_remain into host and path + String host; + String path; + int s = url_remain.indexOf('/'); + if (s < 0) { + host = url_remain; + path = "/"; + } else { + host = url_remain.substring(0, s); + path = url_remain.substring(s); + } + + LOG_DEBUG("Connecting to %s", host.c_str()); + if (!ssl_client.connect(host.c_str(), 443)) { + LOG_ERROR("Firmware upload failed - HTTPS connection failed"); + return false; + } + + // send a minimal HTTP/1.0 GET so we don't have to deal with chunked encoding + ssl_client.print("GET "); + ssl_client.print(path); + ssl_client.println(" HTTP/1.0"); + ssl_client.print("Host: "); + ssl_client.println(host); + ssl_client.println("User-Agent: EMS-ESP"); + ssl_client.println("Connection: close"); + ssl_client.print("\r\n"); + + // wait for the first byte (up to 8s, matching the previous HTTP timeout) + uint32_t ms = millis(); + while (ssl_client.connected() && !ssl_client.available() && millis() - ms < 8000) { + delay(1); + } + + // parse status line: "HTTP/1.x CODE TEXT" + String status_line = ssl_client.readStringUntil('\n'); + int sp = status_line.indexOf(' '); + int http_code = (sp >= 0) ? status_line.substring(sp + 1, sp + 4).toInt() : 0; + + // parse response headers, looking for Content-Length and Location + int content_length = -1; + String location; + while (ssl_client.connected() || ssl_client.available()) { + String line = ssl_client.readStringUntil('\n'); + line.trim(); + if (line.isEmpty()) { + break; // end of headers + } + int colon = line.indexOf(':'); + if (colon < 0) { + continue; + } + String name = line.substring(0, colon); + name.toLowerCase(); + String val = line.substring(colon + 1); + val.trim(); + if (name == "content-length") { + content_length = val.toInt(); + } else if (name == "location") { + location = val; + } + } + + // follow redirects manually (GitHub releases redirect to objects.githubusercontent.com) + if (http_code == 301 || http_code == 302 || http_code == 303 || http_code == 307 || http_code == 308) { + ssl_client.stop(); + if (location.isEmpty() || ++redirect_count > 5) { + LOG_ERROR("Firmware upload failed - too many redirects"); + return false; + } + String lower_loc = location; + lower_loc.toLowerCase(); + if (lower_loc.startsWith("https://")) { + url_remain = location.substring(8); + } else if (location.startsWith("/")) { + url_remain = host + location; // relative redirect, same host + } else { + LOG_ERROR("Firmware upload failed - non-HTTPS redirect to %s", location.c_str()); + return false; + } + LOG_DEBUG("Following redirect to %s", url_remain.c_str()); + continue; + } + + if (http_code != HTTP_CODE_OK) { + ssl_client.stop(); + LOG_ERROR("Firmware upload failed - HTTP code %d", http_code); + return false; + } + + if (content_length <= 0) { + ssl_client.stop(); + LOG_ERROR("Firmware upload failed - missing Content-Length"); + return false; + } + + // wait for the first byte of the body so Update.writeStream's peek() sees real data + // (headers and body may arrive in separate TLS records) + uint32_t body_wait = millis(); + while (ssl_client.connected() && !ssl_client.available() && millis() - body_wait < 8000) { + delay(1); + } + if (!ssl_client.available()) { + ssl_client.stop(); + LOG_ERROR("Firmware upload failed - no body received"); + return false; + } + + stream = &ssl_client; + firmware_size = content_length; + break; + } + } else { + // HTTP path + http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); // important for GitHub 302's + http.setTimeout(8000); + http.useHTTP10(true); // use HTTP/1.0 for update since the update handler does not support any transfer Encoding + http.begin(saved_url); + + int httpCode = http.GET(); + if (httpCode != HTTP_CODE_OK) { + LOG_ERROR("Firmware upload failed - HTTP code %d", httpCode); + http.end(); + return false; + } + + firmware_size = http.getSize(); + stream = http.getStreamPtr(); } - int firmware_size = http.getSize(); - // check we have a valid size - if (firmware_size < 2097152) { // 2MB or greater is required + if (firmware_size < 1677721) { // 1.6MB or greater is required LOG_ERROR("Firmware upload failed - invalid size"); - http.end(); return false; // error } // check we have enough space for the upload in the ota partition if (!Update.begin(firmware_size)) { LOG_ERROR("Firmware upload failed - no space"); - http.end(); return false; // error } - LOG_INFO("Firmware uploading (size: %d KB). Please wait...", firmware_size / 1024); + LOG_INFO("Firmware uploading (size: %d KB) over %s. Please wait...", firmware_size / 1024, is_https ? "HTTPS" : "HTTP"); Shell::loop_all(); // flush log buffers so latest messages are shown in console // we're about to start the upload, set the status so the Web System Monitor spots it EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING); - // set a callback so we can monitor progress in the WebUI - Update.onProgress([](size_t progress, size_t total) { EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING + (progress * 100 / total)); }); + // explicit chunked read loop instead of Update.writeStream(): + constexpr size_t CHUNK_SIZE = 1024; + constexpr uint32_t READ_TIMEOUT_MS = 30000; // overall stall timeout per chunk + uint8_t buf[CHUNK_SIZE]; + size_t total_read = 0; + bool magic_ok = false; + int last_pct = -1; - // get tcp stream and send it to Updater - WiFiClient * stream = http.getStreamPtr(); - if (Update.writeStream(*stream) != firmware_size) { - LOG_ERROR("Firmware upload failed - size differences"); - http.end(); - EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); - return false; // error + while (total_read < (size_t)firmware_size) { + // wait for some data or for the connection to drop + uint32_t wait_start = millis(); + while (!stream->available()) { + const bool still_connected = is_https ? ssl_client.connected() : http.connected(); + if (!still_connected) { + break; + } + if (millis() - wait_start > READ_TIMEOUT_MS) { + break; + } + delay(1); + } + + if (!stream->available()) { + LOG_ERROR("Firmware upload failed - read stalled at %u of %d bytes", (unsigned)total_read, firmware_size); + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); + return false; + } + + size_t want = (size_t)firmware_size - total_read; + if (want > CHUNK_SIZE) { + want = CHUNK_SIZE; + } + + size_t n = stream->readBytes(buf, want); + if (n == 0) { + LOG_ERROR("Firmware upload failed - read returned 0 at %u of %d bytes", (unsigned)total_read, firmware_size); + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); + return false; + } + + // verify the ESP image magic byte the very first time so we fail fast with a + // clear message if the URL points at the wrong asset (HTML, archive, ...) + if (!magic_ok) { + if (buf[0] != 0xE9) { + LOG_ERROR("Firmware upload failed - bad magic byte 0x%02X (expected 0xE9, not an ESP32 firmware image?)", buf[0]); + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); + return false; + } + magic_ok = true; + } + + if (Update.write(buf, n) != n) { + LOG_ERROR("Firmware upload failed - flash write error at %u of %d bytes: %s", + (unsigned)total_read, + firmware_size, + Update.errorString()); + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); + return false; + } + + total_read += n; + + // update the WebUI status, but only when the percentage actually changes + int pct = (int)(total_read * 100 / (size_t)firmware_size); + if (pct != last_pct) { + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING + pct); + last_pct = pct; + } + + yield(); } if (!Update.end(true)) { - LOG_ERROR("Firmware upload failed - general error"); - http.end(); + LOG_ERROR("Firmware upload failed - %s", Update.errorString()); EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); return false; // error } - // finished with upload - http.end(); saved_url.clear(); // prevent from downloading again LOG_INFO("Firmware uploaded successfully. Restarting..."); EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_PENDING_RESTART); diff --git a/src/web/WebSettingsService.cpp b/src/web/WebSettingsService.cpp index 93d3f956b..38769e33d 100644 --- a/src/web/WebSettingsService.cpp +++ b/src/web/WebSettingsService.cpp @@ -83,19 +83,15 @@ void WebSettings::read(WebSettings & settings, JsonObject root) { root["modbus_max_clients"] = settings.modbus_max_clients; root["modbus_timeout"] = settings.modbus_timeout; root["developer_mode"] = settings.developer_mode; -#ifndef NO_TLS_SUPPORT - root["email_enabled"] = settings.email_enabled; -#else - root["email_enabled"] = false; -#endif - root["email_security"] = settings.email_security; - root["email_server"] = settings.email_server; - root["email_port"] = settings.email_port; - root["email_login"] = settings.email_login; - root["email_pass"] = settings.email_pass; - root["email_sender"] = settings.email_sender; - root["email_recp"] = settings.email_recp; - root["email_subject"] = settings.email_subject; + root["email_enabled"] = settings.email_enabled; + root["email_security"] = settings.email_security; + root["email_server"] = settings.email_server; + root["email_port"] = settings.email_port; + root["email_login"] = settings.email_login; + root["email_pass"] = settings.email_pass; + root["email_sender"] = settings.email_sender; + root["email_recp"] = settings.email_recp; + root["email_subject"] = settings.email_subject; } // call on initialization and also when settings are updated/saved via web or console