diff --git a/lib_standalone/WiFiClient.h b/lib_standalone/WiFiClient.h index 3266f5c87..cb8a36306 100644 --- a/lib_standalone/WiFiClient.h +++ b/lib_standalone/WiFiClient.h @@ -41,6 +41,8 @@ class WiFiClient { } void setTimeout(uint32_t){ + } + void setConnectionTimeout(uint32_t) { } // ESP32 socket option passthrough (e.g. TCP_NODELAY) int setSocketOption(int, int, const void *, size_t) { diff --git a/src/core/httpClient.cpp b/src/core/httpClient.cpp new file mode 100644 index 000000000..8f0cfe22d --- /dev/null +++ b/src/core/httpClient.cpp @@ -0,0 +1,152 @@ +/* + * EMS-ESP - https://github.com/emsesp/EMS-ESP + * Copyright 2020-2026 emsesp.org + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "emsesp.h" + +#include "httpClient.h" + +#include +#include + +namespace emsesp { + +int HttpClient::request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result) { + int httpResult = 0; + const bool is_post = value.length() || Helpers::toLower(method) == "post"; + const auto lower_url = Helpers::toLower(url.c_str()); + + const bool is_https = lower_url.starts_with("https://"); + if (!is_https && !lower_url.starts_with("http://")) { + return 0; // unsupported scheme + } + + WiFiClient * basic_client = new WiFiClient; + ESP_SSLClient * ssl_client = new ESP_SSLClient; + if (is_https) { + ssl_client->setInsecure(); + // Notes: with root CA we should set here: ssl_client->setCACert(rootCACert); + // 1 KB RX buffer is fine for small JSON-style endpoints used by the scheduler/shunting-yard but it is NOT enough for servers that send full-size TLS records (>1 KB) + ssl_client->setBufferSizes(16384, 1024); + ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds) + } + // WiFiClient is NetworkClient, which declares no setTimeout() of its own - calling it binds to + // Stream::setTimeout() and only affects readBytes(), leaving the socket on the core's 3s default + basic_client->setConnectionTimeout(CONNECT_TIMEOUT_MS); + ssl_client->setTimeout(5); // seconds, drives BearSSL only - unused on the plain HTTP path + ssl_client->setClient(basic_client, is_https); // enableSSL = false for plain HTTP + + url.replace(0, is_https ? 8 : 7, ""); + std::string host = url; + auto index = url.find_first_of('/'); + if (index != std::string::npos) { + host = url.substr(0, index); + url.replace(0, index, ""); + } else { + url = "/"; + } + + const uint16_t port = is_https ? 443 : 80; + if (ssl_client->connect(host.c_str(), port)) { + bool content_set = false; + bool agent_set = false; + + // assemble the request in one buffer. Sent as a dozen small writes it is left to Nagle to + // dribble them out an ACK at a time, which on a slow link can cost more than the read budget + std::string req = (is_post ? "POST " : "GET ") + url + " HTTP/1.1\r\nHost: " + host + "\r\n"; + for (JsonPairConst p : headers) { + const auto key = Helpers::toLower(p.key().c_str()); + content_set |= (key == "content-type"); + agent_set |= (key == "user-agent"); + req += std::string(p.key().c_str()) + ": " + p.value().as() + "\r\n"; + } + if (!agent_set) { + req += "User-Agent: EMS-ESP\r\n"; // CDNs are more likely to stall on a request without one + } + if (is_post) { + if (!content_set) { + req += "Content-Type: "; + req += value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain; + req += "\r\n"; + } + req += "Content-Length: " + std::to_string(value.length()) + "\r\n"; + } + req += "Connection: close\r\n\r\n"; // the blank line terminates the headers - without it the server never responds + + ssl_client->print(req.c_str()); + if (is_post && value.length()) { + ssl_client->print(value.c_str()); // sent separately so the body isn't copied into req + } + + // available() drops to zero between TCP segments, so stopping at the first gap truncates + // any response that doesn't arrive in a single packet. Keep reading until the peer closes, + // the stream goes idle, or the overall budget runs out + const uint32_t started = millis(); + uint32_t last_data = started; + while (millis() - started < TOTAL_TIMEOUT_MS) { + const int avail = ssl_client->available(); + if (avail > 0) { + uint8_t buf[128]; + const size_t want = (avail < (int)sizeof(buf)) ? (size_t)avail : sizeof(buf); + const int len = ssl_client->read(buf, want); + if (len > 0) { + result.append(reinterpret_cast(buf), len); + last_data = millis(); + } + continue; + } + if (!ssl_client->connected()) { + break; // closed, with nothing left buffered + } + // the server may take a while to start replying + if (millis() - last_data > (result.empty() ? FIRST_BYTE_TIMEOUT_MS : IDLE_TIMEOUT_MS)) { + break; + } + delay(1); + } + ssl_client->stop(); + + const auto received = result.length(); + + // parse the status line "HTTP/1.x ". stoi() would abort rather than throw on + // a malformed response, since the firmware is built with -fno-exceptions + if (result.starts_with("HTTP/")) { + index = result.find_first_of(' '); + if (index != std::string::npos) { + httpResult = Helpers::atoint(result.c_str() + index + 1); + } + } + + index = result.find("\r\n\r\n"); + if (index != std::string::npos) { + result.replace(0, index + 4, ""); + } + + if (httpResult == 0) { + // the TCP connect worked but nothing usable came back + EMSESP::logger().warning("%s no valid response from %s (%u bytes)", is_https ? "HTTPS" : "HTTP", host.c_str(), (unsigned)received); + } + } else { + EMSESP::logger().warning("%s connection to %s failed", is_https ? "HTTPS" : "HTTP", host.c_str()); + } + delete ssl_client; + delete basic_client; + + return httpResult; +} + +} // namespace emsesp diff --git a/src/core/httpClient.h b/src/core/httpClient.h new file mode 100644 index 000000000..c4bd0649a --- /dev/null +++ b/src/core/httpClient.h @@ -0,0 +1,41 @@ +/* + * EMS-ESP - https://github.com/emsesp/EMS-ESP + * Copyright 2020-2026 emsesp.org + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef EMSESP_HTTPCLIENT_H +#define EMSESP_HTTPCLIENT_H + +#include + +#include + +namespace emsesp { + +class HttpClient { + public: + static int request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result); + + private: + static constexpr uint32_t CONNECT_TIMEOUT_MS = 5000; // TCP connect, the core defaults to 3s + static constexpr uint32_t FIRST_BYTE_TIMEOUT_MS = 8000; // how long the server may take to start replying + static constexpr uint32_t IDLE_TIMEOUT_MS = 500; // gap in the stream that marks the end of a response + static constexpr uint32_t TOTAL_TIMEOUT_MS = 10000; // ceiling for the whole read +}; + +} // namespace emsesp + +#endif diff --git a/src/core/shuntingYard.cpp b/src/core/shuntingYard.cpp index 08957e880..202cf1e09 100644 --- a/src/core/shuntingYard.cpp +++ b/src/core/shuntingYard.cpp @@ -22,8 +22,7 @@ #include "shuntingYard.h" -#include -#include +#include "httpClient.h" namespace emsesp { @@ -336,7 +335,6 @@ bool isnum(const std::string & s) { return false; } - // replace commands like "//" with its value" std::string commands(std::string & expr, bool quotes) { auto expr_new = Helpers::toLower(expr); @@ -691,100 +689,6 @@ std::string calculate(const std::string & expr) { return result; } -// perform an HTTP/HTTPS request; returns the HTTP status code (0 on failure or unsupported scheme) -// the response headers are always stripped, so `result` contains only the body -int http_request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result) { - int httpResult = 0; - const bool is_post = value.length() || Helpers::toLower(method) == "post"; - const auto lower_url = Helpers::toLower(url.c_str()); - - const bool is_https = lower_url.starts_with("https://"); - if (!is_https && !lower_url.starts_with("http://")) { - return 0; // unsupported scheme - } - - WiFiClient * basic_client = new WiFiClient; - ESP_SSLClient * ssl_client = new ESP_SSLClient; - if (is_https) { - ssl_client->setInsecure(); // with root CA we should set here: ssl_client->setCACert(rootCACert); - // NOTE: 1 KB RX buffer is fine for small JSON-style endpoints used by the scheduler/shunting-yard, - // but it is NOT enough for servers that send full-size TLS records (>1 KB), e.g. GitHub release - // assets / large CDN responses. Such servers do not negotiate max_fragment_length, so the body - // can't be decoded and reads return 0. If this path is ever used to fetch large or CDN-hosted - // payloads, bump the RX buffer to 16384 (see uploadFirmwareURL in core/system.cpp for reference). - ssl_client->setBufferSizes(16384, 1024); - ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds) - } - basic_client->setTimeout(5000); // socket-level read timeout - ssl_client->setTimeout(5); // Stream::readBytes timeout used by Update - ssl_client->setClient(basic_client, is_https); // enableSSL = false for plain HTTP - - url.replace(0, is_https ? 8 : 7, ""); - std::string host = url; - auto index = url.find_first_of('/'); - if (index != std::string::npos) { - host = url.substr(0, index); - url.replace(0, index, ""); - } else { - url = "/"; - } - - const uint16_t port = is_https ? 443 : 80; - if (ssl_client->connect(host.c_str(), port)) { - bool content_set = false; - ssl_client->print(is_post ? "POST " : "GET "); - ssl_client->print(url.c_str()); - ssl_client->println(" HTTP/1.1"); - ssl_client->print("Host: "); - ssl_client->println(host.c_str()); - for (JsonPairConst p : headers) { - content_set |= (Helpers::toLower(p.key().c_str()) == "content-type"); - ssl_client->print(p.key().c_str()); - ssl_client->print(": "); - ssl_client->println(p.value().as().c_str()); - } - if (is_post) { - if (!content_set) { - ssl_client->print("Content-Type: "); - ssl_client->println(value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain); - } - ssl_client->print("Content-Length: "); - ssl_client->println(value.length()); - ssl_client->println("Connection: close"); - ssl_client->print("\r\n"); - ssl_client->print(value.c_str()); - } else { - ssl_client->println("Connection: close"); - ssl_client->print("\r\n"); // terminate headers - without this the server never responds - } - - auto ms = millis(); - while (ssl_client->connected() && !ssl_client->available() && millis() - ms < 3000) { - delay(1); - } - - while (ssl_client->available()) { - result += (char)ssl_client->read(); - } - ssl_client->stop(); - - index = result.find_first_of(' '); - if (index != std::string::npos) { - httpResult = stoi(result.substr(index + 1, 3)); - } - index = result.find("\r\n\r\n"); - if (index != std::string::npos) { - result.replace(0, index + 4, ""); - } - } else { - EMSESP::logger().warning("%s connection failed", is_https ? "HTTPS" : "HTTP"); - } - delete ssl_client; - delete basic_client; - - return httpResult; -} - // check for multiple instances of ? : std::string compute(const std::string & expr) { std::string expr_new = expr; @@ -829,7 +733,7 @@ std::string compute(const std::string & expr) { std::string method = doc[method_s] | "GET"; std::string result; - int httpResult = http_request(url, method, value, doc[header_s].as(), result); + int httpResult = HttpClient::request(url, method, value, doc[header_s].as(), result); if (httpResult == 200) { std::string key = doc[key_s] | ""; JsonDocument keys_doc; // JsonDocument to hold "keys" after doc is parsed with HTTP body diff --git a/src/core/shuntingYard.h b/src/core/shuntingYard.h index 23e8486ec..7f8f66f77 100644 --- a/src/core/shuntingYard.h +++ b/src/core/shuntingYard.h @@ -84,8 +84,6 @@ std::string calculate(const std::string & expr); // check for multiple instances of ? : std::string compute(const std::string & expr); -int http_request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result); +} // namespace emsesp -#endif - -} // namespace emsesp \ No newline at end of file +#endif \ No newline at end of file diff --git a/src/web/WebCommandService.cpp b/src/web/WebCommandService.cpp index bf71c4e4c..35350599d 100644 --- a/src/web/WebCommandService.cpp +++ b/src/web/WebCommandService.cpp @@ -20,6 +20,7 @@ #include "WebCommandService.h" #include "shuntingYard.h" +#include "httpClient.h" namespace emsesp { @@ -267,7 +268,7 @@ bool WebCommandService::executeCommand(const char * name, const std::string & co auto lower_url = Helpers::toLower(url.c_str()); if (lower_url.starts_with("http://") || lower_url.starts_with("https://")) { std::string result; - int httpResult = http_request(url, method, value, doc["header"].as(), result); + int httpResult = HttpClient::request(url, method, value, doc["header"].as(), result); if (httpResult != 200) { EMSESP::logger().warning("Command '%s': URL command failed with http code %d", name, httpResult); return false; diff --git a/src/web/WebStatusService.cpp b/src/web/WebStatusService.cpp index ce4fd1a61..5938fa009 100644 --- a/src/web/WebStatusService.cpp +++ b/src/web/WebStatusService.cpp @@ -20,10 +20,8 @@ #ifndef EMSESP_STANDALONE #include -#include -#include #endif -#include "shuntingYard.h" +#include "httpClient.h" namespace emsesp { @@ -427,9 +425,9 @@ bool WebStatusService::refresh_versions_cache() { #else std::string result; JsonDocument doc; - auto http_code = http_request(VERSIONS_URL, "GET", "", doc.as(), result); + auto http_code = HttpClient::request(VERSIONS_URL, "GET", "", doc.as(), result); if (http_code != 200) { - EMSESP::logger().warning("refresh_versions_cache() HTTP error code %d", http_code); + EMSESP::logger().warning("Unable to retrieve online version information (HTTP error code %d)", http_code); return false; } DeserializationError err = deserializeJson(doc, result);