improve http fallouts, move to its own class

lib_standalone/WiFiClient.h

@@ -41,6 +41,8 @@ class WiFiClient {
     }
     void setTimeout(uint32_t){
         
+    }
+    void setConnectionTimeout(uint32_t) {
     }
     // ESP32 socket option passthrough (e.g. TCP_NODELAY)
     int setSocketOption(int, int, const void *, size_t) {

src/core/httpClient.cpp

@@ -0,0 +1,152 @@
+/*
+ * EMS-ESP - https://github.com/emsesp/EMS-ESP
+ * Copyright 2020-2026  emsesp.org
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "emsesp.h"
+
+#include "httpClient.h"
+
+#include <WiFiClient.h>
+#include <ESP_SSLClient.h>
+
+namespace emsesp {
+
+int HttpClient::request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result) {
+    int        httpResult = 0;
+    const bool is_post    = value.length() || Helpers::toLower(method) == "post";
+    const auto lower_url  = Helpers::toLower(url.c_str());
+
+    const bool is_https = lower_url.starts_with("https://");
+    if (!is_https && !lower_url.starts_with("http://")) {
+        return 0; // unsupported scheme
+    }
+
+    WiFiClient *    basic_client = new WiFiClient;
+    ESP_SSLClient * ssl_client   = new ESP_SSLClient;
+    if (is_https) {
+        ssl_client->setInsecure();
+        // Notes: with root CA we should set here: ssl_client->setCACert(rootCACert);
+        // 1 KB RX buffer is fine for small JSON-style endpoints used by the scheduler/shunting-yard but it is NOT enough for servers that send full-size TLS records (>1 KB)
+        ssl_client->setBufferSizes(16384, 1024);
+        ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds)
+    }
+    // WiFiClient is NetworkClient, which declares no setTimeout() of its own - calling it binds to
+    // Stream::setTimeout() and only affects readBytes(), leaving the socket on the core's 3s default
+    basic_client->setConnectionTimeout(CONNECT_TIMEOUT_MS);
+    ssl_client->setTimeout(5);                     // seconds, drives BearSSL only - unused on the plain HTTP path
+    ssl_client->setClient(basic_client, is_https); // enableSSL = false for plain HTTP
+
+    url.replace(0, is_https ? 8 : 7, "");
+    std::string host  = url;
+    auto        index = url.find_first_of('/');
+    if (index != std::string::npos) {
+        host = url.substr(0, index);
+        url.replace(0, index, "");
+    } else {
+        url = "/";
+    }
+
+    const uint16_t port = is_https ? 443 : 80;
+    if (ssl_client->connect(host.c_str(), port)) {
+        bool content_set = false;
+        bool agent_set   = false;
+
+        // assemble the request in one buffer. Sent as a dozen small writes it is left to Nagle to
+        // dribble them out an ACK at a time, which on a slow link can cost more than the read budget
+        std::string req = (is_post ? "POST " : "GET ") + url + " HTTP/1.1\r\nHost: " + host + "\r\n";
+        for (JsonPairConst p : headers) {
+            const auto key = Helpers::toLower(p.key().c_str());
+            content_set |= (key == "content-type");
+            agent_set |= (key == "user-agent");
+            req += std::string(p.key().c_str()) + ": " + p.value().as<std::string>() + "\r\n";
+        }
+        if (!agent_set) {
+            req += "User-Agent: EMS-ESP\r\n"; // CDNs are more likely to stall on a request without one
+        }
+        if (is_post) {
+            if (!content_set) {
+                req += "Content-Type: ";
+                req += value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain;
+                req += "\r\n";
+            }
+            req += "Content-Length: " + std::to_string(value.length()) + "\r\n";
+        }
+        req += "Connection: close\r\n\r\n"; // the blank line terminates the headers - without it the server never responds
+
+        ssl_client->print(req.c_str());
+        if (is_post && value.length()) {
+            ssl_client->print(value.c_str()); // sent separately so the body isn't copied into req
+        }
+
+        // available() drops to zero between TCP segments, so stopping at the first gap truncates
+        // any response that doesn't arrive in a single packet. Keep reading until the peer closes,
+        // the stream goes idle, or the overall budget runs out
+        const uint32_t started   = millis();
+        uint32_t       last_data = started;
+        while (millis() - started < TOTAL_TIMEOUT_MS) {
+            const int avail = ssl_client->available();
+            if (avail > 0) {
+                uint8_t      buf[128];
+                const size_t want = (avail < (int)sizeof(buf)) ? (size_t)avail : sizeof(buf);
+                const int    len  = ssl_client->read(buf, want);
+                if (len > 0) {
+                    result.append(reinterpret_cast<const char *>(buf), len);
+                    last_data = millis();
+                }
+                continue;
+            }
+            if (!ssl_client->connected()) {
+                break; // closed, with nothing left buffered
+            }
+            // the server may take a while to start replying
+            if (millis() - last_data > (result.empty() ? FIRST_BYTE_TIMEOUT_MS : IDLE_TIMEOUT_MS)) {
+                break;
+            }
+            delay(1);
+        }
+        ssl_client->stop();
+
+        const auto received = result.length();
+
+        // parse the status line "HTTP/1.x <code> <reason>". stoi() would abort rather than throw on
+        // a malformed response, since the firmware is built with -fno-exceptions
+        if (result.starts_with("HTTP/")) {
+            index = result.find_first_of(' ');
+            if (index != std::string::npos) {
+                httpResult = Helpers::atoint(result.c_str() + index + 1);
+            }
+        }
+
+        index = result.find("\r\n\r\n");
+        if (index != std::string::npos) {
+            result.replace(0, index + 4, "");
+        }
+
+        if (httpResult == 0) {
+            // the TCP connect worked but nothing usable came back
+            EMSESP::logger().warning("%s no valid response from %s (%u bytes)", is_https ? "HTTPS" : "HTTP", host.c_str(), (unsigned)received);
+        }
+    } else {
+        EMSESP::logger().warning("%s connection to %s failed", is_https ? "HTTPS" : "HTTP", host.c_str());
+    }
+    delete ssl_client;
+    delete basic_client;
+
+    return httpResult;
+}
+
+} // namespace emsesp

src/core/httpClient.h

@@ -0,0 +1,41 @@
+/*
+ * EMS-ESP - https://github.com/emsesp/EMS-ESP
+ * Copyright 2020-2026  emsesp.org
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EMSESP_HTTPCLIENT_H
+#define EMSESP_HTTPCLIENT_H
+
+#include <ArduinoJson.h>
+
+#include <string>
+
+namespace emsesp {
+
+class HttpClient {
+  public:
+    static int request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result);
+
+  private:
+    static constexpr uint32_t CONNECT_TIMEOUT_MS    = 5000;  // TCP connect, the core defaults to 3s
+    static constexpr uint32_t FIRST_BYTE_TIMEOUT_MS = 8000;  // how long the server may take to start replying
+    static constexpr uint32_t IDLE_TIMEOUT_MS       = 500;   // gap in the stream that marks the end of a response
+    static constexpr uint32_t TOTAL_TIMEOUT_MS      = 10000; // ceiling for the whole read
+};
+
+} // namespace emsesp
+
+#endif

src/core/shuntingYard.cpp

@@ -22,8 +22,7 @@
 
 #include "shuntingYard.h"
 
-#include <WiFiClient.h>
+#include "httpClient.h"
-#include <ESP_SSLClient.h>
 
 namespace emsesp {
 
@@ -336,7 +335,6 @@ bool isnum(const std::string & s) {
     return false;
 }
 
-
 // replace commands like "<device>/<hc>/<cmd>" with its value"
 std::string commands(std::string & expr, bool quotes) {
     auto expr_new = Helpers::toLower(expr);
@@ -691,100 +689,6 @@ std::string calculate(const std::string & expr) {
     return result;
 }
 
-// perform an HTTP/HTTPS request; returns the HTTP status code (0 on failure or unsupported scheme)
-// the response headers are always stripped, so `result` contains only the body
-int http_request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result) {
-    int        httpResult = 0;
-    const bool is_post    = value.length() || Helpers::toLower(method) == "post";
-    const auto lower_url  = Helpers::toLower(url.c_str());
-
-    const bool is_https = lower_url.starts_with("https://");
-    if (!is_https && !lower_url.starts_with("http://")) {
-        return 0; // unsupported scheme
-    }
-
-    WiFiClient *    basic_client = new WiFiClient;
-    ESP_SSLClient * ssl_client   = new ESP_SSLClient;
-    if (is_https) {
-        ssl_client->setInsecure(); // with root CA we should set here: ssl_client->setCACert(rootCACert);
-        // NOTE: 1 KB RX buffer is fine for small JSON-style endpoints used by the scheduler/shunting-yard,
-        // but it is NOT enough for servers that send full-size TLS records (>1 KB), e.g. GitHub release
-        // assets / large CDN responses. Such servers do not negotiate max_fragment_length, so the body
-        // can't be decoded and reads return 0. If this path is ever used to fetch large or CDN-hosted
-        // payloads, bump the RX buffer to 16384 (see uploadFirmwareURL in core/system.cpp for reference).
-        ssl_client->setBufferSizes(16384, 1024);
-        ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds)
-    }
-    basic_client->setTimeout(5000);                // socket-level read timeout
-    ssl_client->setTimeout(5);                     // Stream::readBytes timeout used by Update
-    ssl_client->setClient(basic_client, is_https); // enableSSL = false for plain HTTP
-
-    url.replace(0, is_https ? 8 : 7, "");
-    std::string host  = url;
-    auto        index = url.find_first_of('/');
-    if (index != std::string::npos) {
-        host = url.substr(0, index);
-        url.replace(0, index, "");
-    } else {
-        url = "/";
-    }
-
-    const uint16_t port = is_https ? 443 : 80;
-    if (ssl_client->connect(host.c_str(), port)) {
-        bool content_set = false;
-        ssl_client->print(is_post ? "POST " : "GET ");
-        ssl_client->print(url.c_str());
-        ssl_client->println(" HTTP/1.1");
-        ssl_client->print("Host: ");
-        ssl_client->println(host.c_str());
-        for (JsonPairConst p : headers) {
-            content_set |= (Helpers::toLower(p.key().c_str()) == "content-type");
-            ssl_client->print(p.key().c_str());
-            ssl_client->print(": ");
-            ssl_client->println(p.value().as<std::string>().c_str());
-        }
-        if (is_post) {
-            if (!content_set) {
-                ssl_client->print("Content-Type: ");
-                ssl_client->println(value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain);
-            }
-            ssl_client->print("Content-Length: ");
-            ssl_client->println(value.length());
-            ssl_client->println("Connection: close");
-            ssl_client->print("\r\n");
-            ssl_client->print(value.c_str());
-        } else {
-            ssl_client->println("Connection: close");
-            ssl_client->print("\r\n"); // terminate headers - without this the server never responds
-        }
-
-        auto ms = millis();
-        while (ssl_client->connected() && !ssl_client->available() && millis() - ms < 3000) {
-            delay(1);
-        }
-
-        while (ssl_client->available()) {
-            result += (char)ssl_client->read();
-        }
-        ssl_client->stop();
-
-        index = result.find_first_of(' ');
-        if (index != std::string::npos) {
-            httpResult = stoi(result.substr(index + 1, 3));
-        }
-        index = result.find("\r\n\r\n");
-        if (index != std::string::npos) {
-            result.replace(0, index + 4, "");
-        }
-    } else {
-        EMSESP::logger().warning("%s connection failed", is_https ? "HTTPS" : "HTTP");
-    }
-    delete ssl_client;
-    delete basic_client;
-
-    return httpResult;
-}
-
 // check for multiple instances of <cond> ? <expr1> : <expr2>
 std::string compute(const std::string & expr) {
     std::string expr_new = expr;
@@ -829,7 +733,7 @@ std::string compute(const std::string & expr) {
             std::string method = doc[method_s] | "GET";
             std::string result;
 
-            int httpResult = http_request(url, method, value, doc[header_s].as<JsonObjectConst>(), result);
+            int httpResult = HttpClient::request(url, method, value, doc[header_s].as<JsonObjectConst>(), result);
             if (httpResult == 200) {
                 std::string  key = doc[key_s] | "";
                 JsonDocument keys_doc; // JsonDocument to hold "keys" after doc is parsed with HTTP body

src/core/shuntingYard.h

@@ -84,8 +84,6 @@ std::string calculate(const std::string & expr);
 // check for multiple instances of <cond> ? <expr1> : <expr2>
 std::string compute(const std::string & expr);
 
-int http_request(std::string url, const std::string & method, const std::string & value, JsonObjectConst headers, std::string & result);
+} // namespace emsesp
 
 #endif
-
-} // namespace emsesp

src/web/WebCommandService.cpp

@@ -20,6 +20,7 @@
 #include "WebCommandService.h"
 
 #include "shuntingYard.h"
+#include "httpClient.h"
 
 namespace emsesp {
 
@@ -267,7 +268,7 @@ bool WebCommandService::executeCommand(const char * name, const std::string & co
         auto lower_url = Helpers::toLower(url.c_str());
         if (lower_url.starts_with("http://") || lower_url.starts_with("https://")) {
             std::string result;
-            int         httpResult = http_request(url, method, value, doc["header"].as<JsonObjectConst>(), result);
+            int         httpResult = HttpClient::request(url, method, value, doc["header"].as<JsonObjectConst>(), result);
             if (httpResult != 200) {
                 EMSESP::logger().warning("Command '%s': URL command failed with http code %d", name, httpResult);
                 return false;

src/web/WebStatusService.cpp

@@ -20,10 +20,8 @@
 
 #ifndef EMSESP_STANDALONE
 #include <esp_ota_ops.h>
-#include <WiFiClient.h>
-#include <ESP_SSLClient.h>
 #endif
-#include "shuntingYard.h"
+#include "httpClient.h"
 
 namespace emsesp {
 
@@ -427,9 +425,9 @@ bool WebStatusService::refresh_versions_cache() {
 #else
     std::string  result;
     JsonDocument doc;
-    auto         http_code = http_request(VERSIONS_URL, "GET", "", doc.as<JsonObjectConst>(), result);
+    auto         http_code = HttpClient::request(VERSIONS_URL, "GET", "", doc.as<JsonObjectConst>(), result);
     if (http_code != 200) {
-        EMSESP::logger().warning("refresh_versions_cache() HTTP error code %d", http_code);
+        EMSESP::logger().warning("Unable to retrieve online version information (HTTP error code %d)", http_code);
         return false;
     }
     DeserializationError err = deserializeJson(doc, result);