large jsondocs in PSRAM

src/core/analogsensor.cpp

@@ -675,7 +675,7 @@ void AnalogSensor::publish_values(const bool force) {
         }
     }
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     JsonObject   obj            = doc.to<JsonObject>();
     bool         ha_dev_created = false;
 
@@ -704,7 +704,7 @@ void AnalogSensor::publish_values(const bool force) {
         if (Mqtt::ha_enabled() && (!sensor.ha_registered || force)) {
             LOG_DEBUG("Recreating HA config for analog sensor GPIO %02d", sensor.gpio());
 
-            JsonDocument config;
+            JsonDocument config(PSRAM_DOC);
             config["~"] = Mqtt::base();
 
             char stat_t[50];

src/core/emsesp.cpp

@@ -436,7 +436,7 @@ void EMSESP::show_device_values(uuid::console::Shell & shell) {
                 // print header, with device type translated
                 shell.printfln("%s: %s (%d)", emsdevice->device_type_2_device_name_translated(), emsdevice->to_string().c_str(), emsdevice->count_entities());
 
-                JsonDocument doc;
+                JsonDocument doc(PSRAM_DOC);
                 JsonObject   json = doc.to<JsonObject>();
 
                 emsdevice->generate_values(json, DeviceValueTAG::TAG_NONE, true, EMSdevice::OUTPUT_TARGET::CONSOLE);
@@ -460,7 +460,7 @@ void EMSESP::show_device_values(uuid::console::Shell & shell) {
     // show any custom entities
     if (webCustomEntityService.count_entities() > 0) {
         shell.printfln("Custom Entities:");
-        JsonDocument custom_doc; // use max size
+        JsonDocument custom_doc(PSRAM_DOC); // use max size
         JsonObject   custom_output = custom_doc.to<JsonObject>();
         webCustomEntityService.show_values(custom_output);
         for (JsonPair p : custom_output) {
@@ -625,7 +625,7 @@ void EMSESP::reset_mqtt_ha() {
 // this will also create the HA /config topic for each device value
 // generate_values_json is called to build the device value (dv) object array
 void EMSESP::publish_device_values(uint8_t device_type) {
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     JsonObject   json         = doc.to<JsonObject>();
     bool         need_publish = false;
     bool         nested       = (Mqtt::is_nested());

src/core/emsesp.h

@@ -56,6 +56,7 @@
 #include "../web/WebCustomEntityService.h"
 #include "../web/WebModulesService.h"
 
+#include "psram_json_allocator.h"
 #include "emsdevicevalue.h"
 #include "emsdevice.h"
 #include "emsfactory.h"
@@ -82,7 +83,19 @@ class Module {}; // forward declaration
 #define WATCH_ID_NONE 0 // no watch id set
 
 // helpers for callback functions
-#define MAKE_PF_CB(__f) [&](const std::shared_ptr<const Telegram> & t) { __f(t); }          // for Process Function callbacks to EMSDevice::process_function_p
+//
+// MAKE_PF_CB(member) produces a non-capturing trampoline that decays to a
+// plain function pointer (EMSdevice::process_function_p). The outer IILE
+// (immediately-invoked lambda expression) captures `this` purely to deduce
+// the derived-class type via decltype; the inner lambda is non-capturing and
+// therefore convertible to a function pointer via the unary `+` operator.
+// Result: zero heap (no std::function control block) and direct dispatch.
+#define MAKE_PF_CB(__f)                                                                                                                                        \
+    ([this]() {                                                                                                                                                \
+        using SelfT = std::remove_pointer_t<decltype(this)>;                                                                                                   \
+        return +[](emsesp::EMSdevice * dev, const std::shared_ptr<const Telegram> & t) { static_cast<SelfT *>(dev)->__f(t); };                                 \
+    }())
+
 #define MAKE_CF_CB(__f) [&](const char * value, const int8_t id) { return __f(value, id); } // for Command Function callbacks Command::cmd_function_p
 
 namespace emsesp {

src/core/mqtt.cpp

@@ -519,7 +519,7 @@ void Mqtt::on_connect() {
 // e.g. homeassistant/sensor/ems-esp/status/config
 // all the values from the heartbeat payload will be added as attributes to the entity state
 void Mqtt::ha_status() {
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
 
     char uniq[70];
     if (Mqtt::entity_format() == entityFormat::MULTI_SHORT) {
@@ -981,7 +981,7 @@ bool Mqtt::publish_ha_sensor_config(uint8_t               type,        // EMSdev
     }
 
     // build the full topic's payload
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     doc["~"]       = Mqtt::base();
     doc["uniq_id"] = uniq_id;
 
@@ -1406,7 +1406,7 @@ bool Mqtt::publish_ha_climate_config(const DeviceValue & dv, const bool has_room
     snprintf(temp_cmd_s, sizeof(temp_cmd_s), "~/%s/%s%d/seltemp", devicename, tagname, hc_num);
     snprintf(mode_cmd_s, sizeof(mode_cmd_s), "~/%s/%s%d/mode", devicename, tagname, hc_num);
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
 
     doc["~"]             = Mqtt::base();
     doc["uniq_id"]       = uniq_id_s;

src/core/psram_json_allocator.h

@@ -0,0 +1,106 @@
+/*
+ * EMS-ESP - https://github.com/emsesp/EMS-ESP
+ * Copyright 2020-2025  emsesp.org
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EMSESP_PSRAM_JSON_ALLOCATOR_H
+#define EMSESP_PSRAM_JSON_ALLOCATOR_H
+
+#include <ArduinoJson.h>
+
+#ifndef EMSESP_STANDALONE
+#include <esp_heap_caps.h>
+#endif
+
+namespace emsesp {
+
+// PSRAM-backed ArduinoJson allocator with internal-heap fallback.
+//
+// Rationale: by default ArduinoJson allocates with malloc(), which on the
+// ESP32 lands in internal DRAM. Large transient JsonDocuments (full MQTT
+// payload, HA discovery configs, Web API responses, settings load/save)
+// were eating multiple KB of the same internal heap that LwIP, mbedTLS and
+// AsyncTCP also need. Routing them through SPIRAM via heap_caps_malloc
+// keeps internal heap available for the network stack, at the cost of a
+// small latency penalty on PSRAM reads/writes (a few cycles per access,
+// negligible for JSON build-up which is dominated by string formatting).
+//
+// On the standalone (Linux) build, PSRAM doesn't exist; the allocator
+// silently falls through to plain malloc/free/realloc.
+//
+// Usage:
+//   JsonDocument doc(emsesp::PsramJsonAllocator::instance());
+// or with the convenience macro:
+//   JsonDocument doc(PSRAM_DOC);
+class PsramJsonAllocator : public ArduinoJson::Allocator {
+  public:
+    void * allocate(size_t size) override {
+#ifdef EMSESP_STANDALONE
+        return malloc(size);
+#else
+        // Try SPIRAM first; fall back to internal heap so we never fail
+        // on boards without PSRAM or when PSRAM is full.
+        void * p = heap_caps_malloc(size, MALLOC_CAP_SPIRAM);
+        if (p == nullptr) {
+            p = malloc(size);
+        }
+        return p;
+#endif
+    }
+
+    void deallocate(void * ptr) override {
+#ifdef EMSESP_STANDALONE
+        free(ptr);
+#else
+        // heap_caps_free handles both PSRAM- and internal-heap pointers.
+        heap_caps_free(ptr);
+#endif
+    }
+
+    void * reallocate(void * ptr, size_t new_size) override {
+#ifdef EMSESP_STANDALONE
+        return realloc(ptr, new_size);
+#else
+        // Prefer keeping the block in PSRAM; heap_caps_realloc will move
+        // the data if the original region can't be grown in-place.
+        void * p = heap_caps_realloc(ptr, new_size, MALLOC_CAP_SPIRAM);
+        if (p == nullptr) {
+            p = realloc(ptr, new_size);
+        }
+        return p;
+#endif
+    }
+
+    static ArduinoJson::Allocator * instance() {
+        static PsramJsonAllocator inst;
+        return &inst;
+    }
+
+  private:
+    PsramJsonAllocator()  = default;
+    ~PsramJsonAllocator() = default;
+};
+
+} // namespace emsesp
+
+// Convenience shorthand. Use only for *large* or *transient* JsonDocuments
+// (MQTT publish payloads, HA discovery, full API responses, big settings
+// load/save). For small hot-path docs (single-command output, parse of a
+// short HTTP body), keep the default allocator: PSRAM has higher access
+// latency than internal SRAM, so tiny docs are faster on the regular heap.
+#define PSRAM_DOC emsesp::PsramJsonAllocator::instance()
+
+#endif

src/core/temperaturesensor.cpp

@@ -494,7 +494,7 @@ void TemperatureSensor::publish_values(const bool force) {
         }
     }
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     bool         ha_dev_created = false;
 
     for (auto & sensor : sensors_) {
@@ -519,7 +519,7 @@ void TemperatureSensor::publish_values(const bool force) {
             } else if (!sensor.ha_registered || force) {
                 LOG_DEBUG("Recreating HA config for sensor ID %s", sensor.id());
 
-                JsonDocument config;
+                JsonDocument config(PSRAM_DOC);
                 config["~"]        = Mqtt::base();
                 config["dev_cla"]  = "temperature";
                 config["stat_cla"] = "measurement";

src/web/WebCustomEntityService.cpp

@@ -69,7 +69,7 @@ void WebCustomEntity::read(WebCustomEntity & webEntity, JsonObject root) {
 StateUpdateResult WebCustomEntity::update(JsonObject root, WebCustomEntity & webCustomEntity) {
     // reset everything to start fresh
     Command::erase_device_commands(EMSdevice::DeviceType::CUSTOM);
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     for (CustomEntityItem & entityItem : webCustomEntity.customEntityItems) {
         if (entityItem.raw) {
             delete[] entityItem.raw;
@@ -453,7 +453,7 @@ void WebCustomEntityService::publish_single(CustomEntityItem & entity) {
         snprintf(topic, sizeof(topic), "%s_data/%s", F_(custom), entity.name);
     }
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     JsonObject   output = doc.to<JsonObject>();
     render_value(output, entity, true);
     Mqtt::queue_publish(topic, output["value"].as<std::string>());
@@ -475,7 +475,7 @@ void WebCustomEntityService::publish(const bool force) {
         }
     }
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     JsonObject   output     = doc.to<JsonObject>();
     bool         ha_created = ha_configdone_;
 
@@ -486,7 +486,7 @@ void WebCustomEntityService::publish(const bool force) {
         render_value(output, entityItem);
         // create HA config
         if (Mqtt::ha_enabled() && !ha_configdone_) {
-            JsonDocument config;
+            JsonDocument config(PSRAM_DOC);
             config["~"] = Mqtt::base();
 
             char stat_t[50];
@@ -566,7 +566,7 @@ uint8_t WebCustomEntityService::count_entities() {
         return 0;
     }
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     JsonObject   output = doc.to<JsonObject>();
     uint8_t      count  = 0;
 

src/web/WebCustomizationService.cpp

@@ -212,7 +212,7 @@ void WebCustomizationService::device_entities(AsyncWebServerRequest * request) {
                 JsonArray output = response->getRoot();
                 emsdevice->generate_values_web_customization(output);
 #else
-                JsonDocument doc;
+                JsonDocument doc(PSRAM_DOC);
                 JsonArray    output = doc.to<JsonArray>();
                 emsdevice->generate_values_web_customization(output);
 #endif

src/web/WebModulesService.cpp

@@ -50,7 +50,7 @@ void WebModulesService::loop() {
 // it adds data to an empty 'root' json object
 // and also calls when the Modules web page is refreshed/loaded
 void WebModules::read(WebModules & webModules, JsonObject root) {
-    JsonDocument doc_modules;
+    JsonDocument doc_modules(PSRAM_DOC);
     auto         root_modules = doc_modules.to<JsonObject>();
     moduleLibrary.list(root_modules); // get list the external library modules, put in a json object
 

src/web/WebSchedulerService.cpp

@@ -266,7 +266,7 @@ void WebSchedulerService::publish(const bool force) {
         }
     }
 
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
     JsonObject   output     = doc.to<JsonObject>();
     bool         ha_created = ha_configdone_;
     for (const ScheduleItem & scheduleItem : *scheduleItems_) {
@@ -275,7 +275,7 @@ void WebSchedulerService::publish(const bool force) {
 
             // create HA config
             if (Mqtt::ha_enabled() && !ha_configdone_) {
-                JsonDocument config;
+                JsonDocument config(PSRAM_DOC);
                 config["~"] = Mqtt::base();
 
                 char stat_t[50];

src/web/WebStatusService.cpp

@@ -433,7 +433,7 @@ bool WebStatusService::refresh_versions_cache() {
         return false;
     }
 
-    JsonDocument         doc;
+    JsonDocument         doc(PSRAM_DOC);
     DeserializationError err = deserializeJson(doc, http.getStream());
     http.end();
     if (err) {
@@ -539,7 +539,7 @@ bool WebStatusService::exportData(JsonObject root, std::string & type) {
 // action = getCustomSupport
 // reads any upload customSupport.json file and sends to to Help page to be shown as Guest
 bool WebStatusService::getCustomSupport(JsonObject root) {
-    JsonDocument doc;
+    JsonDocument doc(PSRAM_DOC);
 
 #if defined(EMSESP_STANDALONE)
     // dummy test data for "test api3"