From 5f9eb8d6d2e28ad0497a859e1103eceb852e1433 Mon Sep 17 00:00:00 2001 From: proddy Date: Sat, 16 May 2026 16:26:04 +0200 Subject: [PATCH] large jsondocs in PSRAM --- src/core/analogsensor.cpp | 4 +- src/core/emsesp.cpp | 6 +- src/core/emsesp.h | 15 +++- src/core/mqtt.cpp | 6 +- src/core/psram_json_allocator.h | 106 ++++++++++++++++++++++++++++ src/core/temperaturesensor.cpp | 4 +- src/web/WebCustomEntityService.cpp | 10 +-- src/web/WebCustomizationService.cpp | 2 +- src/web/WebModulesService.cpp | 2 +- src/web/WebSchedulerService.cpp | 4 +- src/web/WebStatusService.cpp | 4 +- 11 files changed, 141 insertions(+), 22 deletions(-) create mode 100644 src/core/psram_json_allocator.h diff --git a/src/core/analogsensor.cpp b/src/core/analogsensor.cpp index 0a27ebdbf..835d62bd9 100644 --- a/src/core/analogsensor.cpp +++ b/src/core/analogsensor.cpp @@ -675,7 +675,7 @@ void AnalogSensor::publish_values(const bool force) { } } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject obj = doc.to(); bool ha_dev_created = false; @@ -704,7 +704,7 @@ void AnalogSensor::publish_values(const bool force) { if (Mqtt::ha_enabled() && (!sensor.ha_registered || force)) { LOG_DEBUG("Recreating HA config for analog sensor GPIO %02d", sensor.gpio()); - JsonDocument config; + JsonDocument config(PSRAM_DOC); config["~"] = Mqtt::base(); char stat_t[50]; diff --git a/src/core/emsesp.cpp b/src/core/emsesp.cpp index 5055e1af4..7732197e9 100644 --- a/src/core/emsesp.cpp +++ b/src/core/emsesp.cpp @@ -436,7 +436,7 @@ void EMSESP::show_device_values(uuid::console::Shell & shell) { // print header, with device type translated shell.printfln("%s: %s (%d)", emsdevice->device_type_2_device_name_translated(), emsdevice->to_string().c_str(), emsdevice->count_entities()); - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject json = doc.to(); emsdevice->generate_values(json, DeviceValueTAG::TAG_NONE, true, EMSdevice::OUTPUT_TARGET::CONSOLE); @@ -460,7 +460,7 @@ void EMSESP::show_device_values(uuid::console::Shell & shell) { // show any custom entities if (webCustomEntityService.count_entities() > 0) { shell.printfln("Custom Entities:"); - JsonDocument custom_doc; // use max size + JsonDocument custom_doc(PSRAM_DOC); // use max size JsonObject custom_output = custom_doc.to(); webCustomEntityService.show_values(custom_output); for (JsonPair p : custom_output) { @@ -625,7 +625,7 @@ void EMSESP::reset_mqtt_ha() { // this will also create the HA /config topic for each device value // generate_values_json is called to build the device value (dv) object array void EMSESP::publish_device_values(uint8_t device_type) { - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject json = doc.to(); bool need_publish = false; bool nested = (Mqtt::is_nested()); diff --git a/src/core/emsesp.h b/src/core/emsesp.h index 3d7825a25..e4f49da79 100644 --- a/src/core/emsesp.h +++ b/src/core/emsesp.h @@ -56,6 +56,7 @@ #include "../web/WebCustomEntityService.h" #include "../web/WebModulesService.h" +#include "psram_json_allocator.h" #include "emsdevicevalue.h" #include "emsdevice.h" #include "emsfactory.h" @@ -82,7 +83,19 @@ class Module {}; // forward declaration #define WATCH_ID_NONE 0 // no watch id set // helpers for callback functions -#define MAKE_PF_CB(__f) [&](const std::shared_ptr & t) { __f(t); } // for Process Function callbacks to EMSDevice::process_function_p +// +// MAKE_PF_CB(member) produces a non-capturing trampoline that decays to a +// plain function pointer (EMSdevice::process_function_p). The outer IILE +// (immediately-invoked lambda expression) captures `this` purely to deduce +// the derived-class type via decltype; the inner lambda is non-capturing and +// therefore convertible to a function pointer via the unary `+` operator. +// Result: zero heap (no std::function control block) and direct dispatch. +#define MAKE_PF_CB(__f) \ + ([this]() { \ + using SelfT = std::remove_pointer_t; \ + return +[](emsesp::EMSdevice * dev, const std::shared_ptr & t) { static_cast(dev)->__f(t); }; \ + }()) + #define MAKE_CF_CB(__f) [&](const char * value, const int8_t id) { return __f(value, id); } // for Command Function callbacks Command::cmd_function_p namespace emsesp { diff --git a/src/core/mqtt.cpp b/src/core/mqtt.cpp index 81b7b2420..9477571c3 100644 --- a/src/core/mqtt.cpp +++ b/src/core/mqtt.cpp @@ -519,7 +519,7 @@ void Mqtt::on_connect() { // e.g. homeassistant/sensor/ems-esp/status/config // all the values from the heartbeat payload will be added as attributes to the entity state void Mqtt::ha_status() { - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); char uniq[70]; if (Mqtt::entity_format() == entityFormat::MULTI_SHORT) { @@ -981,7 +981,7 @@ bool Mqtt::publish_ha_sensor_config(uint8_t type, // EMSdev } // build the full topic's payload - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); doc["~"] = Mqtt::base(); doc["uniq_id"] = uniq_id; @@ -1406,7 +1406,7 @@ bool Mqtt::publish_ha_climate_config(const DeviceValue & dv, const bool has_room snprintf(temp_cmd_s, sizeof(temp_cmd_s), "~/%s/%s%d/seltemp", devicename, tagname, hc_num); snprintf(mode_cmd_s, sizeof(mode_cmd_s), "~/%s/%s%d/mode", devicename, tagname, hc_num); - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); doc["~"] = Mqtt::base(); doc["uniq_id"] = uniq_id_s; diff --git a/src/core/psram_json_allocator.h b/src/core/psram_json_allocator.h new file mode 100644 index 000000000..7469eaa91 --- /dev/null +++ b/src/core/psram_json_allocator.h @@ -0,0 +1,106 @@ +/* + * EMS-ESP - https://github.com/emsesp/EMS-ESP + * Copyright 2020-2025 emsesp.org + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef EMSESP_PSRAM_JSON_ALLOCATOR_H +#define EMSESP_PSRAM_JSON_ALLOCATOR_H + +#include + +#ifndef EMSESP_STANDALONE +#include +#endif + +namespace emsesp { + +// PSRAM-backed ArduinoJson allocator with internal-heap fallback. +// +// Rationale: by default ArduinoJson allocates with malloc(), which on the +// ESP32 lands in internal DRAM. Large transient JsonDocuments (full MQTT +// payload, HA discovery configs, Web API responses, settings load/save) +// were eating multiple KB of the same internal heap that LwIP, mbedTLS and +// AsyncTCP also need. Routing them through SPIRAM via heap_caps_malloc +// keeps internal heap available for the network stack, at the cost of a +// small latency penalty on PSRAM reads/writes (a few cycles per access, +// negligible for JSON build-up which is dominated by string formatting). +// +// On the standalone (Linux) build, PSRAM doesn't exist; the allocator +// silently falls through to plain malloc/free/realloc. +// +// Usage: +// JsonDocument doc(emsesp::PsramJsonAllocator::instance()); +// or with the convenience macro: +// JsonDocument doc(PSRAM_DOC); +class PsramJsonAllocator : public ArduinoJson::Allocator { + public: + void * allocate(size_t size) override { +#ifdef EMSESP_STANDALONE + return malloc(size); +#else + // Try SPIRAM first; fall back to internal heap so we never fail + // on boards without PSRAM or when PSRAM is full. + void * p = heap_caps_malloc(size, MALLOC_CAP_SPIRAM); + if (p == nullptr) { + p = malloc(size); + } + return p; +#endif + } + + void deallocate(void * ptr) override { +#ifdef EMSESP_STANDALONE + free(ptr); +#else + // heap_caps_free handles both PSRAM- and internal-heap pointers. + heap_caps_free(ptr); +#endif + } + + void * reallocate(void * ptr, size_t new_size) override { +#ifdef EMSESP_STANDALONE + return realloc(ptr, new_size); +#else + // Prefer keeping the block in PSRAM; heap_caps_realloc will move + // the data if the original region can't be grown in-place. + void * p = heap_caps_realloc(ptr, new_size, MALLOC_CAP_SPIRAM); + if (p == nullptr) { + p = realloc(ptr, new_size); + } + return p; +#endif + } + + static ArduinoJson::Allocator * instance() { + static PsramJsonAllocator inst; + return &inst; + } + + private: + PsramJsonAllocator() = default; + ~PsramJsonAllocator() = default; +}; + +} // namespace emsesp + +// Convenience shorthand. Use only for *large* or *transient* JsonDocuments +// (MQTT publish payloads, HA discovery, full API responses, big settings +// load/save). For small hot-path docs (single-command output, parse of a +// short HTTP body), keep the default allocator: PSRAM has higher access +// latency than internal SRAM, so tiny docs are faster on the regular heap. +#define PSRAM_DOC emsesp::PsramJsonAllocator::instance() + +#endif diff --git a/src/core/temperaturesensor.cpp b/src/core/temperaturesensor.cpp index 046c86d9b..493db0da2 100644 --- a/src/core/temperaturesensor.cpp +++ b/src/core/temperaturesensor.cpp @@ -494,7 +494,7 @@ void TemperatureSensor::publish_values(const bool force) { } } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); bool ha_dev_created = false; for (auto & sensor : sensors_) { @@ -519,7 +519,7 @@ void TemperatureSensor::publish_values(const bool force) { } else if (!sensor.ha_registered || force) { LOG_DEBUG("Recreating HA config for sensor ID %s", sensor.id()); - JsonDocument config; + JsonDocument config(PSRAM_DOC); config["~"] = Mqtt::base(); config["dev_cla"] = "temperature"; config["stat_cla"] = "measurement"; diff --git a/src/web/WebCustomEntityService.cpp b/src/web/WebCustomEntityService.cpp index 1e386f1b4..416091c19 100644 --- a/src/web/WebCustomEntityService.cpp +++ b/src/web/WebCustomEntityService.cpp @@ -69,7 +69,7 @@ void WebCustomEntity::read(WebCustomEntity & webEntity, JsonObject root) { StateUpdateResult WebCustomEntity::update(JsonObject root, WebCustomEntity & webCustomEntity) { // reset everything to start fresh Command::erase_device_commands(EMSdevice::DeviceType::CUSTOM); - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); for (CustomEntityItem & entityItem : webCustomEntity.customEntityItems) { if (entityItem.raw) { delete[] entityItem.raw; @@ -453,7 +453,7 @@ void WebCustomEntityService::publish_single(CustomEntityItem & entity) { snprintf(topic, sizeof(topic), "%s_data/%s", F_(custom), entity.name); } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject output = doc.to(); render_value(output, entity, true); Mqtt::queue_publish(topic, output["value"].as()); @@ -475,7 +475,7 @@ void WebCustomEntityService::publish(const bool force) { } } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject output = doc.to(); bool ha_created = ha_configdone_; @@ -486,7 +486,7 @@ void WebCustomEntityService::publish(const bool force) { render_value(output, entityItem); // create HA config if (Mqtt::ha_enabled() && !ha_configdone_) { - JsonDocument config; + JsonDocument config(PSRAM_DOC); config["~"] = Mqtt::base(); char stat_t[50]; @@ -566,7 +566,7 @@ uint8_t WebCustomEntityService::count_entities() { return 0; } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject output = doc.to(); uint8_t count = 0; diff --git a/src/web/WebCustomizationService.cpp b/src/web/WebCustomizationService.cpp index f9d21a95d..0f4ad78dd 100644 --- a/src/web/WebCustomizationService.cpp +++ b/src/web/WebCustomizationService.cpp @@ -212,7 +212,7 @@ void WebCustomizationService::device_entities(AsyncWebServerRequest * request) { JsonArray output = response->getRoot(); emsdevice->generate_values_web_customization(output); #else - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonArray output = doc.to(); emsdevice->generate_values_web_customization(output); #endif diff --git a/src/web/WebModulesService.cpp b/src/web/WebModulesService.cpp index 7638f8aa2..ce79750f8 100644 --- a/src/web/WebModulesService.cpp +++ b/src/web/WebModulesService.cpp @@ -50,7 +50,7 @@ void WebModulesService::loop() { // it adds data to an empty 'root' json object // and also calls when the Modules web page is refreshed/loaded void WebModules::read(WebModules & webModules, JsonObject root) { - JsonDocument doc_modules; + JsonDocument doc_modules(PSRAM_DOC); auto root_modules = doc_modules.to(); moduleLibrary.list(root_modules); // get list the external library modules, put in a json object diff --git a/src/web/WebSchedulerService.cpp b/src/web/WebSchedulerService.cpp index fc4e8d8ba..3eaf73715 100644 --- a/src/web/WebSchedulerService.cpp +++ b/src/web/WebSchedulerService.cpp @@ -266,7 +266,7 @@ void WebSchedulerService::publish(const bool force) { } } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); JsonObject output = doc.to(); bool ha_created = ha_configdone_; for (const ScheduleItem & scheduleItem : *scheduleItems_) { @@ -275,7 +275,7 @@ void WebSchedulerService::publish(const bool force) { // create HA config if (Mqtt::ha_enabled() && !ha_configdone_) { - JsonDocument config; + JsonDocument config(PSRAM_DOC); config["~"] = Mqtt::base(); char stat_t[50]; diff --git a/src/web/WebStatusService.cpp b/src/web/WebStatusService.cpp index 777580f19..c723a3059 100644 --- a/src/web/WebStatusService.cpp +++ b/src/web/WebStatusService.cpp @@ -433,7 +433,7 @@ bool WebStatusService::refresh_versions_cache() { return false; } - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); DeserializationError err = deserializeJson(doc, http.getStream()); http.end(); if (err) { @@ -539,7 +539,7 @@ bool WebStatusService::exportData(JsonObject root, std::string & type) { // action = getCustomSupport // reads any upload customSupport.json file and sends to to Help page to be shown as Guest bool WebStatusService::getCustomSupport(JsonObject root) { - JsonDocument doc; + JsonDocument doc(PSRAM_DOC); #if defined(EMSESP_STANDALONE) // dummy test data for "test api3"