diff --git a/interface/pnpm-lock.yaml b/interface/pnpm-lock.yaml index f682f2f83..c166e16bd 100644 --- a/interface/pnpm-lock.yaml +++ b/interface/pnpm-lock.yaml @@ -780,8 +780,8 @@ packages: peerDependencies: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 - acorn@8.16.0: - resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} + acorn@8.17.0: + resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} engines: {node: '>=0.4.0'} hasBin: true @@ -853,8 +853,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - baseline-browser-mapping@2.10.35: - resolution: {integrity: sha512-honAfLBde0HAFLdNyBEfuuENkF6zR+ozxqxa/2zJKHBe1qzLqyTSeRKpdPEHAP03rlDGyQOPnCSxnVpVqQo9Mg==} + baseline-browser-mapping@2.10.36: + resolution: {integrity: sha512-lVq/Df7LXlO79MVaaUHztSwWiG9oXoWHlgvNS51v8Dpd4+G4/VIy6qYePTw31nAVls33nUtnfezYeLkYAak9dg==} engines: {node: '>=6.0.0'} hasBin: true @@ -1185,8 +1185,8 @@ packages: duplexer3@0.1.5: resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==} - electron-to-chromium@1.5.371: - resolution: {integrity: sha512-e9htk9mAYL6AzmkEhSvVVw7IWGSBJ/Bqdn2eRyRLrj1g6sncN4WbFt5qnILYoCktktr45pyjIrOiRvBThQ808w==} + electron-to-chromium@1.5.372: + resolution: {integrity: sha512-M3yhbAlilnwqC8D21t28UCDGHyitShTmmLRU/H+b74P6Ski16Nb9HONYEaVpMj/pwC7BEo5B95FpjODLCWbtfA==} emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} @@ -3722,11 +3722,11 @@ snapshots: '@typescript-eslint/types': 8.61.0 eslint-visitor-keys: 5.0.1 - acorn-jsx@5.3.2(acorn@8.16.0): + acorn-jsx@5.3.2(acorn@8.17.0): dependencies: - acorn: 8.16.0 + acorn: 8.17.0 - acorn@8.16.0: {} + acorn@8.17.0: {} ajv@6.15.0: dependencies: @@ -3784,7 +3784,7 @@ snapshots: base64-js@1.5.1: {} - baseline-browser-mapping@2.10.35: {} + baseline-browser-mapping@2.10.36: {} bin-build@3.0.0: dependencies: @@ -3845,9 +3845,9 @@ snapshots: browserslist@4.28.2: dependencies: - baseline-browser-mapping: 2.10.35 + baseline-browser-mapping: 2.10.36 caniuse-lite: 1.0.30001799 - electron-to-chromium: 1.5.371 + electron-to-chromium: 1.5.372 node-releases: 2.0.47 update-browserslist-db: 1.2.3(browserslist@4.28.2) @@ -4202,7 +4202,7 @@ snapshots: duplexer3@0.1.5: {} - electron-to-chromium@1.5.371: {} + electron-to-chromium@1.5.372: {} emoji-regex@10.6.0: {} @@ -4368,8 +4368,8 @@ snapshots: espree@11.2.0: dependencies: - acorn: 8.16.0 - acorn-jsx: 5.3.2(acorn@8.16.0) + acorn: 8.17.0 + acorn-jsx: 5.3.2(acorn@8.17.0) eslint-visitor-keys: 5.0.1 esquery@1.7.0: @@ -5696,7 +5696,7 @@ snapshots: terser@5.48.0: dependencies: '@jridgewell/source-map': 0.3.11 - acorn: 8.16.0 + acorn: 8.17.0 commander: 2.20.3 source-map-support: 0.5.21 diff --git a/src/core/command.h b/src/core/command.h index 2644a29b4..f66111ecb 100644 --- a/src/core/command.h +++ b/src/core/command.h @@ -123,6 +123,11 @@ class Command { cmdfunctions_.reserve(num); } + // release any reserved-but-unused capacity once commands have settled + static void compact() { + cmdfunctions_.shrink_to_fit(); + } + static void show_all(uuid::console::Shell & shell); static Command::CmdFunction * find_command(const uint8_t device_type, const uint8_t device_id, const char * cmd, const uint8_t flag); static std::string tagged_cmd(const std::string & cmd, const uint8_t flag); diff --git a/src/core/emsdevice.cpp b/src/core/emsdevice.cpp index 41a83ecb8..45ce91ab9 100644 --- a/src/core/emsdevice.cpp +++ b/src/core/emsdevice.cpp @@ -559,6 +559,7 @@ void EMSdevice::show_mqtt_handlers(uuid::console::Shell & shell) const { // register a callback function for a specific telegram type void EMSdevice::register_telegram_type(const uint16_t telegram_type_id, const char * telegram_type_name, bool fetch, const process_function_p f, uint8_t length) { telegram_functions_.emplace_back(telegram_type_id, telegram_type_name, fetch, false, length, f); + EMSESP::mark_entities_changed(); } // add to device value library, also know now as a "device entity" @@ -675,6 +676,7 @@ void EMSdevice::add_device_value(int8_t tag, // to b // add the device entity devicevalues_.emplace_back( device_type_, tag, value_p, type, options, options_single, numeric_operator, short_name, fullname, custom_fullname, uom, has_cmd, min, max, state); + EMSESP::mark_entities_changed(); // add a new command if it has a function attached if (has_cmd) { @@ -1281,9 +1283,9 @@ void EMSdevice::setCustomizationEntity(const std::string & entity_id) { // set the custom name if it has one, or clear it if (has_custom_name) { - dv.custom_fullname = entity_id.substr(custom_name_pos + 1); + dv.set_custom_fullname(entity_id.substr(custom_name_pos + 1)); } else { - dv.custom_fullname = ""; + dv.set_custom_fullname(""); } auto min = dv.min; @@ -1322,11 +1324,11 @@ void EMSdevice::getCustomizationEntities(std::vector & entity_ids) break; } } - if (!is_set && (mask || !dv.custom_fullname.empty())) { - if (dv.custom_fullname.empty()) { + if (!is_set && (mask || dv.has_custom_fullname())) { + if (!dv.has_custom_fullname()) { entity_ids.push_back(Helpers::hextoa(mask, false) + entity_name); } else { - entity_ids.push_back(Helpers::hextoa(mask, false) + entity_name + "|" + dv.custom_fullname); + entity_ids.push_back(Helpers::hextoa(mask, false) + entity_name + "|" + dv.custom_fullname()); } } } diff --git a/src/core/emsdevice.h b/src/core/emsdevice.h index 93a0595f9..ec136b938 100644 --- a/src/core/emsdevice.h +++ b/src/core/emsdevice.h @@ -550,6 +550,12 @@ class EMSdevice { telegram_functions_.reserve(elements); } + // release any reserved-but-unused capacity once the entity/telegram set has settled + void compact() { + devicevalues_.shrink_to_fit(); + telegram_functions_.shrink_to_fit(); + } + #if defined(EMSESP_STANDALONE) struct TelegramFunctionDump { uint16_t type_id_; diff --git a/src/core/emsdevicevalue.cpp b/src/core/emsdevicevalue.cpp index 8db02d339..3f2eca495 100644 --- a/src/core/emsdevicevalue.cpp +++ b/src/core/emsdevicevalue.cpp @@ -53,8 +53,7 @@ DeviceValue::DeviceValue(uint8_t device_type, , uom(uom) , has_cmd(has_cmd) , min(min) - , max(max) - , custom_fullname(custom_fullname) { + , max(max) { // calculate #options in options list if (options_single) { options_size = 1; @@ -62,7 +61,12 @@ DeviceValue::DeviceValue(uint8_t device_type, options_size = Helpers::count_items(options); } - // set the min/max + // store the custom name on the heap, but only if one was actually provided + if (!custom_fullname.empty()) { + custom_fullname_ = std::make_unique(custom_fullname); + } + + // set the min/max (reads back the custom name set above) set_custom_minmax(); /* @@ -347,11 +351,12 @@ bool DeviceValue::get_min_max(int16_t & dv_set_min, uint32_t & dv_set_max) { // extract custom min from custom_fullname bool DeviceValue::get_custom_min(int16_t & val) { - auto min_pos = custom_fullname.find('>'); - bool has_min = (min_pos != std::string::npos); + const auto & cf = custom_fullname(); + auto min_pos = cf.find('>'); + bool has_min = (min_pos != std::string::npos); uint8_t fahrenheit = !EMSESP::system_.fahrenheit() ? 0 : (uom == DeviceValueUOM::DEGREES) ? 2 : (uom == DeviceValueUOM::DEGREES_R) ? 1 : 0; if (has_min) { - int32_t v = Helpers::atoint(custom_fullname.substr(min_pos + 1).c_str()); + int32_t v = Helpers::atoint(cf.substr(min_pos + 1).c_str()); if (fahrenheit) { v = (v - (32 * (fahrenheit - 1))) / 1.8; // reset to °C } @@ -365,11 +370,12 @@ bool DeviceValue::get_custom_min(int16_t & val) { // extract custom max from custom_fullname bool DeviceValue::get_custom_max(uint32_t & val) { - auto max_pos = custom_fullname.find('<'); - bool has_max = (max_pos != std::string::npos); + const auto & cf = custom_fullname(); + auto max_pos = cf.find('<'); + bool has_max = (max_pos != std::string::npos); uint8_t fahrenheit = !EMSESP::system_.fahrenheit() ? 0 : (uom == DeviceValueUOM::DEGREES) ? 2 : (uom == DeviceValueUOM::DEGREES_R) ? 1 : 0; if (has_max) { - int32_t v = Helpers::atoint(custom_fullname.substr(max_pos + 1).c_str()); + int32_t v = Helpers::atoint(cf.substr(max_pos + 1).c_str()); if (fahrenheit) { v = (v - (32 * (fahrenheit - 1))) / 1.8; // reset to °C } @@ -387,14 +393,32 @@ void DeviceValue::set_custom_minmax() { get_custom_max(max); } -std::string DeviceValue::get_custom_fullname() const { - auto min_pos = custom_fullname.find('>'); - auto max_pos = custom_fullname.find('<'); - auto minmax_pos = min_pos < max_pos ? min_pos : max_pos; - if (minmax_pos != std::string::npos) { - return custom_fullname.substr(0, minmax_pos); +// raw stored custom name (empty string if none was set) +const std::string & DeviceValue::custom_fullname() const { + static const std::string empty_string; + return custom_fullname_ ? *custom_fullname_ : empty_string; +} + +// set or clear the custom name, only allocating heap when there's actually a name +void DeviceValue::set_custom_fullname(const std::string & name) { + if (name.empty()) { + custom_fullname_.reset(); + } else if (custom_fullname_) { + *custom_fullname_ = name; + } else { + custom_fullname_ = std::make_unique(name); } - return custom_fullname; +} + +std::string DeviceValue::get_custom_fullname() const { + const auto & cf = custom_fullname(); + auto min_pos = cf.find('>'); + auto max_pos = cf.find('<'); + auto minmax_pos = min_pos < max_pos ? min_pos : max_pos; + if (minmax_pos != std::string::npos) { + return cf.substr(0, minmax_pos); + } + return cf; } // returns the translated fullname or the custom fullname (if provided) diff --git a/src/core/emsdevicevalue.h b/src/core/emsdevicevalue.h index 0eeaf4f62..25ca1875b 100644 --- a/src/core/emsdevicevalue.h +++ b/src/core/emsdevicevalue.h @@ -23,6 +23,8 @@ #include #include +#include + #include "helpers.h" // for conversions #include "default_settings.h" // for enum types @@ -188,8 +190,6 @@ class DeviceValue { // wider numeric range fields int16_t min; // min range uint32_t max; // max range - // largest member last (cold path: only read during customization save/load and web display) - std::string custom_fullname; // optional, from customization DeviceValue(uint8_t device_type, // EMSdevice::DeviceType int8_t tag, // DeviceValueTAG::* @@ -219,6 +219,13 @@ class DeviceValue { std::string get_fullname() const; static std::string get_name(const std::string & entity); + // raw stored custom name (including any >min custom_fullname_; }; }; // namespace emsesp diff --git a/src/core/emsesp.cpp b/src/core/emsesp.cpp index 582e94fdd..feb747586 100644 --- a/src/core/emsesp.cpp +++ b/src/core/emsesp.cpp @@ -48,6 +48,9 @@ uint16_t EMSESP::wait_validate_ = 0; bool EMSESP::wait_km_ = false; uint32_t EMSESP::last_fetch_ = 0; +uint32_t EMSESP::last_entity_change_ = 0; +bool EMSESP::entity_compaction_pending_ = false; + AsyncWebServer webServer(80); #if defined(EMSESP_STANDALONE) @@ -176,6 +179,37 @@ void EMSESP::clear_all_devices() { // emsdevices.clear(); // remove entries, but doesn't delete actual devices } +// called from EMSdevice/Command whenever an entity or telegram handler is registered. +// Devices reserve their value/telegram vectors generously (to avoid realloc storms while +// heating circuits etc. are discovered incrementally), so once registration settles we +// reclaim the unused capacity - see compact_entities_if_stable(). +void EMSESP::mark_entities_changed() { + last_entity_change_ = uuid::get_uptime(); + entity_compaction_pending_ = true; +} + +// once the entity/telegram set has been stable for ENTITY_COMPACT_DELAY, shrink the +// per-device and command vectors to their actual size. Re-arms automatically if a new +// device/circuit appears later (which just costs a single realloc). +void EMSESP::compact_entities_if_stable() { + if (!entity_compaction_pending_) { + return; // nothing to do (cheap early-out on the hot path) + } + if ((uuid::get_uptime() - last_entity_change_) < ENTITY_COMPACT_DELAY) { + return; // still settling + } + + for (const auto & emsdevice : emsdevices) { + if (emsdevice) { + emsdevice->compact(); + } + } + Command::compact(); + + entity_compaction_pending_ = false; + LOG_DEBUG("Reclaimed unused entity vector capacity"); +} + // return total number of devices excluding the Controller uint8_t EMSESP::count_devices() { if (emsdevices.empty()) { @@ -1860,6 +1894,7 @@ void EMSESP::loop() { webModulesService.loop(); // loop through the external library modules webSchedulerService.loop(); // scheduler timing logic; command execution is offloaded to WebCommandService's worker task scheduled_fetch_values(); // force a query on the EMS devices to fetch latest data at a set interval (1 min) + compact_entities_if_stable(); // reclaim over-reserved entity vector capacity once device discovery settles } // check for GPIO Errors - this is called once when booting if (EMSESP::system_.systemStatus() == SYSTEM_STATUS::SYSTEM_STATUS_INVALID_GPIO) { diff --git a/src/core/emsesp.h b/src/core/emsesp.h index e59db86ae..c75c0fa12 100644 --- a/src/core/emsesp.h +++ b/src/core/emsesp.h @@ -239,6 +239,10 @@ class EMSESP { static void scan_devices(); static void clear_all_devices(); + // called whenever a device entity or telegram handler is registered, so we can + // later reclaim the (deliberately generous) reserved vector capacity once stable + static void mark_entities_changed(); + static std::vector, AllocatorPSRAM>> emsdevices; // services static Mqtt mqtt_; @@ -275,6 +279,9 @@ class EMSESP { static void publish_response(const std::shared_ptr & telegram); static void publish_all_loop(); + // one-time compaction of per-device/command vectors once registration has been stable + static void compact_entities_if_stable(); + void shell_prompt(); void start_serial_console(); @@ -303,6 +310,11 @@ class EMSESP { static bool wait_km_; static uint32_t last_fetch_; + // entity/telegram registration tracking, used to trigger a one-time vector compaction + static constexpr uint32_t ENTITY_COMPACT_DELAY = 60000; // ms of stability before compacting + static uint32_t last_entity_change_; // uptime (ms) of last registration + static bool entity_compaction_pending_; // true while a compaction is owed + // UUID stuff static constexpr auto & serial_console_ = Serial; static constexpr unsigned long SERIAL_CONSOLE_BAUD_RATE = 115200; diff --git a/src/core/system.cpp b/src/core/system.cpp index 208e657c5..4a01ee7e7 100644 --- a/src/core/system.cpp +++ b/src/core/system.cpp @@ -32,6 +32,7 @@ #include #include "firmwareVersion.h" +#include "shuntingYard.h" // for compute() used by the message and sendmail commands #if defined(EMSESP_TEST) #include "../test/test.h" @@ -194,15 +195,11 @@ bool System::command_sendmail(const char * value, const int8_t id) { // msg.headers.addCustom("Importance", PRIORITY); // msg.headers.addCustom("X-MSMail-Priority", PRIORITY); // msg.headers.addCustom("X-Priority", PRIORITY_NUM); - EMSESP::webSchedulerService.computed_value.clear(); - EMSESP::webSchedulerService.raw_value = body.c_str(); - for (uint16_t wait = 0; wait < 2000 && !EMSESP::webSchedulerService.raw_value.empty(); wait++) { - delay(1); - } - if (!EMSESP::webSchedulerService.computed_value.empty()) { - body = EMSESP::webSchedulerService.computed_value.c_str(); - EMSESP::webSchedulerService.computed_value.clear(); - EMSESP::webSchedulerService.computed_value.shrink_to_fit(); // free allocated memory + // run the body through the Shunting Yard calculator (entity substitution, expressions, optional {url} fetch) + // keep the original body if the calculator returns nothing + std::string computed_body = compute(body.c_str()); + if (!computed_body.empty()) { + body = computed_body.c_str(); } msg.text.body(body); @@ -344,22 +341,16 @@ bool System::command_message(const char * value, const int8_t id, JsonObject out return false; // must have a string value } - EMSESP::webSchedulerService.computed_value.clear(); - EMSESP::webSchedulerService.raw_value = value; - for (uint16_t wait = 0; wait < 2000 && !EMSESP::webSchedulerService.raw_value.empty(); wait++) { - delay(1); - } - - if (EMSESP::webSchedulerService.computed_value.empty()) { + // process the message via the Shunting Yard calculator (entity substitution, expressions, optional {url} fetch) + std::string computed_value = compute(value); + if (computed_value.empty()) { LOG_WARNING("Message result is empty"); return false; } - LOG_INFO("Message: %s", EMSESP::webSchedulerService.computed_value.c_str()); // send to log - Mqtt::queue_publish(F_(message), EMSESP::webSchedulerService.computed_value); // send to MQTT if enabled - output["api_data"] = EMSESP::webSchedulerService.computed_value; // send to API - EMSESP::webSchedulerService.computed_value.clear(); - EMSESP::webSchedulerService.computed_value.shrink_to_fit(); + LOG_INFO("Message: %s", computed_value.c_str()); // send to log + Mqtt::queue_publish(F_(message), computed_value); // send to MQTT if enabled + output["api_data"] = computed_value; // send to API return true; } @@ -1285,6 +1276,9 @@ bool System::check_restore() { // continue processing the rest of the sections saveSettings(EMSESP_SETTINGS_FILE, section); } + if (section_type == "commands") { + saveSettings(EMSESP_COMMANDS_FILE, section); + } if (section_type == "schedule") { saveSettings(EMSESP_SCHEDULER_FILE, section); } diff --git a/src/web/WebCommandService.cpp b/src/web/WebCommandService.cpp index 8da74bb35..6f073d2cf 100644 --- a/src/web/WebCommandService.cpp +++ b/src/web/WebCommandService.cpp @@ -141,14 +141,12 @@ bool WebCommandService::dispatchCommand(const char * name, const char * value) { if (isUrlCommand(ci->cmd.c_str())) { return queueCommand(name, value); } - // system/message defers evaluation of its value (via the scheduler's raw_value), - // so executing it never blocks - keep it synchronous even if the value has a {url} - if (Helpers::toLower(ci->cmd.c_str()) != "system/message") { - // the effective value is the override if given, else the command's stored default - const std::string effective_value = value ? value : std::string(ci->value.c_str()); - if (valueContainsUrl(effective_value)) { - return queueCommand(name, value); - } + // internal command whose value embeds a {url} fetch (e.g. system/message) - the value is + // resolved by compute() at execution time and would block, so offload it to the worker task + // the effective value is the override if given, else the command's stored default + const std::string effective_value = value ? value : std::string(ci->value.c_str()); + if (valueContainsUrl(effective_value)) { + return queueCommand(name, value); } } } @@ -240,8 +238,8 @@ bool WebCommandService::executeCommand(const char * name, const std::string & co // run the value through the shunting-yard calculator so expressions like "custom/heatcnt + 1" // are resolved (entity references replaced by their values, then computed). Plain values pass // through unchanged. Applies to both URL and internal commands, like the old scheduler code - // which computed the value before executing. system/message evaluates its own argument later - // (deferred via the scheduler's raw_value), so pre-computing it would run it twice - pass raw. + // which computed the value before executing. system/message runs the shunting-yard on its own + // argument, so pre-computing it here would run it twice - pass it through raw. std::string computed_data = data; if (!data.empty() && cmd != "system/message") { computed_data = compute(data); diff --git a/src/web/WebCustomizationService.cpp b/src/web/WebCustomizationService.cpp index 0f4ad78dd..16086fa49 100644 --- a/src/web/WebCustomizationService.cpp +++ b/src/web/WebCustomizationService.cpp @@ -491,8 +491,8 @@ void WebCustomizationService::load_test_data() { // find the device value and set the mask and custom name to match the above fake data for (auto & dv : emsdevice->devicevalues_) { if (strcmp(dv.short_name, "heatingactive") == 0) { - dv.state = DeviceValueState::DV_FAVORITE; // set as favorite - dv.custom_fullname = "is my heating on?"; + dv.state = DeviceValueState::DV_FAVORITE; // set as favorite + dv.set_custom_fullname("is my heating on?"); } else if (strcmp(dv.short_name, "tapwateractive") == 0) { dv.state = DeviceValueState::DV_FAVORITE; // set as favorite } else if (strcmp(dv.short_name, "selflowtemp") == 0) { diff --git a/src/web/WebSchedulerService.cpp b/src/web/WebSchedulerService.cpp index 79c3fe097..d788e0768 100644 --- a/src/web/WebSchedulerService.cpp +++ b/src/web/WebSchedulerService.cpp @@ -367,11 +367,6 @@ void WebSchedulerService::loop() { static uint32_t last_uptime_min = 0; static uint32_t last_uptime_sec = 0; - if (!raw_value.empty()) { - computed_value = compute(raw_value); - raw_value.clear(); - } - if (scheduleItems_->empty()) { return; } diff --git a/src/web/WebSchedulerService.h b/src/web/WebSchedulerService.h index db21f38cf..299ddd2fa 100644 --- a/src/web/WebSchedulerService.h +++ b/src/web/WebSchedulerService.h @@ -75,9 +75,6 @@ class WebSchedulerService : public StatefulService { std::string get_metrics_prometheus(); - std::string raw_value; - std::string computed_value; - #if defined(EMSESP_TEST) void load_test_data(); #endif