Merge pull request #3120 from proddy/commands

Commands - fixes

interface/pnpm-lock.yaml

@@ -780,8 +780,8 @@ packages:
     peerDependencies:
       acorn: ^6.0.0 || ^7.0.0 || ^8.0.0
 
-  acorn@8.16.0:
+  acorn@8.17.0:
-    resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==}
+    resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==}
     engines: {node: '>=0.4.0'}
     hasBin: true
 
@@ -853,8 +853,8 @@ packages:
   base64-js@1.5.1:
     resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
 
-  baseline-browser-mapping@2.10.35:
+  baseline-browser-mapping@2.10.36:
-    resolution: {integrity: sha512-honAfLBde0HAFLdNyBEfuuENkF6zR+ozxqxa/2zJKHBe1qzLqyTSeRKpdPEHAP03rlDGyQOPnCSxnVpVqQo9Mg==}
+    resolution: {integrity: sha512-lVq/Df7LXlO79MVaaUHztSwWiG9oXoWHlgvNS51v8Dpd4+G4/VIy6qYePTw31nAVls33nUtnfezYeLkYAak9dg==}
     engines: {node: '>=6.0.0'}
     hasBin: true
 
@@ -1185,8 +1185,8 @@ packages:
   duplexer3@0.1.5:
     resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==}
 
-  electron-to-chromium@1.5.371:
+  electron-to-chromium@1.5.372:
-    resolution: {integrity: sha512-e9htk9mAYL6AzmkEhSvVVw7IWGSBJ/Bqdn2eRyRLrj1g6sncN4WbFt5qnILYoCktktr45pyjIrOiRvBThQ808w==}
+    resolution: {integrity: sha512-M3yhbAlilnwqC8D21t28UCDGHyitShTmmLRU/H+b74P6Ski16Nb9HONYEaVpMj/pwC7BEo5B95FpjODLCWbtfA==}
 
   emoji-regex@10.6.0:
     resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==}
@@ -3722,11 +3722,11 @@ snapshots:
       '@typescript-eslint/types': 8.61.0
       eslint-visitor-keys: 5.0.1
 
-  acorn-jsx@5.3.2(acorn@8.16.0):
+  acorn-jsx@5.3.2(acorn@8.17.0):
     dependencies:
-      acorn: 8.16.0
+      acorn: 8.17.0
 
-  acorn@8.16.0: {}
+  acorn@8.17.0: {}
 
   ajv@6.15.0:
     dependencies:
@@ -3784,7 +3784,7 @@ snapshots:
 
   base64-js@1.5.1: {}
 
-  baseline-browser-mapping@2.10.35: {}
+  baseline-browser-mapping@2.10.36: {}
 
   bin-build@3.0.0:
     dependencies:
@@ -3845,9 +3845,9 @@ snapshots:
 
   browserslist@4.28.2:
     dependencies:
-      baseline-browser-mapping: 2.10.35
+      baseline-browser-mapping: 2.10.36
       caniuse-lite: 1.0.30001799
-      electron-to-chromium: 1.5.371
+      electron-to-chromium: 1.5.372
       node-releases: 2.0.47
       update-browserslist-db: 1.2.3(browserslist@4.28.2)
 
@@ -4202,7 +4202,7 @@ snapshots:
 
   duplexer3@0.1.5: {}
 
-  electron-to-chromium@1.5.371: {}
+  electron-to-chromium@1.5.372: {}
 
   emoji-regex@10.6.0: {}
 
@@ -4368,8 +4368,8 @@ snapshots:
 
   espree@11.2.0:
     dependencies:
-      acorn: 8.16.0
+      acorn: 8.17.0
-      acorn-jsx: 5.3.2(acorn@8.16.0)
+      acorn-jsx: 5.3.2(acorn@8.17.0)
       eslint-visitor-keys: 5.0.1
 
   esquery@1.7.0:
@@ -5696,7 +5696,7 @@ snapshots:
   terser@5.48.0:
     dependencies:
       '@jridgewell/source-map': 0.3.11
-      acorn: 8.16.0
+      acorn: 8.17.0
       commander: 2.20.3
       source-map-support: 0.5.21
 

src/core/command.h

@@ -123,6 +123,11 @@ class Command {
         cmdfunctions_.reserve(num);
     }
 
+    // release any reserved-but-unused capacity once commands have settled
+    static void compact() {
+        cmdfunctions_.shrink_to_fit();
+    }
+
     static void                   show_all(uuid::console::Shell & shell);
     static Command::CmdFunction * find_command(const uint8_t device_type, const uint8_t device_id, const char * cmd, const uint8_t flag);
     static std::string            tagged_cmd(const std::string & cmd, const uint8_t flag);

src/core/emsdevice.cpp

@@ -559,6 +559,7 @@ void EMSdevice::show_mqtt_handlers(uuid::console::Shell & shell) const {
 // register a callback function for a specific telegram type
 void EMSdevice::register_telegram_type(const uint16_t telegram_type_id, const char * telegram_type_name, bool fetch, const process_function_p f, uint8_t length) {
     telegram_functions_.emplace_back(telegram_type_id, telegram_type_name, fetch, false, length, f);
+    EMSESP::mark_entities_changed();
 }
 
 // add to device value library, also know now as a "device entity"
@@ -675,6 +676,7 @@ void EMSdevice::add_device_value(int8_t                tag,              // to b
     // add the device entity
     devicevalues_.emplace_back(
         device_type_, tag, value_p, type, options, options_single, numeric_operator, short_name, fullname, custom_fullname, uom, has_cmd, min, max, state);
+    EMSESP::mark_entities_changed();
 
     // add a new command if it has a function attached
     if (has_cmd) {
@@ -1281,9 +1283,9 @@ void EMSdevice::setCustomizationEntity(const std::string & entity_id) {
 
             // set the custom name if it has one, or clear it
             if (has_custom_name) {
-                dv.custom_fullname = entity_id.substr(custom_name_pos + 1);
+                dv.set_custom_fullname(entity_id.substr(custom_name_pos + 1));
             } else {
-                dv.custom_fullname = "";
+                dv.set_custom_fullname("");
             }
 
             auto min = dv.min;
@@ -1322,11 +1324,11 @@ void EMSdevice::getCustomizationEntities(std::vector<std::string> & entity_ids)
                 break;
             }
         }
-        if (!is_set && (mask || !dv.custom_fullname.empty())) {
+        if (!is_set && (mask || dv.has_custom_fullname())) {
-            if (dv.custom_fullname.empty()) {
+            if (!dv.has_custom_fullname()) {
                 entity_ids.push_back(Helpers::hextoa(mask, false) + entity_name);
             } else {
-                entity_ids.push_back(Helpers::hextoa(mask, false) + entity_name + "|" + dv.custom_fullname);
+                entity_ids.push_back(Helpers::hextoa(mask, false) + entity_name + "|" + dv.custom_fullname());
             }
         }
     }

src/core/emsdevice.h

@@ -550,6 +550,12 @@ class EMSdevice {
         telegram_functions_.reserve(elements);
     }
 
+    // release any reserved-but-unused capacity once the entity/telegram set has settled
+    void compact() {
+        devicevalues_.shrink_to_fit();
+        telegram_functions_.shrink_to_fit();
+    }
+
 #if defined(EMSESP_STANDALONE)
     struct TelegramFunctionDump {
         uint16_t     type_id_;

src/core/emsdevicevalue.cpp

@@ -53,8 +53,7 @@ DeviceValue::DeviceValue(uint8_t               device_type,
     , uom(uom)
     , has_cmd(has_cmd)
     , min(min)
-    , max(max)
+    , max(max) {
-    , custom_fullname(custom_fullname) {
     // calculate #options in options list
     if (options_single) {
         options_size = 1;
@@ -62,7 +61,12 @@ DeviceValue::DeviceValue(uint8_t               device_type,
         options_size = Helpers::count_items(options);
     }
 
-    // set the min/max
+    // store the custom name on the heap, but only if one was actually provided
+    if (!custom_fullname.empty()) {
+        custom_fullname_ = std::make_unique<std::string>(custom_fullname);
+    }
+
+    // set the min/max (reads back the custom name set above)
     set_custom_minmax();
 
     /*
@@ -347,11 +351,12 @@ bool DeviceValue::get_min_max(int16_t & dv_set_min, uint32_t & dv_set_max) {
 
 // extract custom min from custom_fullname
 bool DeviceValue::get_custom_min(int16_t & val) {
-    auto    min_pos    = custom_fullname.find('>');
+    const auto & cf        = custom_fullname();
+    auto         min_pos   = cf.find('>');
     bool         has_min   = (min_pos != std::string::npos);
     uint8_t fahrenheit = !EMSESP::system_.fahrenheit() ? 0 : (uom == DeviceValueUOM::DEGREES) ? 2 : (uom == DeviceValueUOM::DEGREES_R) ? 1 : 0;
     if (has_min) {
-        int32_t v = Helpers::atoint(custom_fullname.substr(min_pos + 1).c_str());
+        int32_t v = Helpers::atoint(cf.substr(min_pos + 1).c_str());
         if (fahrenheit) {
             v = (v - (32 * (fahrenheit - 1))) / 1.8; // reset to °C
         }
@@ -365,11 +370,12 @@ bool DeviceValue::get_custom_min(int16_t & val) {
 
 // extract custom max from custom_fullname
 bool DeviceValue::get_custom_max(uint32_t & val) {
-    auto    max_pos    = custom_fullname.find('<');
+    const auto & cf        = custom_fullname();
+    auto         max_pos   = cf.find('<');
     bool         has_max   = (max_pos != std::string::npos);
     uint8_t fahrenheit = !EMSESP::system_.fahrenheit() ? 0 : (uom == DeviceValueUOM::DEGREES) ? 2 : (uom == DeviceValueUOM::DEGREES_R) ? 1 : 0;
     if (has_max) {
-        int32_t v = Helpers::atoint(custom_fullname.substr(max_pos + 1).c_str());
+        int32_t v = Helpers::atoint(cf.substr(max_pos + 1).c_str());
         if (fahrenheit) {
             v = (v - (32 * (fahrenheit - 1))) / 1.8; // reset to °C
         }
@@ -387,14 +393,32 @@ void DeviceValue::set_custom_minmax() {
     get_custom_max(max);
 }
 
+// raw stored custom name (empty string if none was set)
+const std::string & DeviceValue::custom_fullname() const {
+    static const std::string empty_string;
+    return custom_fullname_ ? *custom_fullname_ : empty_string;
+}
+
+// set or clear the custom name, only allocating heap when there's actually a name
+void DeviceValue::set_custom_fullname(const std::string & name) {
+    if (name.empty()) {
+        custom_fullname_.reset();
+    } else if (custom_fullname_) {
+        *custom_fullname_ = name;
+    } else {
+        custom_fullname_ = std::make_unique<std::string>(name);
+    }
+}
+
 std::string DeviceValue::get_custom_fullname() const {
-    auto min_pos    = custom_fullname.find('>');
+    const auto & cf         = custom_fullname();
-    auto max_pos    = custom_fullname.find('<');
+    auto         min_pos    = cf.find('>');
+    auto         max_pos    = cf.find('<');
     auto         minmax_pos = min_pos < max_pos ? min_pos : max_pos;
     if (minmax_pos != std::string::npos) {
-        return custom_fullname.substr(0, minmax_pos);
+        return cf.substr(0, minmax_pos);
     }
-    return custom_fullname;
+    return cf;
 }
 
 // returns the translated fullname or the custom fullname (if provided)

src/core/emsdevicevalue.h

@@ -23,6 +23,8 @@
 #include <Arduino.h>
 #include <ArduinoJson.h>
 
+#include <memory>
+
 #include "helpers.h"          // for conversions
 #include "default_settings.h" // for enum types
 
@@ -188,8 +190,6 @@ class DeviceValue {
     // wider numeric range fields
     int16_t  min; // min range
     uint32_t max; // max range
-    // largest member last (cold path: only read during customization save/load and web display)
-    std::string custom_fullname; // optional, from customization
 
     DeviceValue(uint8_t               device_type,    // EMSdevice::DeviceType
                 int8_t                tag,            // DeviceValueTAG::*
@@ -219,6 +219,13 @@ class DeviceValue {
     std::string        get_fullname() const;
     static std::string get_name(const std::string & entity);
 
+    // raw stored custom name (including any >min<max suffix), empty if none. Stored on heap only when set.
+    const std::string & custom_fullname() const;
+    void                set_custom_fullname(const std::string & name);
+    bool                has_custom_fullname() const {
+        return (bool)custom_fullname_;
+    }
+
     // dv state flags
     void add_state(uint8_t s) {
         state |= s;
@@ -237,6 +244,11 @@ class DeviceValue {
     static const char * const * DeviceValueTAG_s[];
     static const char * const   DeviceValueTAG_mqtt[];
     static uint8_t              NUM_TAGS; // # tags
+
+  private:
+    // optional custom name from customization. Allocated on heap only when actually set,
+    // so unnamed entities (the vast majority) don't pay for an inline std::string.
+    std::unique_ptr<std::string> custom_fullname_;
 };
 
 }; // namespace emsesp

src/core/emsesp.cpp

@@ -48,6 +48,9 @@ uint16_t          EMSESP::wait_validate_    = 0;
 bool              EMSESP::wait_km_          = false;
 uint32_t          EMSESP::last_fetch_       = 0;
 
+uint32_t EMSESP::last_entity_change_        = 0;
+bool     EMSESP::entity_compaction_pending_ = false;
+
 AsyncWebServer webServer(80);
 
 #if defined(EMSESP_STANDALONE)
@@ -176,6 +179,37 @@ void EMSESP::clear_all_devices() {
     // emsdevices.clear(); // remove entries, but doesn't delete actual devices
 }
 
+// called from EMSdevice/Command whenever an entity or telegram handler is registered.
+// Devices reserve their value/telegram vectors generously (to avoid realloc storms while
+// heating circuits etc. are discovered incrementally), so once registration settles we
+// reclaim the unused capacity - see compact_entities_if_stable().
+void EMSESP::mark_entities_changed() {
+    last_entity_change_        = uuid::get_uptime();
+    entity_compaction_pending_ = true;
+}
+
+// once the entity/telegram set has been stable for ENTITY_COMPACT_DELAY, shrink the
+// per-device and command vectors to their actual size. Re-arms automatically if a new
+// device/circuit appears later (which just costs a single realloc).
+void EMSESP::compact_entities_if_stable() {
+    if (!entity_compaction_pending_) {
+        return; // nothing to do (cheap early-out on the hot path)
+    }
+    if ((uuid::get_uptime() - last_entity_change_) < ENTITY_COMPACT_DELAY) {
+        return; // still settling
+    }
+
+    for (const auto & emsdevice : emsdevices) {
+        if (emsdevice) {
+            emsdevice->compact();
+        }
+    }
+    Command::compact();
+
+    entity_compaction_pending_ = false;
+    LOG_DEBUG("Reclaimed unused entity vector capacity");
+}
+
 // return total number of devices excluding the Controller
 uint8_t EMSESP::count_devices() {
     if (emsdevices.empty()) {
@@ -1860,6 +1894,7 @@ void EMSESP::loop() {
         webModulesService.loop();   // loop through the external library modules
         webSchedulerService.loop(); // scheduler timing logic; command execution is offloaded to WebCommandService's worker task
         scheduled_fetch_values(); // force a query on the EMS devices to fetch latest data at a set interval (1 min)
+        compact_entities_if_stable(); // reclaim over-reserved entity vector capacity once device discovery settles
     }
     // check for GPIO Errors - this is called once when booting
     if (EMSESP::system_.systemStatus() == SYSTEM_STATUS::SYSTEM_STATUS_INVALID_GPIO) {

src/core/emsesp.h

@@ -239,6 +239,10 @@ class EMSESP {
     static void scan_devices();
     static void clear_all_devices();
 
+    // called whenever a device entity or telegram handler is registered, so we can
+    // later reclaim the (deliberately generous) reserved vector capacity once stable
+    static void mark_entities_changed();
+
     static std::vector<std::unique_ptr<EMSdevice>, AllocatorPSRAM<std::unique_ptr<EMSdevice>>> emsdevices;
     // services
     static Mqtt              mqtt_;
@@ -275,6 +279,9 @@ class EMSESP {
     static void        publish_response(const std::shared_ptr<const Telegram> & telegram);
     static void        publish_all_loop();
 
+    // one-time compaction of per-device/command vectors once registration has been stable
+    static void compact_entities_if_stable();
+
     void shell_prompt();
     void start_serial_console();
 
@@ -303,6 +310,11 @@ class EMSESP {
     static bool     wait_km_;
     static uint32_t last_fetch_;
 
+    // entity/telegram registration tracking, used to trigger a one-time vector compaction
+    static constexpr uint32_t ENTITY_COMPACT_DELAY = 60000; // ms of stability before compacting
+    static uint32_t           last_entity_change_;          // uptime (ms) of last registration
+    static bool               entity_compaction_pending_;   // true while a compaction is owed
+
     // UUID stuff
     static constexpr auto &        serial_console_          = Serial;
     static constexpr unsigned long SERIAL_CONSOLE_BAUD_RATE = 115200;

src/core/system.cpp

@@ -32,6 +32,7 @@
 #include <map>
 
 #include "firmwareVersion.h"
+#include "shuntingYard.h" // for compute() used by the message and sendmail commands
 
 #if defined(EMSESP_TEST)
 #include "../test/test.h"
@@ -194,15 +195,11 @@ bool System::command_sendmail(const char * value, const int8_t id) {
     // msg.headers.addCustom("Importance", PRIORITY);
     // msg.headers.addCustom("X-MSMail-Priority", PRIORITY);
     // msg.headers.addCustom("X-Priority", PRIORITY_NUM);
-    EMSESP::webSchedulerService.computed_value.clear();
+    // run the body through the Shunting Yard calculator (entity substitution, expressions, optional {url} fetch)
-    EMSESP::webSchedulerService.raw_value = body.c_str();
+    // keep the original body if the calculator returns nothing
-    for (uint16_t wait = 0; wait < 2000 && !EMSESP::webSchedulerService.raw_value.empty(); wait++) {
+    std::string computed_body = compute(body.c_str());
-        delay(1);
+    if (!computed_body.empty()) {
-    }
+        body = computed_body.c_str();
-    if (!EMSESP::webSchedulerService.computed_value.empty()) {
-        body = EMSESP::webSchedulerService.computed_value.c_str();
-        EMSESP::webSchedulerService.computed_value.clear();
-        EMSESP::webSchedulerService.computed_value.shrink_to_fit(); // free allocated memory
     }
     msg.text.body(body);
 
@@ -344,22 +341,16 @@ bool System::command_message(const char * value, const int8_t id, JsonObject out
         return false; // must have a string value
     }
 
-    EMSESP::webSchedulerService.computed_value.clear();
+    // process the message via the Shunting Yard calculator (entity substitution, expressions, optional {url} fetch)
-    EMSESP::webSchedulerService.raw_value = value;
+    std::string computed_value = compute(value);
-    for (uint16_t wait = 0; wait < 2000 && !EMSESP::webSchedulerService.raw_value.empty(); wait++) {
+    if (computed_value.empty()) {
-        delay(1);
-    }
-
-    if (EMSESP::webSchedulerService.computed_value.empty()) {
         LOG_WARNING("Message result is empty");
         return false;
     }
 
-    LOG_INFO("Message: %s", EMSESP::webSchedulerService.computed_value.c_str());  // send to log
+    LOG_INFO("Message: %s", computed_value.c_str());  // send to log
-    Mqtt::queue_publish(F_(message), EMSESP::webSchedulerService.computed_value); // send to MQTT if enabled
+    Mqtt::queue_publish(F_(message), computed_value); // send to MQTT if enabled
-    output["api_data"] = EMSESP::webSchedulerService.computed_value;              // send to API
+    output["api_data"] = computed_value;              // send to API
-    EMSESP::webSchedulerService.computed_value.clear();
-    EMSESP::webSchedulerService.computed_value.shrink_to_fit();
     return true;
 }
 
@@ -1285,6 +1276,9 @@ bool System::check_restore() {
                         // continue processing the rest of the sections
                         saveSettings(EMSESP_SETTINGS_FILE, section);
                     }
+                    if (section_type == "commands") {
+                        saveSettings(EMSESP_COMMANDS_FILE, section);
+                    }
                     if (section_type == "schedule") {
                         saveSettings(EMSESP_SCHEDULER_FILE, section);
                     }

src/web/WebCommandService.cpp

@@ -141,9 +141,8 @@ bool WebCommandService::dispatchCommand(const char * name, const char * value) {
             if (isUrlCommand(ci->cmd.c_str())) {
                 return queueCommand(name, value);
             }
-            // system/message defers evaluation of its value (via the scheduler's raw_value),
+            // internal command whose value embeds a {url} fetch (e.g. system/message) - the value is
-            // so executing it never blocks - keep it synchronous even if the value has a {url}
+            // resolved by compute() at execution time and would block, so offload it to the worker task
-            if (Helpers::toLower(ci->cmd.c_str()) != "system/message") {
             // the effective value is the override if given, else the command's stored default
             const std::string effective_value = value ? value : std::string(ci->value.c_str());
             if (valueContainsUrl(effective_value)) {
@@ -151,7 +150,6 @@ bool WebCommandService::dispatchCommand(const char * name, const char * value) {
             }
         }
     }
-    }
 #endif
     return executeCommand(name, value);
 }
@@ -240,8 +238,8 @@ bool WebCommandService::executeCommand(const char * name, const std::string & co
     // run the value through the shunting-yard calculator so expressions like "custom/heatcnt + 1"
     // are resolved (entity references replaced by their values, then computed). Plain values pass
     // through unchanged. Applies to both URL and internal commands, like the old scheduler code
-    // which computed the value before executing. system/message evaluates its own argument later
+    // which computed the value before executing. system/message runs the shunting-yard on its own
-    // (deferred via the scheduler's raw_value), so pre-computing it would run it twice - pass raw.
+    // argument, so pre-computing it here would run it twice - pass it through raw.
     std::string computed_data = data;
     if (!data.empty() && cmd != "system/message") {
         computed_data = compute(data);

src/web/WebCustomizationService.cpp

@@ -492,7 +492,7 @@ void WebCustomizationService::load_test_data() {
                 for (auto & dv : emsdevice->devicevalues_) {
                     if (strcmp(dv.short_name, "heatingactive") == 0) {
                         dv.state = DeviceValueState::DV_FAVORITE; // set as favorite
-                        dv.custom_fullname = "is my heating on?";
+                        dv.set_custom_fullname("is my heating on?");
                     } else if (strcmp(dv.short_name, "tapwateractive") == 0) {
                         dv.state = DeviceValueState::DV_FAVORITE; // set as favorite
                     } else if (strcmp(dv.short_name, "selflowtemp") == 0) {

src/web/WebSchedulerService.cpp

@@ -367,11 +367,6 @@ void WebSchedulerService::loop() {
     static uint32_t last_uptime_min = 0;
     static uint32_t last_uptime_sec = 0;
 
-    if (!raw_value.empty()) {
-        computed_value = compute(raw_value);
-        raw_value.clear();
-    }
-
     if (scheduleItems_->empty()) {
         return;
     }

src/web/WebSchedulerService.h

@@ -75,9 +75,6 @@ class WebSchedulerService : public StatefulService<WebScheduler> {
 
     std::string get_metrics_prometheus();
 
-    std::string raw_value;
-    std::string computed_value;
-
 #if defined(EMSESP_TEST)
     void load_test_data();
 #endif