Merge pull request #2871 from proddy/dev

overcome strange chars in header files

scripts/run_executable.py

@@ -30,6 +30,8 @@ def run_with_streaming_input(program_path, test_command, output_file=None):
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
+            encoding='utf-8',
+            errors='replace',  # Replace invalid UTF-8 bytes instead of crashing
             bufsize=1  # Line buffered
         )
         
@@ -70,7 +72,7 @@ def run_with_streaming_input(program_path, test_command, output_file=None):
                 os.remove(output_file)
             
             # Export CSV output to file
-            with open(output_file, 'w') as f:
+            with open(output_file, 'w', encoding='utf-8', errors='replace') as f:
                 f.writelines(csv_output)
             print(f"CSV file created: {output_file} ({os.path.getsize(output_file)} bytes)")
 

scripts/update_modbus_registers.py

@@ -1,7 +1,10 @@
 #
 # Update modbus parameters from entity definitions.
 # This script generates c++ code for the modbus parameter definitions.
-# Called by /scripts/generate_csv_and_headers.sh
+#
+# Called by /scripts/generate_csv_and_headers.sh and pio build_modbus target.
+# can be called manually with:
+#   cat ./docs/dump_entities.csv | python3 ./scripts/update_modbus_registers.py > ./src/core/modbus_entity_parameters.hpp
 
 import fileinput
 import csv
@@ -159,23 +162,22 @@ cpp_entry_template = Template(
 listNames = {}
 transre = re.compile(r'^MAKE_TRANSLATION\(([^,\s]+)\s*,\s*\"([^\"]+)\"')
 try:
-    with open('./src/core/locale_translations.h', 'r') as transf:
+    with open('./src/core/locale_translations.h', 'r', encoding='utf-8', errors='replace') as transf:
         for line in transf:
             m = transre.match(line)
             if m is not None:
                 listNames[m.group(2)] = m.group(1)
 except FileNotFoundError:
     # Handle case where file doesn't exist
-    pass
+    raise Exception('Error! locale_translations.h not found')
 
 entities = []
 
-with fileinput.input() as f_input:
+with fileinput.input(encoding='utf-8', errors='replace') as f_input:
     entities_reader = csv.reader(f_input, delimiter=',', quotechar='"')
     headers = next(entities_reader)
 
     for row in entities_reader:
-        # Use dict comprehension for better performance
         entity = {headers[i]: val for i, val in enumerate(row)}
         entities.append(entity)
 
@@ -208,7 +210,7 @@ for entity in entities:
             (-string_sizes[entity_dev_name] // 2)  # divide and round up
 
     if int(entity["modbus count"]) <= 0:
-        raise Exception('Entity "' + entity_dev_name + ' (' + entity_shortname + ')' +
+        raise Exception('Error! Entity "' + entity_dev_name + ' (' + entity_shortname + ')' +
                         '" does not have a size - string sizes need to be added manually to update_modbus_registers.py/string_sizes[]')
 
     #    if entity["modbus count"] == "0":
@@ -267,10 +269,16 @@ for device_type_name in device_type_names:
                 sorted_entities = sorted(
                     entities.items(), key=lambda x: int(x[1]["modbus offset"]))
                 for entity_name, modbus_info in sorted_entities:
+                    # Strip device type prefix (e.g., "dhw.nrg" -> "nrg") for translation lookup
+                    lookup_name = entity_name.split('.')[-1] if '.' in entity_name else entity_name
+                    
+                    if lookup_name not in listNames:
+                        raise KeyError(f"Error! Translation not found for '{lookup_name}' (entity: '{entity_name}'). Please add it to locale_translations.h")
+                    
                     params = {
                         'devtype': "dt::" + device_type_name,
                         "tagtype": tag_to_tagtype[int(tag)],
-                        "shortname": 'FL_(' + listNames[entity_name] + ")",
+                        "shortname": 'FL_(' + listNames[lookup_name] + ")",
                         "entity_name": entity_name,
                         'registeroffset': modbus_info["modbus offset"],
                         'registercount': modbus_info["modbus count"]

src/ESP32React/UploadFileService.cpp

@@ -82,7 +82,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
             }
 #endif
             // it's firmware - initialize the ArduinoOTA updater
-            emsesp::EMSESP::logger().info("Uploading firmware file %s (size: %d bytes)", filename.c_str(), filesize);
+            emsesp::EMSESP::logger().info("Uploading firmware file %s (size: %d KB). Please wait...", filename.c_str(), filesize / 1024);
             // turn off UART to prevent interference with the upload
             emsesp::EMSuart::stop();
 

src/core/emsesp.cpp

@@ -1747,11 +1747,13 @@ void EMSESP::start() {
 
     webSettingsService.begin(); // load EMS-ESP Application settings
 
-    // do any system upgrades
-    if (system_.check_upgrade(factory_settings)) {
-        LOG_WARNING("System needs a restart to apply new settings. Please wait.");
-        system_.system_restart();
-    };
+    // perform any system upgrades
+    if (!factory_settings) {
+        if (system_.check_upgrade()) {
+            LOG_WARNING("System needs a restart to apply new settings. Please wait.");
+            system_.system_restart();
+        };
+    }
 
     // Load our library of known devices into stack mem. Names are stored in Flash memory
     device_library_ = {
@@ -1831,7 +1833,7 @@ void EMSESP::loop() {
 
     // run the loop, unless we're in the middle of an OTA upload
     if (EMSESP::system_.systemStatus() == SYSTEM_STATUS::SYSTEM_STATUS_NORMAL || EMSESP::system_.systemStatus() == SYSTEM_STATUS::SYSTEM_STATUS_INVALID_GPIO) {
-        // check for GPIO Errors
+        // check for GPIO Errors - this is called once when booting
         if (EMSESP::system_.systemStatus() == SYSTEM_STATUS::SYSTEM_STATUS_INVALID_GPIO) {
             static bool only_once = false;
             if (!only_once) {

src/core/system.cpp

@@ -1412,58 +1412,49 @@ bool System::check_restore() {
 // handle upgrades from previous versions
 // this function will not be called on a clean install, with no settings files yet created
 // returns true if we need a reboot
-bool System::check_upgrade(bool factory_settings) {
+bool System::check_upgrade() {
     bool        missing_version = true;
     std::string settingsVersion;
 
-    if (!factory_settings) {
-        // fetch current version from settings file
-        EMSESP::webSettingsService.read([&](WebSettings const & settings) { settingsVersion = settings.version.c_str(); });
+    // fetch current version from settings file
+    EMSESP::webSettingsService.read([&](WebSettings const & settings) { settingsVersion = settings.version.c_str(); });
 
-        // see if we're missing a version, will be < 3.5.0b13 from Dec 23 2022
-        missing_version = (settingsVersion.empty() || (settingsVersion.length() < 5));
-        if (missing_version) {
-            LOG_WARNING("No version information found");
-            settingsVersion = "3.5.0"; // this was the last stable version without version info
-        }
-    } else {
-        settingsVersion = EMSESP_APP_VERSION; // use the current version
+    // see if we're missing a version, will be < 3.5.0b13 from Dec 23 2022
+    missing_version = (settingsVersion.empty() || (settingsVersion.length() < 5));
+    if (missing_version) {
+        LOG_WARNING("No version information found");
+        settingsVersion = "3.5.0"; // this was the last stable version without version info
     }
 
     version::Semver200_version settings_version(settingsVersion);
-
-    if (!missing_version) {
-        LOG_DEBUG("Checking for version upgrades (settings file is v%d.%d.%d-%s)",
-                  settings_version.major(),
-                  settings_version.minor(),
-                  settings_version.patch(),
-                  settings_version.prerelease().c_str());
-    }
-
-    if (factory_settings) {
-        return true; // fresh install, do nothing, no reboot required
-    }
-
     version::Semver200_version this_version(EMSESP_APP_VERSION);
 
-    bool save_version    = true;
-    bool reboot_required = false;
+    std::string settings_version_type = settings_version.prerelease().empty() ? "" : ("-" + settings_version.prerelease());
+    std::string this_version_type     = this_version.prerelease().empty() ? "" : ("-" + this_version.prerelease());
+    bool        save_version          = true;
+    bool        reboot_required       = false;
+
+    LOG_DEBUG("Checking for version upgrades (settings file is v%d.%d.%d%s)",
+              settings_version.major(),
+              settings_version.minor(),
+              settings_version.patch(),
+              settings_version_type.c_str());
 
     // compare versions
     if (this_version > settings_version) {
         // we need to do an upgrade
         if (missing_version) {
-            LOG_NOTICE("Upgrading to version %d.%d.%d-%s", this_version.major(), this_version.minor(), this_version.patch(), this_version.prerelease().c_str());
+            LOG_NOTICE("Upgrading to version %d.%d.%d%s", this_version.major(), this_version.minor(), this_version.patch(), this_version_type);
         } else {
-            LOG_NOTICE("Upgrading from version %d.%d.%d-%s to %d.%d.%d-%s",
+            LOG_NOTICE("Upgrading from version %d.%d.%d%s to %d.%d.%d%s",
                        settings_version.major(),
                        settings_version.minor(),
                        settings_version.patch(),
-                       settings_version.prerelease().c_str(),
+                       settings_version_type.c_str(),
                        this_version.major(),
                        this_version.minor(),
                        this_version.patch(),
-                       this_version.prerelease().c_str());
+                       this_version_type.c_str());
         }
 
         // if we're coming from 3.4.4 or 3.5.0b14 which had no version stored then we need to apply new settings
@@ -1518,11 +1509,9 @@ bool System::check_upgrade(bool factory_settings) {
             }
             return StateUpdateResult::UNCHANGED;
         });
-
-
     } else if (this_version < settings_version) {
         // downgrading
-        LOG_NOTICE("Downgrading to version %d.%d.%d-%s", this_version.major(), this_version.minor(), this_version.patch(), this_version.prerelease().c_str());
+        LOG_NOTICE("Downgrading to version %d.%d.%d%s", this_version.major(), this_version.minor(), this_version.patch(), this_version_type.c_str());
     } else {
         save_version = false; // same version, do nothing
     }
@@ -2686,23 +2675,29 @@ bool System::uploadFirmwareURL(const char * url) {
         return false; // error
     }
 
-    // check we have enough space for the upload in the ota partition
     int firmware_size = http.getSize();
-    LOG_INFO("Firmware uploading (size: %d bytes). Please wait...", firmware_size);
+
+    // check we have a valid size
+    if (firmware_size < 2097152) { // 2MB or greater is required
+        LOG_ERROR("Firmware upload failed - invalid size");
+        http.end();
+        return false; // error
+    }
+
+    // check we have enough space for the upload in the ota partition
     if (!Update.begin(firmware_size)) {
         LOG_ERROR("Firmware upload failed - no space");
         http.end();
         return false; // error
     }
 
+    LOG_INFO("Firmware uploading (size: %d KB). Please wait...", firmware_size / 1024);
+
     Shell::loop_all(); // flush log buffers so latest messages are shown in console
 
     // we're about to start the upload, set the status so the Web System Monitor spots it
     EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING);
 
-    // set a callback so we can monitor progress in the WebUI
-    Update.onProgress([](size_t progress, size_t total) { EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING + (progress * 100 / total)); });
-
     // get tcp stream and send it to Updater
     WiFiClient * stream = http.getStreamPtr();
     if (Update.writeStream(*stream) != firmware_size) {
@@ -2792,8 +2787,10 @@ bool System::command_read(const char * value, const int8_t id) {
 
 // set the system status code - SYSTEM_STATUS in system.h
 void System::systemStatus(uint8_t status_code) {
-    systemStatus_ = status_code;
-    LOG_DEBUG("Setting System status code %d", status_code);
+    if (systemStatus_ != status_code) {
+        systemStatus_ = status_code;
+        LOG_DEBUG("Setting System status code %d", status_code);
+    }
 }
 
 uint8_t System::systemStatus() {

src/core/system.h

@@ -122,7 +122,7 @@ class System {
     void show_mem(const char * note);
     void store_settings(class WebSettings & settings);
     void syslog_init();
-    bool check_upgrade(bool factory_settings);
+    bool check_upgrade();
     bool check_restore();
     void heartbeat_json(JsonObject output);
 

src/devices/thermostat.cpp

@@ -1226,13 +1226,14 @@ void Thermostat::process_RC300Summer(std::shared_ptr<const Telegram> telegram) {
         has_update(telegram, hc->summersetmode, 7);
     }
 
-    if (hc->heatingtype < 3) {
+    if (hc->heatingtype != 3) {
         has_update(telegram, hc->designtemp, 4);
+        has_update(telegram, hc->minflowtemp, 13);
     } else {
         has_update(telegram, hc->designtemp, 5);
+        has_update(telegram, hc->minflowtemp, 8);
     }
 
-    has_update(telegram, hc->minflowtemp, 8);
     has_update(telegram, hc->fastHeatup, 10);
 }
 
@@ -4019,18 +4020,14 @@ bool Thermostat::set_temperature(const float temperature, const uint8_t mode, co
         case HeatingCircuit::Mode::MINFLOW:
             set_typeid      = summer_typeids[hc->hc()];
             validate_typeid = set_typeid;
-            offset          = 8;
+            offset          = hc->heatingtype == 3 ? 8 : 13;
             factor          = 1;
             break;
         case HeatingCircuit::Mode::MAXFLOW:
             set_typeid      = curve_typeids[hc->hc()];
             validate_typeid = set_typeid;
-            if (hc->heatingtype == 3) {
-                offset = 7;
-            } else {
-                offset = 8;
-            }
-            factor = 1;
+            offset          = hc->heatingtype == 3 ? 7 : 8;
+            factor          = 1;
             break;
         case HeatingCircuit::Mode::NOFROST:
             set_typeid      = curve_typeids[hc->hc()];