enable OTA uploads of filesystem (pre_load.json)

.gitignore

@@ -77,3 +77,4 @@ pnpm-lock.yaml
 interface/.tsbuildinfo
 test/test_api/package-lock.json
 .clangd
+mklittlefs

data/pre_load.json

@@ -1,7 +1,6 @@
 {
   "type": "systembackup",
-  "version": "3.8.2",
+  "version": "3.9.0",
-  "date": "2026-03-29T13:28:15",
   "systembackup": [
     {
       "type": "settings",
@@ -9,7 +8,7 @@
         "ssid": "",
         "bssid": "",
         "password": "",
-        "hostname": "ems-esp",
+        "hostname": "ems-esp2",
         "static_ip_config": false,
         "bandwidth20": false,
         "nosleep": true,
@@ -19,7 +18,7 @@
         "tx_power": 0
       },
       "AP": {
-        "provision_mode": 2,
+        "provision_mode": 1,
         "ssid": "ems-esp",
         "password": "ems-esp-neo",
         "channel": 1,
@@ -62,7 +61,7 @@
         "send_response": false
       },
       "NTP": {
-        "enabled": true,
+        "enabled": false,
         "server": "time.google.com",
         "tz_label": "Europe/Amsterdam",
         "tz_format": "CET-1CEST,M3.5.0,M10.5.0/3"
@@ -83,7 +82,7 @@
         ]
       },
       "Settings": {
-        "version": "3.8.2",
+        "version": "3.9.0",
         "board_profile": "E32V2_2",
         "platform": "ESP32",
         "locale": "en",
@@ -132,7 +131,7 @@
         "modbus_port": 502,
         "modbus_max_clients": 10,
         "modbus_timeout": 300,
-        "developer_mode": true,
+        "developer_mode": false,
         "email_enabled": false,
         "email_ssl": false,
         "email_starttls": true,
@@ -154,14 +153,6 @@
     {
       "type": "customizations",
       "Customizations": {
-        "ts": [
-          {
-            "id": "28_1767_7B13_2502",
-            "name": "gateway_temperature",
-            "offset": 0,
-            "is_system": true
-          }
-        ],
         "as": [
           {
             "gpio": 39,
@@ -207,22 +198,14 @@
       }
     },
     {
-      "type": "customSupport",
+      "type": "nvs",
-      "Support": {
+      "nvs": [
-        "html": [
+        {
-          "This product is installed and managed by:",
+          "type": 1,
-          "",
+          "key": "fresh_firmware",
-          "<b>Bosch Installer Example</b>",
+          "value": 0
-          "",
+        }
-          "Nefit Road 12",
+      ]
-          "1234 AB Amsterdam",
-          "Phone: +31 123 456 789",
-          "email: support@boschinstaller.nl",
-          "",
-          "For help and questions please <a target='_blank' href='https://emsesp.org'>contact</a> your installer."
-        ],
-        "img_url": "https://emsesp.org/media/images/designer.png"
-      }
     }
   ]
 }

src/ESP32React/UploadFileService.cpp

@@ -4,6 +4,7 @@
 
 #include <esp_app_format.h>
 #include <esp_ota_ops.h>
+// #include <esp_partition.h>
 
 static String getFilenameExtension(const String & filename) {
     const auto pos = filename.lastIndexOf('.');
@@ -16,8 +17,8 @@ static String getFilenameExtension(const String & filename) {
 UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * securityManager)
     : _securityManager(securityManager)
     , _is_firmware(false)
+    , _is_filesystem(false)
     , _md5() {
-    // upload a file via a form
     server->on(
         UPLOAD_FILE_PATH,
         HTTP_POST,
@@ -41,8 +42,14 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
         const String      extension = getFilenameExtension(filename);
         const std::size_t filesize  = request->contentLength();
 
-        _is_firmware = false;
+        _is_firmware   = false;
-        if ((extension == "bin") && (filesize > 1000000)) {
+        _is_filesystem = false;
+
+        if (extension == "bin" && filename.endsWith("littlefs.bin")) {
+            // LittleFS filesystem image
+            _is_filesystem = true;
+            _md5[0]        = '\0'; // clear any stale md5 so Update.end() doesn't compare against it
+        } else if ((extension == "bin") && (filesize > 2000000)) {
             _is_firmware = true;
         } else if (extension == "json") {
             _md5[0] = '\0'; // clear md5
@@ -88,6 +95,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
 #endif
             // it's firmware - initialize the ArduinoOTA updater
             emsesp::EMSESP::logger().info("Uploading firmware file %s (size: %d KB). Please wait...", filename.c_str(), filesize / 1024);
+
             // turn off UART to prevent interference with the upload
             emsesp::EMSuart::stop();
 
@@ -96,28 +104,55 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
                     Update.setMD5(_md5.data());
                     _md5.front() = '\0';
                 }
-                request->onDisconnect([this] { handleEarlyDisconnect(); }); // success, let's make sure we end the update if the client hangs up
+                request->onDisconnect([this] { handleDisconnect(); }); // success, let's make sure we end the update if the client hangs up
             } else {
                 handleError(request, 507); // failed to begin, send an error response Insufficient Storage
                 return;
             }
+        } else if (_is_filesystem) {
+            // LittleFS filesystem image - flash directly to the spiffs/littlefs partition
+            emsesp::EMSESP::logger().info("Uploading filesystem image %s (size: %u KB). Please wait...", filename.c_str(), static_cast<unsigned>(filesize / 1024));
+            emsesp::EMSuart::stop();
+            LittleFS.end(); // unmount LittleFS before we overwrite the partition under it
+
+            // request->contentLength() is the multipart HTTP body size, not the file size,
+            // so it can exceed the partition by a few hundred bytes. Use UPDATE_SIZE_UNKNOWN
+            // and let the Update library size against the whole partition.
+            if (Update.begin(UPDATE_SIZE_UNKNOWN, U_SPIFFS)) {
+                // emsesp::EMSESP::logger().info("Update.begin(U_SPIFFS) ok, partition size %u bytes", static_cast<unsigned>(Update.size()));
+                request->onDisconnect([this] { handleDisconnect(); });
+            } else {
+                emsesp::EMSESP::logger().err("Update.begin(U_SPIFFS) failed: %s", Update.errorString());
+                handleError(request, 507);
+                return;
+            }
         } else {
             // its a normal file, open a new temp file to write the contents too
             request->_tempFile = LittleFS.open(TEMP_FILENAME_PATH, "w");
         }
     }
 
-    if (!_is_firmware) {
+    if (_is_firmware || _is_filesystem) {
-        if (len && len != request->_tempFile.write(data, len)) { // stream the incoming chunk to the opened file
+        if (!request->_tempObject) { // if we haven't delt with an error, continue with the OTA update
-            handleError(request, 507);                           // 507-Insufficient Storage
+            if (Update.write(data, len) != len) {
-        }
+                emsesp::EMSESP::logger().err("Update.write failed at offset %u (chunk %u): %s",
-    } else if (!request->_tempObject) { // if we haven't delt with an error, continue with the firmware update
+                                             static_cast<unsigned>(Update.progress()),
-        if (Update.write(data, len) != len) {
+                                             static_cast<unsigned>(len),
-            handleError(request, 500); // internal error, failed
+                                             Update.errorString());
-            return;
+                handleError(request, 500); // internal error, failed
-        }
+                return;
-        if (final && !Update.end(true)) {
+            }
-            handleError(request, 500); // internal error, failed
+            if (final) {
+                if (!Update.end(true)) {
+                    emsesp::EMSESP::logger().err("Update.end failed: %s", Update.errorString());
+                    handleError(request, 500);
+                    return;
+                }
+            }
+        } else {
+            if (len && len != request->_tempFile.write(data, len)) { // stream the incoming chunk to the opened file
+                handleError(request, 507);                           // 507-Insufficient Storage
+            }
         }
     }
 }
@@ -135,11 +170,13 @@ void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
         return;
     }
 
-    // check if it was a firmware upgrade
+    // check if it was a firmware or filesystem image upgrade
-    // if no error, send the success response as a JSON
+    // if no error, send the success response and request a restart
-    if (_is_firmware && !request->_tempObject) {
+    if ((_is_firmware || _is_filesystem) && !request->_tempObject) {
-        // set NVS to tell EMS-ESP this is a new fresh firmware on next restart
+        if (_is_firmware) {
-        emsesp::EMSESP::nvs_.putBool(emsesp::EMSESP_NVS_BOOT_NEW_FIRMWARE, true);
+            // set NVS to tell EMS-ESP this is a new fresh firmware on next restart
+            emsesp::EMSESP::nvs_.putBool(emsesp::EMSESP_NVS_BOOT_NEW_FIRMWARE, true);
+        }
 
         AsyncWebServerResponse * response = request->beginResponse(200);
         request->send(response);
@@ -178,15 +215,21 @@ void UploadFileService::handleError(AsyncWebServerRequest * request, int code) {
     // that is caught by the web code. Unfortunately the http error code is not sent to the client on fast network connections
     if (code == 406) {
         request->client()->close();
-        _is_firmware = false;
+        _is_firmware   = false;
+        _is_filesystem = false;
         Update.abort();
     }
+
+    // if we aborted a filesystem upload, remount LittleFS so the device keeps working
+    if (_is_filesystem) {
+        LittleFS.begin();
+    }
 }
 
-void UploadFileService::handleEarlyDisconnect() {
+void UploadFileService::handleDisconnect() {
     emsesp::EMSESP::logger().info("Upload finished");
     emsesp::EMSESP::system_.uart_init(); // re-enable UART
 
-    _is_firmware = false;
+    _is_firmware   = false;
-    Update.abort();
+    _is_filesystem = false;
 }

src/ESP32React/UploadFileService.h

@@ -22,13 +22,14 @@ class UploadFileService {
   private:
     SecurityManager *    _securityManager;
     bool                 _is_firmware;
+    bool                 _is_filesystem;
     std::array<char, 33> _md5;
 
     void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final);
     void uploadComplete(AsyncWebServerRequest * request);
     void handleError(AsyncWebServerRequest * request, int code);
 
-    void handleEarlyDisconnect();
+    void handleDisconnect();
 };
 
 #endif

src/core/emsesp.cpp

@@ -1711,6 +1711,11 @@ void EMSESP::start() {
     bool factory_settings = false;
 #endif
 
+#if defined(EMSESP_DEBUG)
+    // LOG_DEBUG("Listing root directory before:");
+    // system_.listDir("/", 3); // show the contents of the root directory
+#endif
+
     // start NVS storage
     if (!nvs_.begin("ems-esp", false, "nvs1")) { // try bigger nvs partition on 16M flash first
         nvs_.begin("ems-esp", false, "nvs");     // fallback to small nvs
@@ -1725,6 +1730,11 @@ void EMSESP::start() {
     // loads core system services settings (mqtt, ap, ntp etc)
     esp32React.begin();
 
+#if defined(EMSESP_DEBUG)
+    // LOG_DEBUG("Listing root directory after:");
+    // system_.listDir("/", 3); // show the contents of the root directory
+#endif
+
 #ifndef EMSESP_STANDALONE
     if (factory_settings) {
         LOG_WARNING("No settings found on filesystem. Using factory settings.");

src/core/network.cpp

@@ -412,7 +412,7 @@ void Network::startmDNS() const {
     MDNS.addService("telnet", "tcp", 23); // add our telnet console
     MDNS.addServiceTxt("http", "tcp", "address", address_s.c_str());
 
-    emsesp::EMSESP::logger().info("Starting mDNS Responder service");
+    emsesp::EMSESP::logger().info("Starting mDNS Responder service for %s", address_s.c_str());
 #endif
 }
 

src/core/system.cpp

@@ -732,18 +732,16 @@ void System::start() {
     last_system_check_ = 0; // force the LED to go from fast flash to pulse
     uart_init();            // start UART
     syslog_init();          // start syslog
-    modbus_init();   // start modbus
+    modbus_init();          // start modbus
 }
 
 // button single click
 void System::button_OnClick(PButton & b) {
     LOG_NOTICE("Button pressed - single click");
 
-#if defined(EMSESP_TEST)
 #ifndef EMSESP_STANDALONE
     // show filesystem
-    Test::listDir(LittleFS, "/", 3);
+    listDir("/", 3);
-#endif
 #endif
 }
 
@@ -3478,4 +3476,39 @@ void System::restore_snapshot_gpios(std::vector<int8_t> & u_gpios, std::vector<i
     }
 }
 
+// show the contents of a directory in the LittleFS filesystem
+void System::listDir(const char * dirname, uint8_t levels) {
+#if defined(EMSESP_DEBUG)
+#ifndef EMSESP_STANDALONE
+
+    File root = LittleFS.open(dirname);
+    if (!root) {
+        LOG_DEBUG("Failed to open directory %s", dirname);
+        return;
+    }
+    if (!root.isDirectory()) {
+        LOG_DEBUG("%s is not a directory", dirname);
+        return;
+    }
+
+    LOG_DEBUG("(directory) %s", dirname);
+
+    File file = root.openNextFile();
+    while (file) {
+        if (file.isDirectory()) {
+            std::string line = std::string(file.name()) + "/";
+            if (levels) {
+                // prefix a / to the name to make it a full path
+                listDir(("/" + String(file.name())).c_str(), levels - 1);
+            }
+        } else {
+            std::string line = "  (file) " + std::string(file.name()) + " (" + std::to_string(file.size()) + " bytes)";
+            LOG_DEBUG("%s", line.c_str());
+        }
+        file = root.openNextFile();
+    }
+#endif
+#endif
+}
+
 } // namespace emsesp

src/core/system.h

@@ -102,6 +102,8 @@ class System {
     static void        get_value_json(JsonObject output, const std::string & circuit, const std::string & name, JsonVariant val);
     static std::string get_metrics_prometheus();
 
+    static void listDir(const char * dirname, uint8_t levels);
+
 #if defined(EMSESP_TEST)
     static bool command_test(const char * value, const int8_t id);
 #endif

src/test/test.cpp

@@ -2666,45 +2666,6 @@ void Test::add_device(uint8_t device_id, uint8_t product_id) {
     uart_telegram({device_id, EMSESP_DEFAULT_EMS_BUS_ID, EMSdevice::EMS_TYPE_VERSION, 0, product_id, 1, 0});
 }
 
-#ifdef EMSESP_TEST
-#ifndef EMSESP_STANDALONE
-void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) {
-    Serial.println();
-    Serial.printf("%s\r\n", dirname);
-
-    File root = fs.open(dirname);
-    if (!root) {
-        Serial.println("- failed to open directory");
-        return;
-    }
-    if (!root.isDirectory()) {
-        Serial.println(" - not a directory");
-        return;
-    }
-
-    File file = root.openNextFile();
-    while (file) {
-        if (file.isDirectory()) {
-            Serial.print(file.name());
-            Serial.println("/");
-            if (levels) {
-                // prefix a / to the name to make it a full path
-                listDir(fs, ("/" + String(file.name())).c_str(), levels - 1);
-            }
-            Serial.println();
-        } else {
-            Serial.print(" ");
-            Serial.print(file.name());
-            Serial.print(" (");
-            Serial.print(file.size());
-            Serial.println(" bytes)");
-        }
-        file = root.openNextFile();
-    }
-}
-#endif
-#endif
-
 } // namespace emsesp
 
 #endif

src/test/test.h

@@ -80,7 +80,6 @@ class Test {
     static void uart_telegram_withCRC(const char * rx_data);
     static void add_device(uint8_t device_id, uint8_t product_id);
     static void refresh();
-    static void listDir(fs::FS & fs, const char * dirname, uint8_t levels);
 };
 
 } // namespace emsesp