support filesystem ota

src/ESP32React/UploadFileService.cpp

@@ -4,6 +4,7 @@
 
 #include <esp_app_format.h>
 #include <esp_ota_ops.h>
+// #include <esp_partition.h>
 
 static String getFilenameExtension(const String & filename) {
     const auto pos = filename.lastIndexOf('.');
@@ -16,8 +17,8 @@ static String getFilenameExtension(const String & filename) {
 UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * securityManager)
     : _securityManager(securityManager)
     , _is_firmware(false)
+    , _is_filesystem(false)
     , _md5() {
-    // upload a file via a form
     server->on(
         UPLOAD_FILE_PATH,
         HTTP_POST,
@@ -42,7 +43,13 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
         const std::size_t filesize  = request->contentLength();
 
         _is_firmware   = false;
-        if ((extension == "bin") && (filesize > 1000000)) {
+        _is_filesystem = false;
+
+        if (extension == "bin" && filename.endsWith("littlefs.bin")) {
+            // LittleFS filesystem image
+            _is_filesystem = true;
+            _md5[0]        = '\0'; // clear any stale md5 so Update.end() doesn't compare against it
+        } else if ((extension == "bin") && (filesize > 2000000)) {
             _is_firmware = true;
         } else if (extension == "json") {
             _md5[0] = '\0'; // clear md5
@@ -88,6 +95,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
 #endif
             // it's firmware - initialize the ArduinoOTA updater
             emsesp::EMSESP::logger().info("Uploading firmware file %s (size: %d KB). Please wait...", filename.c_str(), filesize / 1024);
+
             // turn off UART to prevent interference with the upload
             emsesp::EMSuart::stop();
 
@@ -96,28 +104,55 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
                     Update.setMD5(_md5.data());
                     _md5.front() = '\0';
                 }
-                request->onDisconnect([this] { handleEarlyDisconnect(); }); // success, let's make sure we end the update if the client hangs up
+                request->onDisconnect([this] { handleDisconnect(); }); // success, let's make sure we end the update if the client hangs up
             } else {
                 handleError(request, 507); // failed to begin, send an error response Insufficient Storage
                 return;
             }
+        } else if (_is_filesystem) {
+            // LittleFS filesystem image - flash directly to the spiffs/littlefs partition
+            emsesp::EMSESP::logger().info("Uploading filesystem image %s (size: %u KB). Please wait...", filename.c_str(), static_cast<unsigned>(filesize / 1024));
+            emsesp::EMSuart::stop();
+            LittleFS.end(); // unmount LittleFS before we overwrite the partition under it
+
+            // request->contentLength() is the multipart HTTP body size, not the file size,
+            // so it can exceed the partition by a few hundred bytes. Use UPDATE_SIZE_UNKNOWN
+            // and let the Update library size against the whole partition.
+            if (Update.begin(UPDATE_SIZE_UNKNOWN, U_SPIFFS)) {
+                // emsesp::EMSESP::logger().info("Update.begin(U_SPIFFS) ok, partition size %u bytes", static_cast<unsigned>(Update.size()));
+                request->onDisconnect([this] { handleDisconnect(); });
+            } else {
+                emsesp::EMSESP::logger().err("Update.begin(U_SPIFFS) failed: %s", Update.errorString());
+                handleError(request, 507);
+                return;
+            }
         } else {
             // its a normal file, open a new temp file to write the contents too
             request->_tempFile = LittleFS.open(TEMP_FILENAME_PATH, "w");
         }
     }
 
-    if (!_is_firmware) {
+    if (_is_firmware || _is_filesystem) {
-        if (len && len != request->_tempFile.write(data, len)) { // stream the incoming chunk to the opened file
+        if (!request->_tempObject) { // if we haven't delt with an error, continue with the OTA update
-            handleError(request, 507);                           // 507-Insufficient Storage
-        }
-    } else if (!request->_tempObject) { // if we haven't delt with an error, continue with the firmware update
             if (Update.write(data, len) != len) {
+                emsesp::EMSESP::logger().err("Update.write failed at offset %u (chunk %u): %s",
+                                             static_cast<unsigned>(Update.progress()),
+                                             static_cast<unsigned>(len),
+                                             Update.errorString());
                 handleError(request, 500); // internal error, failed
                 return;
             }
-        if (final && !Update.end(true)) {
+            if (final) {
-            handleError(request, 500); // internal error, failed
+                if (!Update.end(true)) {
+                    emsesp::EMSESP::logger().err("Update.end failed: %s", Update.errorString());
+                    handleError(request, 500);
+                    return;
+                }
+            }
+        } else {
+            if (len && len != request->_tempFile.write(data, len)) { // stream the incoming chunk to the opened file
+                handleError(request, 507);                           // 507-Insufficient Storage
+            }
         }
     }
 }
@@ -135,11 +170,13 @@ void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
         return;
     }
 
-    // check if it was a firmware upgrade
+    // check if it was a firmware or filesystem image upgrade
-    // if no error, send the success response as a JSON
+    // if no error, send the success response and request a restart
-    if (_is_firmware && !request->_tempObject) {
+    if ((_is_firmware || _is_filesystem) && !request->_tempObject) {
+        if (_is_firmware) {
             // set NVS to tell EMS-ESP this is a new fresh firmware on next restart
             emsesp::EMSESP::nvs_.putBool(emsesp::EMSESP_NVS_BOOT_NEW_FIRMWARE, true);
+        }
 
         AsyncWebServerResponse * response = request->beginResponse(200);
         request->send(response);
@@ -179,14 +216,20 @@ void UploadFileService::handleError(AsyncWebServerRequest * request, int code) {
     if (code == 406) {
         request->client()->close();
         _is_firmware   = false;
+        _is_filesystem = false;
         Update.abort();
     }
+
+    // if we aborted a filesystem upload, remount LittleFS so the device keeps working
+    if (_is_filesystem) {
+        LittleFS.begin();
+    }
 }
 
-void UploadFileService::handleEarlyDisconnect() {
+void UploadFileService::handleDisconnect() {
     emsesp::EMSESP::logger().info("Upload finished");
     emsesp::EMSESP::system_.uart_init(); // re-enable UART
 
     _is_firmware   = false;
-    Update.abort();
+    _is_filesystem = false;
 }

src/ESP32React/UploadFileService.h

@@ -22,13 +22,14 @@ class UploadFileService {
   private:
     SecurityManager *    _securityManager;
     bool                 _is_firmware;
+    bool                 _is_filesystem;
     std::array<char, 33> _md5;
 
     void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final);
     void uploadComplete(AsyncWebServerRequest * request);
     void handleError(AsyncWebServerRequest * request, int code);
 
-    void handleEarlyDisconnect();
+    void handleDisconnect();
 };
 
 #endif

src/core/emsesp.cpp

@@ -1710,6 +1710,11 @@ void EMSESP::start() {
     bool factory_settings = false;
 #endif
 
+#if defined(EMSESP_DEBUG)
+    // LOG_DEBUG("Listing root directory before:");
+    // system_.listDir("/", 3); // show the contents of the root directory
+#endif
+
     // start NVS storage
     if (!nvs_.begin("ems-esp", false, "nvs1")) { // try bigger nvs partition on 16M flash first
         nvs_.begin("ems-esp", false, "nvs");     // fallback to small nvs
@@ -1724,6 +1729,11 @@ void EMSESP::start() {
     // loads core system services settings (network, mqtt, ap, ntp etc)
     esp32React.begin();
 
+#if defined(EMSESP_DEBUG)
+    // LOG_DEBUG("Listing root directory before:");
+    // system_.listDir("/", 3); // show the contents of the root directory
+#endif
+
 #ifndef EMSESP_STANDALONE
     if (factory_settings) {
         LOG_WARNING("No settings found on filesystem. Using factory settings.");

src/core/system.cpp

@@ -642,11 +642,9 @@ void System::start() {
 void System::button_OnClick(PButton & b) {
     LOG_NOTICE("Button pressed - single click");
 
-#if defined(EMSESP_TEST)
 #ifndef EMSESP_STANDALONE
     // show filesystem
-    Test::listDir(LittleFS, "/", 3);
+    listDir("/", 3);
-#endif
 #endif
 }
 
@@ -3435,4 +3433,39 @@ void System::restore_snapshot_gpios(std::vector<int8_t> & u_gpios, std::vector<i
     }
 }
 
+// show the contents of a directory in the LittleFS filesystem
+void System::listDir(const char * dirname, uint8_t levels) {
+#if defined(EMSESP_DEBUG)
+#ifndef EMSESP_STANDALONE
+
+    File root = LittleFS.open(dirname);
+    if (!root) {
+        LOG_DEBUG("Failed to open directory %s", dirname);
+        return;
+    }
+    if (!root.isDirectory()) {
+        LOG_DEBUG("%s is not a directory", dirname);
+        return;
+    }
+
+    LOG_DEBUG("(directory) %s", dirname);
+
+    File file = root.openNextFile();
+    while (file) {
+        if (file.isDirectory()) {
+            std::string line = std::string(file.name()) + "/";
+            if (levels) {
+                // prefix a / to the name to make it a full path
+                listDir(("/" + String(file.name())).c_str(), levels - 1);
+            }
+        } else {
+            std::string line = "  (file) " + std::string(file.name()) + " (" + std::to_string(file.size()) + " bytes)";
+            LOG_DEBUG("%s", line.c_str());
+        }
+        file = root.openNextFile();
+    }
+#endif
+#endif
+}
+
 } // namespace emsesp

src/core/system.h

@@ -109,6 +109,8 @@ class System {
     static void        get_value_json(JsonObject output, const std::string & circuit, const std::string & name, JsonVariant val);
     static std::string get_metrics_prometheus();
 
+    static void listDir(const char * dirname, uint8_t levels);
+
 #if defined(EMSESP_TEST)
     static bool command_test(const char * value, const int8_t id);
 #endif

src/core/telegram.h

@@ -173,7 +173,8 @@ class EMSbus {
     }
 
     static void set_ems2() {
-        isEMS2_ = true;;
+        isEMS2_ = true;
+        ;
     }
 
     static uint8_t ems_mask() {

src/test/test.cpp

@@ -450,7 +450,7 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
 // THESE ONLY WORK WITH AN ESP32, not in standalone/native mode
 #ifndef EMSESP_STANDALONE
     if (command == "ls") {
-        listDir(LittleFS, "/", 3);
+        System::listDir("/", 3);
         ok = true;
     }
 
@@ -2676,45 +2676,6 @@ void Test::add_device(uint8_t device_id, uint8_t product_id) {
     uart_telegram({device_id, EMSESP_DEFAULT_EMS_BUS_ID, EMSdevice::EMS_TYPE_VERSION, 0, product_id, 1, 0});
 }
 
-#ifdef EMSESP_TEST
-#ifndef EMSESP_STANDALONE
-void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) {
-    Serial.println();
-    Serial.printf("%s\r\n", dirname);
-
-    File root = fs.open(dirname);
-    if (!root) {
-        Serial.println("- failed to open directory");
-        return;
-    }
-    if (!root.isDirectory()) {
-        Serial.println(" - not a directory");
-        return;
-    }
-
-    File file = root.openNextFile();
-    while (file) {
-        if (file.isDirectory()) {
-            Serial.print(file.name());
-            Serial.println("/");
-            if (levels) {
-                // prefix a / to the name to make it a full path
-                listDir(fs, ("/" + String(file.name())).c_str(), levels - 1);
-            }
-            Serial.println();
-        } else {
-            Serial.print(" ");
-            Serial.print(file.name());
-            Serial.print(" (");
-            Serial.print(file.size());
-            Serial.println(" bytes)");
-        }
-        file = root.openNextFile();
-    }
-}
-#endif
-#endif
-
 } // namespace emsesp
 
 #endif

src/test/test.h

@@ -80,7 +80,6 @@ class Test {
     static void uart_telegram_withCRC(const char * rx_data);
     static void add_device(uint8_t device_id, uint8_t product_id);
     static void refresh();
-    static void listDir(fs::FS & fs, const char * dirname, uint8_t levels);
 };
 
 } // namespace emsesp