Merge branch 'dev' of https://github.com/emsesp/EMS-ESP32 into disinfect-duration-3278

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/ESP32React/UploadFileService.cpp

@@ -2,44 +2,14 @@
 
 #include <emsesp.h>
 
-#include <esp_app_format.h>
 #include <esp_ota_ops.h>
 
-static String getFilenameExtension(const String & filename) {
-    const auto pos = filename.lastIndexOf('.');
-    if (pos != -1) {
-        return filename.substring(static_cast<unsigned int>(pos) + 1);
-    }
-    return {};
-}
-
-// Accepts a raw 32-char hex digest, optionally with trailing newline or a GNU "hash  filename" suffix.
-static bool parseMd5Digest(const uint8_t * data, size_t len, std::array<char, 33> & out) {
-    size_t i = 0;
-    while (i < len && (data[i] == ' ' || data[i] == '\t' || data[i] == '\r' || data[i] == '\n')) {
-        ++i;
-    }
-    if (len - i < 32) {
-        return false;
-    }
-    for (size_t n = 0; n < 32; n++) {
-        const char c   = static_cast<char>(data[i + n]);
-        const bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
-        if (!hex) {
-            return false;
-        }
-        out[n] = (c >= 'A' && c <= 'F') ? static_cast<char>(c - 'A' + 'a') : c;
-    }
-    out[32] = '\0';
-    return true;
-}
-
 UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * securityManager)
     : _securityManager(securityManager)
     , _is_firmware(false)
     , _is_filesystem(false)
-    , _md5_applied(false)
-    , _md5() {
+    , _is_recovery(false)
+    , _response_sent(false) {
     server->on(
         UPLOAD_FILE_PATH,
         HTTP_POST,
@@ -60,68 +30,61 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
     // at init
     if (!index) {
         // check details of the file, to see if its a valid bin or json file
-        const String      extension = getFilenameExtension(filename);
-        const std::size_t filesize  = request->contentLength();
+        const std::size_t filesize = request->contentLength();
 
         _is_firmware   = false;
         _is_filesystem = false;
+        _is_recovery   = false;
+        _response_sent = false;
 
-        if (extension == "bin" && filename.endsWith("littlefs.bin")) {
-            // LittleFS filesystem image
+        switch (OtaUpdater::classify(filename, filesize, emsesp::System::MIN_FIRMWARE_SIZE)) {
+        case OtaUpdater::FileType::FILESYSTEM:
             _is_filesystem = true;
-            _md5[0]        = '\0'; // clear any stale md5 so Update.end() doesn't compare against it
-            _md5_applied   = false;
-        } else if ((extension == "bin") && (filesize >= emsesp::System::MIN_FIRMWARE_SIZE)) {
+            _ota.clearMd5(); // clear any stale md5 so Update.end() doesn't compare against it
+            break;
+        case OtaUpdater::FileType::FIRMWARE:
             _is_firmware = true;
-        } else if (extension == "json") {
-            _md5[0]      = '\0'; // clear md5
-            _md5_applied = false;
-        } else if (extension == "md5") {
-            if (!parseMd5Digest(data, len, _md5)) {
+            break;
+        case OtaUpdater::FileType::JSON:
+            _ota.clearMd5();
+            break;
+        case OtaUpdater::FileType::MD5:
+            if (!_ota.setMd5(data, len)) {
                 emsesp::EMSESP::logger().err("Invalid MD5 digest file");
                 handleError(request, 406); // Not Acceptable
             } else {
-                emsesp::EMSESP::logger().info("MD5 digest received (%s). Now upload the firmware BIN", _md5.data());
+                emsesp::EMSESP::logger().info("MD5 digest received (%s). Now upload the firmware BIN", _ota.md5());
             }
             return;
-        } else {
-            _md5.front() = '\0';
-            _md5_applied = false;
+        default:
+            _ota.clearMd5();
             emsesp::EMSESP::logger().err("Unsupported file type: %s, size: %u", filename.c_str(), filesize);
             handleError(request, 406); // Not Acceptable - unsupported file type
             return;
         }
 
         if (_is_firmware) {
-            // Check firmware header, 0xE9 magic offset 0 indicates esp bin, chip offset 12: esp32:0, S2:2, C3:5
-#if CONFIG_IDF_TARGET_ESP32 // ESP32/PICO-D4
-            if (len > 12 && (data[0] != ESP_IMAGE_HEADER_MAGIC || data[12] != ESP_CHIP_ID_ESP32)) {
-                handleError(request, 503); // service unavailable
-                return;
-            }
-#elif CONFIG_IDF_TARGET_ESP32S2
-            if (len > 12 && (data[0] != ESP_IMAGE_HEADER_MAGIC || data[12] != ESP_CHIP_ID_ESP32S2)) {
-                handleError(request, 503); // service unavailable
-                return;
-            }
-#elif CONFIG_IDF_TARGET_ESP32C3
-            if (len > 12 && (data[0] != ESP_IMAGE_HEADER_MAGIC || data[12] != ESP_CHIP_ID_ESP32C3)) {
-                handleError(request, 503); // service unavailable
-                return;
-            }
-#elif CONFIG_IDF_TARGET_ESP32S3
-            if (len > 12 && (data[0] != ESP_IMAGE_HEADER_MAGIC || data[12] != ESP_CHIP_ID_ESP32S3)) {
-                handleError(request, 503); // service unavailable
-                return;
-            }
-#elif CONFIG_IDF_TARGET_ESP32C6
-            if (len > 12 && (data[0] != ESP_IMAGE_HEADER_MAGIC || data[12] != ESP_CHIP_ID_ESP32C6)) {
+            if (!OtaUpdater::isCompatibleFirmware(data, len)) {
                 handleError(request, 503); // service unavailable
                 return;
             }
+
+            const esp_partition_t * target = nullptr;
+#ifdef EMSESP_HAS_RECOVERY
+            if (OtaUpdater::isRecoveryFirmware(data, len)) {
+                target = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr);
+                if (target == nullptr || target == esp_ota_get_running_partition()) {
+                    emsesp::EMSESP::logger().err("The recovery firmware can't be installed on this board");
+                    handleError(request, 406);
+                    return;
+                }
+                _is_recovery = true;
+                emsesp::EMSESP::logger().info("Recovery firmware uploading to the %s partition (size: %dKB). Please wait...", target->label, filesize / 1024);
+            } else
 #endif
-            // it's firmware - initialize the ArduinoOTA updater
-            emsesp::EMSESP::logger().info("Firmware uploading (file %s, size: %dKB). Please wait...", filename.c_str(), filesize / 1024);
+            {
+                emsesp::EMSESP::logger().info("Firmware uploading (file %s, size: %dKB). Please wait...", filename.c_str(), filesize / 1024);
+            }
 
             // turn off UART to prevent interference with the upload
             emsesp::EMSuart::stop();
@@ -129,15 +92,13 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
             // tell main loop we're uploading so services as paused (e.g. MQTT)
             emsesp::EMSESP::system_.systemStatus(emsesp::SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING);
 
-            if (Update.begin(filesize - sizeof(esp_image_header_t))) {
-                _md5_applied = false;
-                if (strlen(_md5.data()) == _md5.size() - 1) {
-                    Update.setMD5(_md5.data());
-                    _md5_applied = true;
-                    emsesp::EMSESP::logger().info("Firmware MD5 check enabled (%s)", _md5.data());
+            if (_ota.beginFirmware(filesize, target, !_is_recovery)) {
+                if (_ota.md5Applied()) {
+                    emsesp::EMSESP::logger().info("Firmware MD5 check enabled (%s)", _ota.md5());
                 }
                 request->onDisconnect([this] { handleDisconnect(); }); // success, let's make sure we end the update if the client hangs up
             } else {
+                emsesp::EMSESP::logger().err("OTA begin failed: %s", _ota.errorString());
                 handleError(request, 507); // failed to begin, send an error response Insufficient Storage
                 return;
             }
@@ -147,14 +108,10 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
             emsesp::EMSuart::stop();
             LittleFS.end(); // unmount LittleFS before we overwrite the partition under it
 
-            // request->contentLength() is the multipart HTTP body size, not the file size,
-            // so it can exceed the partition by a few hundred bytes. Use UPDATE_SIZE_UNKNOWN
-            // and let the Update library size against the whole partition.
-            if (Update.begin(UPDATE_SIZE_UNKNOWN, U_SPIFFS)) {
-                // emsesp::EMSESP::logger().info("Update.begin(U_SPIFFS) ok, partition size %u bytes", static_cast<unsigned>(Update.size()));
+            if (_ota.beginFilesystem()) {
                 request->onDisconnect([this] { handleDisconnect(); });
             } else {
-                emsesp::EMSESP::logger().err("Update.begin(U_SPIFFS) failed: %s", Update.errorString());
+                emsesp::EMSESP::logger().err("Update.begin(U_SPIFFS) failed: %s", _ota.errorString());
                 handleError(request, 507);
                 return;
             }
@@ -167,17 +124,18 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
     if (_is_firmware || _is_filesystem) {
         if (!request->_tempObject) {
             //continue with the OTA update
-            if (Update.write(data, len) != len) {
+            if (!_ota.write(data, len)) {
                 emsesp::EMSESP::logger().err("OTA update failed at offset %u (chunk %u): %s",
-                                             static_cast<unsigned>(Update.progress()),
+                                             static_cast<unsigned>(_ota.progress()),
                                              static_cast<unsigned>(len),
-                                             Update.errorString());
+                                             _ota.errorString());
+                _ota.abort();
                 handleError(request, 500); // internal error, failed
                 return;
             }
             if (final) {
-                if (!Update.end(true)) {
-                    emsesp::EMSESP::logger().err("OTA update failed: %s", Update.errorString());
+                if (!_ota.end()) {
+                    emsesp::EMSESP::logger().err("OTA update failed: %s", _ota.errorString());
                     handleError(request, 500); // internal error, failed
                     return;
                 }
@@ -192,6 +150,19 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
 }
 
 void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
+    if (request->_tempObject) {
+        const int code = static_cast<int>(reinterpret_cast<intptr_t>(request->_tempObject));
+        if (!_response_sent && code >= 400) {
+            request->send(request->beginResponse(code));
+            _response_sent = true;
+        }
+        _is_firmware   = false;
+        _is_filesystem = false;
+        _is_recovery   = false;
+        emsesp::EMSESP::system_.systemStatus(emsesp::SYSTEM_STATUS::SYSTEM_STATUS_NORMAL);
+        return;
+    }
+
     emsesp::EMSESP::logger().info("Upload successful");
 
     // did we just complete uploading a json file?
@@ -206,21 +177,39 @@ void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
 
     // check if it was a firmware or filesystem image upgrade
     // if no error, send the success response and request a restart
+#ifdef EMSESP_HAS_RECOVERY
+    if (_is_recovery && !request->_tempObject) {
+        emsesp::EMSESP::system_.recovery_installed();
+        auto *     response = new emsesp::PsramAsyncJsonResponse(false);
+        JsonObject root     = response->getRoot();
+        root["recovery"]    = true;
+        if (_ota.md5Applied()) {
+            emsesp::EMSESP::logger().info("Firmware MD5 matches");
+            root["md5_ok"] = true;
+        }
+        response->setLength();
+        request->send(response);
+        _ota.clearMd5();
+        _is_recovery = false;
+        emsesp::EMSESP::system_.systemStatus(emsesp::SYSTEM_STATUS::SYSTEM_STATUS_NORMAL); // no restart needed
+        return;
+    }
+#endif
+
     if ((_is_firmware || _is_filesystem) && !request->_tempObject) {
         if (_is_firmware) {
             // set NVS to tell EMS-ESP this is a new fresh firmware on next restart
             emsesp::EMSESP::nvs_.putBool(emsesp::EMSESP_NVS_BOOT_NEW_FIRMWARE, true);
         }
 
-        if (_is_firmware && _md5_applied) {
+        if (_is_firmware && _ota.md5Applied()) {
             emsesp::EMSESP::logger().info("Firmware MD5 matches");
             auto *     response = new emsesp::PsramAsyncJsonResponse(false);
             JsonObject root     = response->getRoot();
             root["md5_ok"]      = true;
             response->setLength();
             request->send(response);
-            _md5.front() = '\0';
-            _md5_applied = false;
+            _ota.clearMd5();
         } else {
             AsyncWebServerResponse * response = request->beginResponse(200);
             request->send(response);
@@ -231,10 +220,10 @@ void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
     }
 
     // add MD5 to the response
-    if (strlen(_md5.data()) == _md5.size() - 1) {
+    if (_ota.hasMd5()) {
         auto *     response = new emsesp::PsramAsyncJsonResponse(false);
         JsonObject root     = response->getRoot();
-        root["md5"]         = _md5.data();
+        root["md5"]         = _ota.md5();
         response->setLength();
         request->send(response);
         return;
@@ -244,25 +233,25 @@ void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
 }
 
 void UploadFileService::handleError(AsyncWebServerRequest * request, int code) {
-    emsesp::EMSESP::logger().info("Upload error: %d", code);
-    emsesp::EMSESP::system_.uart_init(); // re-enable UART
-
     // if we have had an error already, do nothing
     if (request->_tempObject) {
         return;
     }
+    request->_tempObject = reinterpret_cast<void *>(static_cast<intptr_t>(code)); // stop later chunks from retrying the write
 
-    // send the error code to the client and record the error code in the temp object
-    AsyncWebServerResponse * response = request->beginResponse(code);
-    request->send(response);
+    emsesp::EMSESP::logger().info("Upload error: %d", code);
+    emsesp::EMSESP::system_.uart_init(); // re-enable UART
+    _ota.abort();
 
-    // check for invalid extension and immediately kill the connection, which will throw an error
-    // that is caught by the web code. Unfortunately the http error code is not sent to the client on fast network connections
+    // 406: the client sent a file we won't accept. Close now so the browser fails fast.
+    // Other codes wait for uploadComplete() to send, so we don't destroy the request mid-stream.
     if (code == 406) {
+        request->send(request->beginResponse(code));
+        _response_sent = true;
         request->client()->close();
         _is_firmware   = false;
         _is_filesystem = false;
-        Update.abort();
+        _is_recovery   = false;
     }
 
     // if we aborted a filesystem upload, remount LittleFS so the device keeps working
@@ -275,6 +264,12 @@ void UploadFileService::handleDisconnect() {
     emsesp::EMSESP::logger().info("Upload finished");
     emsesp::EMSESP::system_.uart_init(); // re-enable UART
 
+    _ota.abort();
+    if (_is_recovery || _is_firmware || _is_filesystem) {
+        emsesp::EMSESP::system_.systemStatus(emsesp::SYSTEM_STATUS::SYSTEM_STATUS_NORMAL);
+    }
+
     _is_firmware   = false;
     _is_filesystem = false;
+    _is_recovery   = false;
 }

src/ESP32React/UploadFileService.h

@@ -6,11 +6,9 @@
 #include <Arduino.h>
 #include <ESPAsyncWebServer.h>
 #include <LittleFS.h>
-#include <Update.h>
+#include <OtaUpdater.h>
 #include <WiFi.h>
 
-#include <array>
-
 #define UPLOAD_FILE_PATH "/rest/uploadFile"
 
 #define TEMP_FILENAME_PATH "/pre_load.json" // for uploaded json files, handled by System::check_restore()
@@ -20,11 +18,12 @@ class UploadFileService {
     UploadFileService(AsyncWebServer * server, SecurityManager * securityManager);
 
   private:
-    SecurityManager *    _securityManager;
-    bool                 _is_firmware;
-    bool                 _is_filesystem;
-    bool                 _md5_applied; // true if an MD5 digest was applied to the current firmware upload
-    std::array<char, 33> _md5;
+    SecurityManager * _securityManager;
+    bool              _is_firmware;
+    bool              _is_filesystem;
+    bool              _is_recovery; // firmware for the factory partition, which doesn't change the boot partition
+    bool              _response_sent;
+    OtaUpdater        _ota;
 
     void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final);
     void uploadComplete(AsyncWebServerRequest * request);

src/core/emsesp.cpp

@@ -1897,6 +1897,8 @@ void EMSESP::start() {
     nvs_.begin("ems-esp", false, "nvs");
 #endif
 
+    System::check_crash_loop();
+
     // set valid GPIOs list based on ESP32 chip/platform type
     system_.set_valid_system_gpios();
 

src/core/emsesp.h

@@ -116,6 +116,8 @@ class EMSESPShell;
 class Shower;
 
 static constexpr const char * EMSESP_NVS_BOOT_NEW_FIRMWARE = "fresh_firmware"; // max 15 characters
+// why EMS-ESP switched to the recovery firmware, read and removed by the recovery app (recovery/main.cpp)
+static constexpr const char * EMSESP_NVS_RECOVERY_REASON = "rec_reason";
 
 class EMSESP {
   public:

src/core/main.cpp

@@ -18,12 +18,32 @@
 
 #include "emsesp.h"
 
+#ifdef EMSESP_CRASH_TEST
+#include <esp_ota_ops.h>
+#endif
+
 using namespace emsesp;
 
 static EMSESP application; // the main application
 
+#ifdef EMSESP_CRASH_TEST
+// test builds for the bootloader rollback and the recovery fallback, see [env:crashtest_s3_16M_P] in platformio.ini
+static void crash_test() {
+    esp_ota_img_states_t state = ESP_OTA_IMG_UNDEFINED;
+    esp_ota_get_state_partition(esp_ota_get_running_partition(), &state);
+    if (EMSESP_CRASH_TEST == 1 || state == ESP_OTA_IMG_VALID) {
+        Serial.println("Crash test, aborting");
+        Serial.flush();
+        abort();
+    }
+}
+#endif
+
 void setup() {
     application.start();
+#ifdef EMSESP_CRASH_TEST
+    crash_test();
+#endif
 }
 
 void loop() {

src/core/system.cpp

@@ -27,6 +27,7 @@
 #include <esp_mac.h>
 #include "esp_efuse.h"
 #include <nvs.h>
+#include <Update.h>
 #endif
 
 #include <map>
@@ -43,10 +44,24 @@
 #define ENABLE_SMTP
 #include <ESP_SSLClient.h>
 #include <ReadyMail.h>
+#include <OtaUpdater.h>
+
+// Keeps new firmware in the bootloader's pending verification state until System::check_firmware_health()
+// confirms it, so a crash or restart before then rolls back to the previous image
+extern "C" bool verifyRollbackLater() {
+    return true;
+}
 #endif
 
 namespace emsesp {
 
+#ifndef EMSESP_STANDALONE
+// survives restarts, panics and watchdog resets but not a power cycle, hence the magic
+static constexpr uint32_t       CRASH_COUNT_MAGIC = 0x43524153;
+RTC_NOINIT_ATTR static uint32_t crash_count_magic_;
+RTC_NOINIT_ATTR static uint32_t crash_count_;
+#endif
+
 // Languages supported. Note: the order is important
 // and must match locale_translations.h and common.h
 #if defined(EMSESP_TEST)
@@ -517,6 +532,10 @@ void System::get_partition_info() {
             }
         }
 
+#ifdef EMSESP_HAS_RECOVERY
+        // the recovery firmware is shown separately
+        is_valid = is_valid && !OtaUpdater::isRecoveryPartition(part);
+#endif
         // get the version from the NVS store, and add to map
         if (is_valid) {
             PartitionInfo p_info;
@@ -547,6 +566,35 @@ void System::get_partition_info() {
 #endif
 }
 
+void System::recovery_installed() {
+#ifdef EMSESP_HAS_RECOVERY
+    // drop the version info of an EMS-ESP firmware that was in the factory partition before
+    const esp_partition_t * factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr);
+    if (factory != nullptr) {
+        char c[20];
+        snprintf(c, sizeof(c), "d_%s", factory->label);
+        if (EMSESP::nvs_.isKey(factory->label)) {
+            EMSESP::nvs_.remove(factory->label);
+        }
+        if (EMSESP::nvs_.isKey(c)) {
+            EMSESP::nvs_.remove(c);
+        }
+        partition_info_.erase(factory->label); // on the AsyncTCP task, same as the web handlers reading it
+    }
+    LOG_INFO("Recovery firmware v%s installed", recovery_version().c_str());
+#endif
+}
+
+std::string System::recovery_version() const {
+#ifdef EMSESP_HAS_RECOVERY
+    esp_app_desc_t desc;
+    if (OtaUpdater::isRecoveryPartition(esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr), &desc)) {
+        return desc.version;
+    }
+#endif
+    return "";
+}
+
 // set install time/date for the current partition, in UTC
 // assumes NTP is connected and working
 void System::set_partition_install_date() {
@@ -595,6 +643,12 @@ bool System::set_partition([[maybe_unused]] const char * partitionname) {
         return false;
     }
 
+#ifdef EMSESP_HAS_RECOVERY
+    if (partition->subtype == ESP_PARTITION_SUBTYPE_APP_FACTORY) {
+        EMSESP::nvs_.putString(EMSESP_NVS_RECOVERY_REASON, "request");
+    }
+#endif
+
     // initiate the restart
     EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_RESTART_REQUESTED);
     return true;
@@ -610,7 +664,13 @@ void System::system_restart(const char * partitionname) {
         // Factory partition - label will be "factory"
         const esp_partition_t * partition = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, NULL);
         if (partition && !strcmp(partition->label, partitionname)) {
+#ifdef EMSESP_HAS_RECOVERY
+            if (esp_ota_set_boot_partition(partition) == ESP_OK) {
+                EMSESP::nvs_.putString(EMSESP_NVS_RECOVERY_REASON, "request");
+            }
+#else
             esp_ota_set_boot_partition(partition);
+#endif
         } else
             // try and find the partition by name
             if (strcmp(esp_ota_get_running_partition()->label, partitionname)) {
@@ -762,11 +822,74 @@ void System::store_settings(WebSettings & settings) {
     auto_fw_check_  = settings.auto_fw_check;
 }
 
+// counts consecutive crashes and falls back to the recovery firmware in the factory partition (16MB boards)
+void System::check_crash_loop() {
+#ifndef EMSESP_STANDALONE
+    if (crash_count_magic_ != CRASH_COUNT_MAGIC) {
+        crash_count_magic_ = CRASH_COUNT_MAGIC;
+        crash_count_       = 0;
+    }
+
+    switch (esp_reset_reason()) {
+    case ESP_RST_PANIC:
+    case ESP_RST_INT_WDT:
+    case ESP_RST_TASK_WDT:
+    case ESP_RST_WDT:
+        crash_count_++;
+        break;
+    default:
+        crash_count_ = 0;
+        return;
+    }
+
+#ifdef EMSESP_HAS_RECOVERY
+    if (crash_count_ < CRASH_LOOP_LIMIT) {
+        return;
+    }
+
+    const esp_partition_t * factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr);
+    if (factory == nullptr || factory == esp_ota_get_running_partition()) {
+        return;
+    }
+    // fails if the factory partition doesn't hold a valid image, then keep trying the current one
+    if (esp_ota_set_boot_partition(factory) == ESP_OK) {
+        crash_count_ = 0;
+        EMSESP::nvs_.putString(EMSESP_NVS_RECOVERY_REASON, "crash");
+        Serial.printf("Crashed %d times in a row, restarting into the %s partition\n", CRASH_LOOP_LIMIT, factory->label);
+        Serial.flush();
+        esp_restart();
+    }
+#endif
+#endif
+}
+
+// once the firmware has run long enough, confirm it to the bootloader so it isn't rolled back
+void System::check_firmware_health() {
+#ifndef EMSESP_STANDALONE
+    if (firmware_healthy_ || uuid::get_uptime_sec() < FIRMWARE_HEALTHY_UPTIME) {
+        return;
+    }
+    firmware_healthy_ = true;
+    crash_count_      = 0;
+    if (OtaUpdater::confirmRunningApp()) {
+        LOG_INFO("New firmware confirmed as working");
+    }
+#endif
+}
+
 // Starts up core services
 void System::start() {
     get_partition_info(); // get the partition info
 
 #ifndef EMSESP_STANDALONE
+    if (crash_count_) {
+        LOG_WARNING("Restarted after a crash (%u in a row)", crash_count_);
+    }
+    const esp_partition_t * invalid = esp_ota_get_last_invalid_partition();
+    if (invalid != nullptr) {
+        LOG_WARNING("Firmware in partition %s failed to start and was rolled back", invalid->label);
+    }
+
     // disable bluetooth module
     // periph_module_disable(PERIPH_BT_MODULE);
     if (low_clock_) {
@@ -836,23 +959,30 @@ void System::button_OnDblClick(PButton & b) {
 #endif
 }
 
-// button long press
+// button long press, held for BUTTON_LongPressDelay (3000 ms, 3 seconds)
+// restart EMS-ESP
 void System::button_OnLongPress(PButton & b) {
-    LOG_NOTICE("Button pressed - long press - restart EMS-ESP");
-    EMSESP::system_.system_restart("boot");
-}
-
-// button indefinite press
-void System::button_OnVLongPress(PButton & b) {
     if (EMSESP::system_.disable_reset()) {
-        LOG_NOTICE("Factory reset disabled");
+        LOG_NOTICE("reset disabled, can't boot");
         return;
     }
-    LOG_NOTICE("Button pressed - very long press - perform factory reset");
-    EMSESP::led_.start_led_fast_flash(5); // Start LED flash timer for 5 seconds
+    LOG_NOTICE("Button pressed - long press - restart EMS-ESP");
+    EMSESP::system_.system_restart();
 }
 
-// push button
+// button indefinite press, held for BUTTON_VLongPressDelay (6000 ms, 6 seconds)
+// restart EMS_ESP into the Recovery boot menu, where you can optionally do a factory reset
+void System::button_OnVLongPress(PButton & b) {
+    if (EMSESP::system_.disable_reset()) {
+        LOG_NOTICE("reset disabled, can't boot");
+        return;
+    }
+    LOG_NOTICE("Button pressed - very long press - perform factory reset in Recovery Mode");
+    EMSESP::system_.system_restart("boot");
+    // EMSESP::led_.start_led_fast_flash(5); // Start LED flash timer for 5 seconds
+}
+
+// push button, held for BUTTON_Debounce (200 ms)
 void System::button_init() {
 #ifndef EMSESP_STANDALONE
     if (!myPButton_.init(pbutton_gpio_, HIGH)) {
@@ -1028,6 +1158,8 @@ void System::system_check() {
     if (!last_system_check_ || ((uint32_t)(current_uptime - last_system_check_) >= SYSTEM_CHECK_FREQUENCY)) {
         last_system_check_ = current_uptime;
 
+        check_firmware_health();
+
 #ifndef EMSESP_STANDALONE
 #if CONFIG_IDF_TARGET_ESP32S3 || CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S2
         temperature_sensor_get_celsius(temperature_handle_, &temperature_);
@@ -1208,6 +1340,16 @@ void System::show_system(uuid::console::Shell & shell) {
     shell.printfln(" [total %d]", available.size());
     // List all partitions and their version info
     shell.println(" Partitions:");
+#ifndef EMSESP_STANDALONE
+    const esp_partition_t * running       = esp_ota_get_running_partition();
+    const esp_partition_t * boot          = esp_ota_get_boot_partition();
+    const char *            running_label = running ? running->label : "";
+    const char *            boot_label    = boot ? boot->label : "";
+#else
+    const char * running_label = "";
+    const char * boot_label    = "";
+#endif
+    bool boot_listed = false;
     for (const auto & partition : partition_info_) {
         if (partition.second.version.empty()) {
             continue; // no version, empty string
@@ -1219,13 +1361,42 @@ void System::show_system(uuid::console::Shell & shell) {
             strftime(time_string, sizeof(time_string), "%FT%T", localtime(&d));
             installed = std::string(", installed on ") + time_string;
         }
-        shell.printfln("  %s: v%s (%d KB%s) %s",
-                       partition.first.c_str(),
-                       partition.second.version.c_str(),
-                       partition.second.size,
-                       installed.c_str(),
-                       (strcmp(esp_ota_get_running_partition()->label, partition.first.c_str()) == 0) ? "** active **" : "");
+        const bool is_running = strcmp(running_label, partition.first.c_str()) == 0;
+        const bool is_boot    = strcmp(boot_label, partition.first.c_str()) == 0;
+        if (is_boot) {
+            boot_listed = true;
+        }
+        const char * mark = is_running && is_boot ? "** active, boot **" : is_running ? "** active **" : is_boot ? "** boot **" : "";
+        shell.printfln("  %s: v%s (%d KB%s) %s", partition.first.c_str(), partition.second.version.c_str(), partition.second.size, installed.c_str(), mark);
     }
+#ifndef EMSESP_STANDALONE
+#ifdef EMSESP_HAS_RECOVERY
+    // 16MB boards: factory ("boot") holds the recovery app and is omitted from partition_info_
+    {
+        const esp_partition_t * factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr);
+        esp_app_desc_t          desc;
+        if (OtaUpdater::isRecoveryPartition(factory, &desc)) {
+            const bool is_running = running && factory == running;
+            const bool is_boot    = boot && factory == boot;
+            if (is_boot) {
+                boot_listed = true;
+            }
+            int                  size_kb  = 0;
+            esp_image_metadata_t meta     = {};
+            esp_partition_pos_t  part_pos = {.offset = factory->address, .size = factory->size};
+            if (esp_image_verify(ESP_IMAGE_VERIFY_SILENT, &part_pos, &meta) == ESP_OK) {
+                size_kb = static_cast<int>(meta.image_len / 1024);
+            }
+            const char * mark = is_running && is_boot ? "** active, boot **" : is_running ? "** active **" : is_boot ? "** boot **" : "";
+            shell.printfln("  %s: Recovery v%s (%d KB) %s", factory->label, desc.version, size_kb, mark);
+        }
+    }
+#endif
+    // recovery/factory is omitted from partition_info_, still show it if it's the boot target
+    if (!boot_listed && boot_label[0] != '\0') {
+        shell.printfln("  %s: ** boot **", boot_label);
+    }
+#endif
 // List all NVS values
 #ifndef EMSESP_STANDALONE
     shell.println(" NVS values:");
@@ -3346,6 +3517,8 @@ bool System::uploadFirmwareURL(const char * url) {
         return false; // error
     }
 
+    OtaUpdater::confirmRunningApp();
+
     // check we have enough space for the upload in the ota partition
     if (!Update.begin(firmware_size)) {
         LOG_ERROR("Firmware upload failed - no space");
@@ -3768,9 +3941,9 @@ void System::remove_optional_gpio(uint8_t pin) {
 // set unused gpios to default state input high-Z
 void System::reset_unused_gpios() {
 #if CONFIG_IDF_TARGET_ESP32
-constexpr uint8_t tx0 = 1; // don't change tx0 pin
+    constexpr uint8_t tx0 = 1; // don't change tx0 pin
 #else
-constexpr uint8_t tx0 = 255; // no valid pin, chips have native USB, tx0 not connected to transceiver
+    constexpr uint8_t tx0 = 255; // no valid pin, chips have native USB, tx0 not connected to transceiver
 #endif
     for (const auto & pin : valid_system_gpios_) {
         auto it = std::find_if(used_gpios_.begin(), used_gpios_.end(), [pin](const GpioUsage & usage) { return usage.pin == pin; });

src/core/system.h

@@ -81,6 +81,11 @@ class System {
     void start();
     bool loop(); // returns true if the LED flash is active
 
+    static void check_crash_loop(); // call at boot once NVS is open, may restart into the factory partition
+
+    void        recovery_installed();     // called on the AsyncTCP task after the recovery firmware was written
+    std::string recovery_version() const; // empty if there's no recovery firmware
+
     // commands
     static bool command_read(const char * value, const int8_t id);
     static bool command_send(const char * value, const int8_t id);
@@ -370,7 +375,8 @@ class System {
     static constexpr uint8_t HEALTHCHECK_NO_NETWORK = (1 << 1); // 2
     static constexpr uint8_t HEALTHCHECK_RESET      = (1 << 7); // 128
 
-    // smallest .bin accepted as a firmware image, for both URL and web uploads - 1.6 MB
+    // smallest .bin accepted as a main firmware image, for both URL and web uploads - 1.6 MB
+    // recovery firmware uses OtaUpdater::MIN_RECOVERY_FIRMWARE_SIZE when the filename contains "recovery"
     static constexpr size_t MIN_FIRMWARE_SIZE = 1677721;
 
   private:
@@ -385,6 +391,14 @@ class System {
 
     void set_partition_install_date();
 
+    // firmware is confirmed to the bootloader and the crash counter cleared once it has been up this long
+    static constexpr uint32_t FIRMWARE_HEALTHY_UPTIME = 120; // seconds
+    // consecutive crashes before falling back to the recovery firmware in the factory partition
+    static constexpr uint8_t CRASH_LOOP_LIMIT = 5;
+
+    bool firmware_healthy_ = false;
+    void check_firmware_health();
+
     // button
     static PButton            myPButton_; // PButton instance
     static void               button_OnClick(PButton & b);
@@ -395,8 +409,8 @@ class System {
     static constexpr uint32_t BUTTON_DblClickDelay = 250; // Max period between clicks for a double click event (in ms)
 
     // button press delays
-    static constexpr uint32_t BUTTON_LongPressDelay  = 3000; // Hold period for a long press event (in ms) - ~3 seconds
-    static constexpr uint32_t BUTTON_VLongPressDelay = 9500; // Hold period for a very long press event (in ms) - !10 seconds
+    static constexpr uint32_t BUTTON_LongPressDelay  = 3000; // Hold period for a long press event (in ms) - 3 seconds
+    static constexpr uint32_t BUTTON_VLongPressDelay = 9500; // Hold period for a very long press event (in ms) - 9.5 seconds
 
     // healthcheck
 #ifdef EMSESP_PINGTEST

src/emsesp_version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.9.0-dev.16"
+#define EMSESP_APP_VERSION "3.9.0-dev.17"

src/web/WebStatusService.cpp

@@ -134,10 +134,12 @@ void WebStatusService::systemStatus(AsyncWebServerRequest * request) {
     root["temperature"] = (int)Helpers::transformNumFloat(EMSESP::system_.temperature(), 0, EMSESP::system_.fahrenheit() ? 2 : 0); // only 2 decimal places
 #endif
 
-    // check for a factory partition first
+    // factory partition is only on 16MB boards; recovery_version is empty until a recovery .bin is installed
     const esp_partition_t * partition = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr);
-    root["has_loader"]                = partition != NULL && partition != esp_ota_get_running_partition();
-    partition                         = esp_ota_get_next_update_partition(nullptr);
+    if (partition != NULL) {
+        root["recovery_version"] = EMSESP::system_.recovery_version();
+    }
+    partition = esp_ota_get_next_update_partition(nullptr);
     if (partition) {
         uint64_t buffer;
         esp_partition_read(partition, 0, &buffer, 8);