diff --git a/CHANGELOG_LATEST.md b/CHANGELOG_LATEST.md index e2caf3112..440123db4 100644 --- a/CHANGELOG_LATEST.md +++ b/CHANGELOG_LATEST.md @@ -26,8 +26,8 @@ This release is based on the latest Espressif/Arduino core version 3. It brings - the Version page shows the installed Recovery version on 16MB boards - crash test builds to try out the rollback and recovery, see `[env:crashtest_s3_16M_P]` in `platformio.ini` - `s_16M_P_custom` PlatformIO env that HybridCompiles Arduino/IDF with a trimmed sdkconfig (no Bluetooth, PPP/modem, IP forward/NAPT, ULP, SPI Ethernet, GDBSTUB, FreeRTOS trace, mDNS browse/CLI) and places the UART ISR in IRAM -- 16MB builds use EMS-ESP's own Arduino/IDF libs from [esp32-arduino-lib-builder](https://github.com/emsesp/esp32-arduino-lib-builder) (`0910-1604-5.5.5`) instead of Tasmota's prebuilt cores -- optional `ccache` wrapping when it is installed (`scripts/enable_ccache.py`) +- 16MB builds use EMS-ESP's own Arduino/IDF libs from [esp32-arduino-lib-builder](https://github.com/emsesp/esp32-arduino-lib-builder) instead of Tasmota's prebuilt cores +- optional `ccache` wrapping when it is installed (`scripts/enable_ccache.py`) to speed up CI ## Fixed diff --git a/interface/package.json b/interface/package.json index 56e2d985d..3ef0a90b0 100644 --- a/interface/package.json +++ b/interface/package.json @@ -52,7 +52,7 @@ "eslint-config-prettier": "^10.1.8", "etag": "^1.8.1", "mime-types": "^3.0.2", - "prettier": "^3.9.9", + "prettier": "^3.9.10", "rollup-plugin-visualizer": "^7.1.1", "terser": "^5.51.2", "typescript": "^6.0.3", diff --git a/interface/pnpm-lock.yaml b/interface/pnpm-lock.yaml index 1c23a059e..f4bbfb0cb 100644 --- a/interface/pnpm-lock.yaml +++ b/interface/pnpm-lock.yaml @@ -217,7 +217,7 @@ importers: version: 2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.4(@types/node@26.6.5)(terser@5.51.2)) '@trivago/prettier-plugin-sort-imports': specifier: ^6.0.2 - version: 6.0.2(prettier@3.9.9)(supports-color@10.2.2) + version: 6.0.2(prettier@3.9.10)(supports-color@10.2.2) '@types/node': specifier: ^26.6.5 version: 26.6.5 @@ -243,8 +243,8 @@ importers: specifier: ^3.0.2 version: 3.0.2 prettier: - specifier: ^3.9.9 - version: 3.9.9 + specifier: ^3.9.10 + version: 3.9.10 rollup-plugin-visualizer: specifier: ^7.1.1 version: 7.1.1(rolldown@1.2.13) @@ -2554,8 +2554,8 @@ packages: resolution: {integrity: sha512-ravE6m9Atw9Z/jjttRUZ+clIXogdghyZAuWJ3qEzjT+jI/dL1ifAqhZeC5VHzQp1MSt1+jxKkFNemj/iO7tVUA==} engines: {node: '>=4'} - prettier@3.9.9: - resolution: {integrity: sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==} + prettier@3.9.10: + resolution: {integrity: sha512-skGHTH1Yb9QvaTC4hWnRTzEYtCpApF0UMBPz+t/0ZPzP8ihefo/btKd+QA7pUE0fEwA1SbS/NmhyEcstdXOIdA==} engines: {node: '>=14'} hasBin: true @@ -3708,7 +3708,7 @@ snapshots: react-virtualized-auto-sizer: 1.0.26(react-dom@19.3.0(react@19.3.0))(react@19.3.0) react-window: 1.8.11(react-dom@19.3.0(react@19.3.0))(react@19.3.0) - '@trivago/prettier-plugin-sort-imports@6.0.2(prettier@3.9.9)(supports-color@10.2.2)': + '@trivago/prettier-plugin-sort-imports@6.0.2(prettier@3.9.10)(supports-color@10.2.2)': dependencies: '@babel/generator': 7.29.8 '@babel/parser': 7.29.9 @@ -3718,7 +3718,7 @@ snapshots: lodash-es: 4.18.1 minimatch: 9.0.9 parse-imports-exports: 0.2.4 - prettier: 3.9.9 + prettier: 3.9.10 transitivePeerDependencies: - supports-color @@ -5514,7 +5514,7 @@ snapshots: prepend-http@2.0.0: {} - prettier@3.9.9: {} + prettier@3.9.10: {} process-nextick-args@2.0.1: {} diff --git a/interface/src/app/settings/Version.tsx b/interface/src/app/settings/Version.tsx index 1b27d791a..582d24e7e 100644 --- a/interface/src/app/settings/Version.tsx +++ b/interface/src/app/settings/Version.tsx @@ -3,11 +3,14 @@ import { Link } from 'react-router'; import CancelIcon from '@mui/icons-material/Cancel'; import CloseIcon from '@mui/icons-material/Close'; +import DescriptionIcon from '@mui/icons-material/Description'; import CheckIcon from '@mui/icons-material/Done'; import DownloadIcon from '@mui/icons-material/GetApp'; import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; -import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import RestartAltIcon from '@mui/icons-material/RestartAlt'; import SettingsBackupRestoreIcon from '@mui/icons-material/SettingsBackupRestore'; +import SwapHorizIcon from '@mui/icons-material/SwapHoriz'; import WarningIcon from '@mui/icons-material/Warning'; import { Box, @@ -51,6 +54,8 @@ const STABLE_RELNOTES_URL = const DEV_URL = 'https://github.com/emsesp/EMS-ESP32/releases/download/latest/'; const DEV_RELNOTES_URL = 'https://github.com/emsesp/EMS-ESP32/blob/dev/CHANGELOG_LATEST.md'; +const RELEASES_API_URL = + 'https://api.github.com/repos/emsesp/EMS-ESP32/releases/tags/'; const MAX_VERSION_POLLS = 5; // approx 15 seconds of waiting // Types for better type safety @@ -75,6 +80,96 @@ interface VersionData { disable_reset: boolean; } +interface RecoveryRelease { + url: string; + version: string; +} + +// finds the recovery firmware for this board in the GitHub releases, trying each release tag in turn. +// The version is in the filename, e.g. EMS-ESP-Recovery-1_0_0-ESP32S3-16MB+.bin is 1.0.0 +const findRecovery = async ( + tags: string[], + platform: string +): Promise => { + const prefix = 'ems-esp-recovery-'; + const suffix = `-${platform}.bin`.toLowerCase(); + for (const tag of tags) { + try { + const response = await fetch(`${RELEASES_API_URL}${tag}`); + if (!response.ok) { + continue; + } + const release = (await response.json()) as { + assets?: { name: string; browser_download_url: string }[]; + }; + const asset = release.assets?.find((a) => { + const name = a.name.toLowerCase(); + return name.startsWith(prefix) && name.endsWith(suffix); + }); + if (asset) { + return { + url: asset.browser_download_url, + version: asset.name + .slice(prefix.length, asset.name.length - suffix.length) + .replaceAll('_', '.') + }; + } + } catch { + // try the next one + } + } + return undefined; +}; + +// the GitHub API only allows 60 requests an hour, so only look once per page load for each board and channel +const recoveryLookups = new Map>(); +const findRecoveryCached = (tags: string[], platform: string) => { + const key = `${tags.join(',')}|${platform}`; + let lookup = recoveryLookups.get(key); + if (!lookup) { + lookup = findRecovery(tags, platform); + recoveryLookups.set(key, lookup); + void lookup.then((release) => { + if (!release) { + recoveryLookups.delete(key); // try again next time + } + }); + } + return lookup; +}; + +// true if version a is newer than b, like 1.0.1 and 1.0.0. A prerelease, like 1.1.0-dev.1, is older than the release +const isNewerVersion = (a: string, b: string) => { + const parse = (v: string) => { + const dash = v.indexOf('-'); + const numbers = (text: string) => + text.split('.').map((n) => Number.parseInt(n, 10) || 0); + return { + main: numbers(dash < 0 ? v : v.slice(0, dash)), + pre: dash < 0 ? [] : numbers(v.slice(dash + 1)), + isPre: dash >= 0 + }; + }; + const x = parse(a); + const y = parse(b); + for (let i = 0; i < Math.max(x.main.length, y.main.length); i++) { + const diff = (x.main[i] ?? 0) - (y.main[i] ?? 0); + if (diff !== 0) { + return diff > 0; + } + } + if (x.isPre !== y.isPre) { + return !x.isPre; + } + for (let i = 0; i < Math.max(x.pre.length, y.pre.length); i++) { + const diff = (x.pre[i] ?? 0) - (y.pre[i] ?? 0); + if (diff !== 0) { + return diff > 0; + } + } + return false; +}; + // Memoized components for better performance const VersionInfoDialog = memo( ({ @@ -235,6 +330,7 @@ const VersionInfoDialog = memo( {!isPartition && ( )} - @@ -372,9 +473,7 @@ const InstallPartitionDialog = memo( return ( - - {isRecovery ? LL.RECOVERY() : `${LL.INSTALL()} ${LL.STORED_VERSIONS()}`} - + {isRecovery ? LL.RECOVERY_APP() : LL.INSTALL()} {isRecovery @@ -394,7 +493,7 @@ const InstallPartitionDialog = memo( + )} + {data.recovery_version && me.admin && ( + + )} + + + )} {internetLive ? ( @@ -749,96 +958,6 @@ const Version = () => { alignItems: 'center' }} > - {data.developer_mode && ( - <> - {otherPartitions.length > 0 && ( - <> - - {LL.STORED_VERSIONS()} - - - {otherPartitions.map((partition) => ( - - - {partition.version} - - - setPartitionVersionInfo(partition.partition) - } - aria-label={LL.FIRMWARE_VERSION_INFO()} - sx={{ p: 0.5 }} - > - - - - - ))} - - - )} - {data.recovery_version !== undefined && ( - <> - {LL.RECOVERY()} - - - {data.recovery_version || LL.RECOVERY_NOT_INSTALLED()} - - {data.recovery_version && me.admin && ( - - )} - - - - )} - - )} {LL.STABLE()} { {showButtons(true)} + + {data.recovery_version && latestRecovery && ( + <> + {LL.RECOVERY_APP()} + + + {latestRecovery.version} + + {isNewerVersion( + latestRecovery.version, + data.recovery_version + ) ? ( + me.admin && ( + + ) + ) : ( + <> + + + {LL.LATEST_VERSION(LL.RECOVERY_APP())} + + + )} + + + )} ) : ( @@ -978,7 +1143,7 @@ const Version = () => { {LL.CANCEL()} )} + + setInfoPartition(undefined)} + > + Firmware Version Info + + {infoPartition && ( + + + + + + + + +
+ )} + + + + + ); }; diff --git a/mock-api/package.json b/mock-api/package.json index 236bcf618..d60acebc1 100644 --- a/mock-api/package.json +++ b/mock-api/package.json @@ -15,7 +15,7 @@ }, "devDependencies": { "@trivago/prettier-plugin-sort-imports": "^6.0.2", - "prettier": "^3.9.9" + "prettier": "^3.9.10" }, "packageManager": "pnpm@12.11.2" } diff --git a/mock-api/pnpm-lock.yaml b/mock-api/pnpm-lock.yaml index c36db7f17..ce51c63cb 100644 --- a/mock-api/pnpm-lock.yaml +++ b/mock-api/pnpm-lock.yaml @@ -175,10 +175,10 @@ importers: devDependencies: '@trivago/prettier-plugin-sort-imports': specifier: ^6.0.2 - version: 6.0.2(prettier@3.9.9) + version: 6.0.2(prettier@3.9.10) prettier: - specifier: ^3.9.9 - version: 3.9.9 + specifier: ^3.9.10 + version: 3.9.10 packages: @@ -303,8 +303,8 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - prettier@3.9.9: - resolution: {integrity: sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==} + prettier@3.9.10: + resolution: {integrity: sha512-skGHTH1Yb9QvaTC4hWnRTzEYtCpApF0UMBPz+t/0ZPzP8ihefo/btKd+QA7pUE0fEwA1SbS/NmhyEcstdXOIdA==} engines: {node: '>=14'} hasBin: true @@ -373,7 +373,7 @@ snapshots: '@msgpack/msgpack@3.1.3': {} - '@trivago/prettier-plugin-sort-imports@6.0.2(prettier@3.9.9)': + '@trivago/prettier-plugin-sort-imports@6.0.2(prettier@3.9.10)': dependencies: '@babel/generator': 7.29.8 '@babel/parser': 7.29.9 @@ -383,7 +383,7 @@ snapshots: lodash-es: 4.18.1 minimatch: 9.0.9 parse-imports-exports: 0.2.4 - prettier: 3.9.9 + prettier: 3.9.10 transitivePeerDependencies: - supports-color @@ -421,4 +421,4 @@ snapshots: picocolors@1.1.1: {} - prettier@3.9.9: {} + prettier@3.9.10: {} diff --git a/mock-api/restServer.ts b/mock-api/restServer.ts index 29ab5d875..aa2470b60 100644 --- a/mock-api/restServer.ts +++ b/mock-api/restServer.ts @@ -116,8 +116,8 @@ let system_status = { psram: true, psram_size: 8189, free_psram: 8166, - // recovery_version: '', // upload an EMS-ESP-Recovery-*.bin to set it - recovery_version: '1.0.0', + recovery_version: '', // upload an EMS-ESP-Recovery-*.bin to set it + // recovery_version: '1.0.0', has_partition: true, partitions: [ { @@ -5447,6 +5447,10 @@ router } else if (action === 'uploadURL') { // upload URL console.log('upload File from URL', content.param); + if (String(content.param).toLowerCase().includes('recovery')) { + // the recovery app is installed without a restart + system_status.recovery_version = '1.0.0'; + } return status(200); } else if (action === 'resetMQTT') { // reset MQTT diff --git a/pio_local.ini_example b/pio_local.ini_example index c1bd4f59d..6a227502d 100644 --- a/pio_local.ini_example +++ b/pio_local.ini_example @@ -2,36 +2,46 @@ [common] ; custom build flags to use in my_build_flags -; -DEMSESP_UART_DEBUG ; debugging UART -; -DEMSESP_DEBUG ; enables DEBUG to the log. Will generate a lot of extra traffic on Console and Syslog -; -DEMSESP_DEFAULT_BOARD_PROFILE=\"NODEMCU\" ; hard code the default board name -; -DEMSESP_DEBUG_SENSOR ; additional debug for the sensors -; -DCORE_DEBUG_LEVEL=5 ; verbose core level debugging 5=verbose, 4=debug, 3=info -; -DEMSESP_TEST ; enable the tests. EN language only -; -DEMSESP_DE_ONLY ; only DE translated entity names -; -DEMSESP_EN_ONLY ; only EN translated entity names -; -DEMSESP_PINGTEST ; send log message every 1/2 second +; -D EMSESP_UART_DEBUG ; debugging UART +; -D EMSESP_DEBUG ; enables DEBUG to the log. Will generate a lot of extra traffic on Console and Syslog +; -D EMSESP_DEFAULT_BOARD_PROFILE=\"NODEMCU\" ; hard code the default board name +; -D EMSESP_DEBUG_SENSOR ; additional debug for the sensors +; -D CORE_DEBUG_LEVEL=5 ; verbose core level debugging 5=verbose, 4=debug, 3=info +; -D EMSESP_TEST ; enable the tests. EN language only +; -D EMSESP_DE_ONLY ; only DE translated entity names +; -D EMSESP_EN_ONLY ; only EN translated entity names +; -D EMSESP_PINGTEST ; send log message every 1/2 second +; -D EMSESP_GATEWAY_TX_HOLD ; hold Tx while a K30RF/KM200 gateway is syncing, see issue #3094 +; -D EMSESP_TX_PRE_DELAY=1000 ; wait this many us before each Tx so we don't overlap the master's break, see issue #3094 ; my_build_flags = -DEMSESP_TEST -DEMSESP_DEBUG ; my_build_flags = -DEMSESP_DEBUG -DEMSESP_TEST -DEMSESP_PINGTEST ; my_build_flags = -DEMSESP_DEBUG -DCORE_DEBUG_LEVEL=5 ; my_build_flags = -DEMSESP_EN_ONLY -DEMSESP_TEST -DCORE_DEBUG_LEVEL=5 -DARDUINO_USB_CDC_ON_BOOT=1 -my_build_flags = +; my_build_flags = -DEMSESP_EN_ONLY +; my_build_flags = -D EMSESP_GATEWAY_TX_HOLD -D EMSESP_TX_PRE_DELAY=1000 +my_build_flags = -DEMSESP_DEBUG +; my_build_flags = [platformio] -default_envs = s_16M_P ; BBQKees E32V2 -; default_envs = build-webUI ; build the web interface only -; default_envs = s3_16M_P ; BBQKees S3 -; default_envs = s_4M ; BBQKees older S32, 4MB no psram -; default_envs = s_16M ; BBQKees newer S32 V2, 16MB no psram -; default_envs = c6 ; XIAO ESP32C -; default_envs = debug +default_envs = s_16M_P, recovery_s_16M_P, build-recovery, build-webUI +; default_envs = s_16M_P ; BBQKees E32V2 +; default_envs = s_16M_P_custom ; BBQKees E32V2 with custom Arduino/IDF libs +; default_envs = recovery_s_16M_P ; recovery firmware +; default_envs = crashtest_s_16M_P ; crash test +; default_envs = build-webUI ; build the main ems-esp webUI only +; default_envs = build-recovery ; build the recovery WebUI only +; default_envs = s3_16M_P ; BBQKees S3 firmware +; default_envs = s_4M ; firmware forBBQKees older S32, 4MB no psram +; default_envs = s_16M ; firmware for BBQKees newer S32 V2, 16MB no psram +; default_envs = c6 ; firmware for XIAO ESP32C +; default_envs = debug ; see below [env] +; uncomment if you want to upload the firmware via OTA (must have upload_protocol = custom) extra_scripts = - pre:scripts/build_interface.py ; builds the WebUI - scripts/rename_fw.py ; renames the firmware .bin file - comment out if not needed + post:scripts/rename_fw.py ; renames the firmware .bin file scripts/upload.py ; optionally upload the firmware via OTA (must have upload_protocol = custom) ; set the username and password for the admin account (default password is admin) @@ -40,13 +50,33 @@ custom_password = admin ; ** upload settings ** ; for direct USB, which is the default you can set an optional port for example: -; upload_port = /dev/ttyUSB0 or COM3 -; upload_speed = 921600 +; upload_port = /dev/ttyUSB0 +; upload_port = /dev/ttyACM0 + +; CH340G bridge tops out here reliably; drop back to 460800 if uploads fail +; 2000000 is the max speed for the CH340G bridge +upload_speed = 921600 ; for direct via HTTP (OTA) set: ; upload_protocol = custom ; custom_emsesp_ip = or ems-esp.local +upload_protocol = custom + +custom_emsesp_ip = ems-esp.local ; E32V2.2 (test) +; custom_emsesp_ip = ems-espP.local ; E32V2.2 (prod) +; custom_emsesp_ip = ems-esps32.local ; S32, 4MB no psramS32 +; custom_emsesp_ip = ems-espS3.local ; ems-espS3 + +; lib_deps = +; bblanchon/ArduinoJson @ 7.4.3 +; ESP32Async/AsyncTCP @ 3.4.10 +; ESP32Async/ESPAsyncWebServer @ 3.11.0 +; https://github.com/mobizt/ReadyMail.git @ 0.4.0 +; https://github.com/mobizt/ESP_SSLClient.git @ 3.1.3 + ; https://github.com/emsesp/EMS-ESP-Modules.git @ 1.0.8 +; ; file://${PROJECT_DIR}/../modules/EMS-ESP-Modules + ; ** debug settings ** ; to be used with an ESP32-S3 which has an onboard JTAG, connect the OTG USB port to the PC ; if using an external JTAG board like the ESP-PROG set debug_tool = esp-prog, and use zadig to set the correct driver for the USB port @@ -69,4 +99,3 @@ debug_init_break = upload_speed = 2000000 monitor_speed = 115200 extra_scripts = -; debug_build_flags = -Og -g2 -ggdb2 \ No newline at end of file diff --git a/platformio.ini b/platformio.ini index 7a59c57cd..e18746977 100644 --- a/platformio.ini +++ b/platformio.ini @@ -116,6 +116,14 @@ extra_scripts = pre:scripts/build_interface.py ; Exclude all source files so the C code doesn't build build_src_filter = -<*> +; builds the recovery web interface only (recovery/WWWData.h), not the recovery firmware +[env:build-recovery] +platform = native +targets = build +extra_scripts = pre:scripts/build_recovery_interface.py +; Exclude all source files so the C code doesn't build +build_src_filter = -<*> + ; ; Builds for different board types ; If you're building for a single target environment, we recommend creating a pio_local.ini (see example file) diff --git a/recovery/main.cpp b/recovery/main.cpp index 23089223f..c115c4b0d 100644 --- a/recovery/main.cpp +++ b/recovery/main.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include "WWWData.h" // generated by `pnpm build-recovery` in ./interface @@ -37,18 +38,18 @@ #define FACTORY_AP_CHANNEL 1 #endif -static constexpr size_t MIN_FIRMWARE_SIZE = 512 * 1024; -static constexpr uint32_t RESTART_DELAY_MS = 500; -static constexpr uint32_t ETH_WAIT_MS = 15000; // how long Ethernet gets to obtain an address before WiFi is tried +static constexpr size_t MIN_FIRMWARE_SIZE = 512 * 1024; // smallest .bin accepted by the upload, anything smaller is rejected +static constexpr uint32_t RESTART_DELAY_MS = 500; // delay before restarting, so the HTTP response is sent first +static constexpr uint32_t ETH_WAIT_MS = 15000; // how long Ethernet gets to obtain an address before WiFi is tried static constexpr char NVS_NAMESPACE[] = "ems-esp"; static constexpr char NVS_FRESH_FW[] = "fresh_firmware"; // EMSESP_NVS_BOOT_NEW_FIRMWARE in emsesp.h static constexpr char NVS_REASON[] = "rec_reason"; // EMSESP_NVS_RECOVERY_REASON in emsesp.h -#define LOG(...) \ - do { \ - Serial.print("[recovery] "); \ - Serial.printf(__VA_ARGS__); \ - Serial.println(); \ +#define LOG(...) \ + do { \ + Serial.print("[recovery] "); \ + Serial.printf(__VA_ARGS__); \ + Serial.println(); \ } while (0) struct AdminUser { @@ -102,7 +103,7 @@ static std::vector partitions; static uint32_t restart_at = 0; static String session_token; static String ap_ssid; -static String recovery_reason; // set by EMS-ESP when it switched to the recovery +static String recovery_reason; // set by EMS-ESP when it switched to the recovery static const char * eth_state = ""; // "", "disabled", "failed" or "started" static uint32_t wifi_join_at = 0; // when to try WiFi if Ethernet has no address, 0 if not waiting static bool wifi_started = false; @@ -113,6 +114,8 @@ static bool ethConnected(); static int upload_error = 0; static bool upload_done = false; static String upload_target; +static size_t upload_size = 0; // content length, for the progress +static size_t upload_step = 0; // last progress step logged, in tens of percent // ---------------------------------------------------------------------------- // settings (read-only) @@ -256,6 +259,7 @@ static void loadRecoveryReason() { if (nvs.isKey(NVS_REASON)) { recovery_reason = nvs.getString(NVS_REASON); nvs.remove(NVS_REASON); + LOG("EMS-ESP switched to the recovery: %s", recovery_reason.c_str()); } nvs.end(); } @@ -272,8 +276,8 @@ static void loadPartitions() { const esp_partition_t * part = esp_partition_get(it); PartitionInfo info; - info.label = part->label; - info.factory = part->subtype == ESP_PARTITION_SUBTYPE_APP_FACTORY; + info.label = part->label; + info.factory = part->subtype == ESP_PARTITION_SUBTYPE_APP_FACTORY; info.address = part->address; info.size = part->size; info.image_size = 0; @@ -293,6 +297,10 @@ static void loadPartitions() { if (part == running) { info.version = RECOVERY_VERSION; + // EMS-ESP stores the install date when it installs the recovery. Not set if it was flashed over serial + if (has_nvs) { + info.install_date = nvs.getULong(("d_" + info.label).c_str(), 0); + } } else if (info.valid) { // EMS-ESP stores the version and install date of each partition in NVS, keyed by the label if (has_nvs && nvs.isKey(part->label)) { @@ -305,6 +313,17 @@ static void loadPartitions() { } } + LOG("Partition %s: %s%s, %uKB at 0x%x%s%s%s%s", + info.label.c_str(), + info.factory ? "factory, " : "", + info.valid ? "valid" : "empty", + static_cast(info.size / 1024), + static_cast(info.address), + info.version.isEmpty() ? "" : ", v", + info.version.c_str(), + info.ota_state.isEmpty() ? "" : ", ota state ", + info.ota_state.c_str()); + partitions.push_back(info); } esp_partition_iterator_release(it); @@ -353,6 +372,7 @@ static const esp_partition_t * uploadTarget(AsyncWebServerRequest * request) { static void markFreshFirmware(const String & label) { Preferences nvs; if (!openNvs(nvs, false)) { + LOG("Cannot open NVS, EMS-ESP won't know it's a new firmware"); return; } nvs.putBool(NVS_FRESH_FW, true); @@ -381,10 +401,11 @@ static String generateToken() { } static bool isAuthorized(AsyncWebServerRequest * request) { - if (session_token.isEmpty() || !request->hasHeader("Authorization")) { - return false; + const bool ok = !session_token.isEmpty() && request->hasHeader("Authorization") && request->header("Authorization") == "Bearer " + session_token; + if (!ok) { + LOG("Unauthorized request to %s", request->url().c_str()); } - return request->header("Authorization") == "Bearer " + session_token; + return ok; } // ---------------------------------------------------------------------------- @@ -529,6 +550,7 @@ static void handleSetBoot(AsyncWebServerRequest * request, JsonVariant & json) { const String label = json["partition"] | ""; const esp_partition_t * part = findAppPartition(label.c_str()); if (part == nullptr) { + LOG("Cannot boot from %s: no such partition", label.c_str()); request->send(404); return; } @@ -569,6 +591,7 @@ static void handleFactoryReset(AsyncWebServerRequest * request) { return; } + LOG("Factory reset requested"); const bool ok = LittleFS.format(); // the only place the recovery writes to the filesystem LOG("Factory reset %s", ok ? "done" : "failed"); if (!ok) { @@ -580,12 +603,12 @@ static void handleFactoryReset(AsyncWebServerRequest * request) { restart_at = millis() + RESTART_DELAY_MS; } -static void uploadFail(AsyncWebServerRequest * request, int code) { +static void uploadFail(AsyncWebServerRequest * request, int code, const char * reason) { if (upload_error) { return; } upload_error = code; - LOG("Upload error %d", code); + LOG("Upload error %d: %s", code, reason); request->send(code); if (code == 406) { request->client()->close(); @@ -596,10 +619,12 @@ static void handleUpload(AsyncWebServerRequest * request, const String & filenam if (!index) { upload_error = 0; upload_done = false; + upload_size = 0; + upload_step = 0; upload_target.clear(); if (!isAuthorized(request)) { - uploadFail(request, 403); + uploadFail(request, 403, "not signed in"); return; } @@ -611,41 +636,51 @@ static void handleUpload(AsyncWebServerRequest * request, const String & filenam if (ota.setMd5(data, len)) { LOG("MD5 digest received (%s)", ota.md5()); } else { - uploadFail(request, 406); + uploadFail(request, 406, "invalid MD5 digest"); } return; default: // the recovery app only flashes firmware, it never touches the settings ota.clearMd5(); LOG("Unsupported file %s (%u bytes)", filename.c_str(), static_cast(filesize)); - uploadFail(request, 406); + uploadFail(request, 406, "unsupported file"); return; } if (!OtaUpdater::isCompatibleFirmware(data, len)) { - uploadFail(request, 503); + uploadFail(request, 503, "not a firmware image for this chip"); return; } if (OtaUpdater::isRecoveryFirmware(data, len)) { - uploadFail(request, 409); + uploadFail(request, 409, "the recovery firmware can't be installed from the recovery"); return; } const esp_partition_t * target = uploadTarget(request); if (target == nullptr) { - uploadFail(request, 400); + if (request->hasParam("partition")) { + LOG("Invalid upload target %s", request->getParam("partition")->value().c_str()); + } + uploadFail(request, 400, "no valid target partition"); return; } if (!ota.beginFirmware(filesize, target)) { LOG("Cannot write to %s: %s", target->label, ota.errorString()); - uploadFail(request, 507); + uploadFail(request, 507, "cannot start the update"); return; } upload_target = target->label; - LOG("Uploading %s (%uKB) to %s%s", filename.c_str(), static_cast(filesize / 1024), target->label, ota.md5Applied() ? ", MD5 check enabled" : ""); + upload_size = filesize; + LOG("Uploading %s (%uKB) to %s (%s)%s", + filename.c_str(), + static_cast(filesize / 1024), + target->label, + request->hasParam("partition") ? "selected" : "default target", + ota.md5Applied() ? ", MD5 check enabled" : ""); request->onDisconnect([] { if (!upload_done && !upload_target.isEmpty()) { + LOG("Upload aborted at %uKB, the client disconnected", static_cast(ota.progress() / 1024)); ota.abort(); } }); @@ -658,17 +693,27 @@ static void handleUpload(AsyncWebServerRequest * request, const String & filenam if (!ota.write(data, len)) { LOG("Write failed at %u: %s", static_cast(ota.progress()), ota.errorString()); ota.abort(); - uploadFail(request, 500); + uploadFail(request, 500, "write failed"); return; } + // log the progress every 10%, the content length is a little more than the image so it may stop just short of 100% + if (upload_size > 0) { + const size_t step = std::min(ota.progress() * 10 / upload_size, 10); + if (step > upload_step) { + upload_step = step; + LOG("Uploading to %s, %u%%", upload_target.c_str(), static_cast(step * 10)); + } + } + if (final) { // on success this also makes the new partition the boot partition if (!ota.end()) { LOG("Upload failed: %s", ota.errorString()); - uploadFail(request, 500); + uploadFail(request, 500, "image check failed"); return; } + LOG("Upload finished, the image is verified"); upload_done = true; } } @@ -785,6 +830,17 @@ static bool ethConnected() { } static void startNetwork() { + // the reason is logged once per change, since a missing network disconnects again every few seconds + WiFi.onEvent( + [](arduino_event_id_t, arduino_event_info_t info) { + static uint8_t last_reason = 0; + if (info.wifi_sta_disconnected.reason != last_reason) { + last_reason = info.wifi_sta_disconnected.reason; + LOG("WiFi disconnected, reason %d", last_reason); + } + }, + ARDUINO_EVENT_WIFI_STA_DISCONNECTED); + WiFi.persistent(false); WiFi.setHostname(settings.hostname.c_str()); // also the default for the Ethernet interface WiFi.mode(WIFI_AP); @@ -826,6 +882,9 @@ void setup() { Serial.begin(115200); LOG("EMS-ESP Recovery v%s", RECOVERY_VERSION); + const esp_partition_t * boot = esp_ota_get_boot_partition(); + LOG("Reset reason: %s, running %s, next boot %s", resetReasonName(esp_reset_reason()), esp_ota_get_running_partition()->label, boot ? boot->label : "unknown"); + loadSettings(); loadRecoveryReason(); loadPartitions(); @@ -841,6 +900,8 @@ void loop() { was_connected = WiFi.isConnected(); if (was_connected) { LOG("WiFi connected, IP %s", WiFi.localIP().toString().c_str()); + } else { + LOG("WiFi connection lost"); } } diff --git a/scripts/build_interface.py b/scripts/build_interface.py index 0be356f58..7202e1dad 100755 --- a/scripts/build_interface.py +++ b/scripts/build_interface.py @@ -110,6 +110,12 @@ def build_webUI(*args, **kwargs): env.Exit(0) +# The web interface is only built by the "build" target. Skip any other, e.g. `pio run -t upload` also runs this +# environment when it's one of the default_envs, and it has nothing to upload +if "build" not in [t.strip() for t in COMMAND_LINE_TARGETS]: + print("Skipping the web interface build, the target is not 'build'") + env.Exit(0) + # Create custom target that only runs the script and then exits, without continuing with the pio workflow env.AddCustomTarget( name="build", diff --git a/scripts/build_recovery_interface.py b/scripts/build_recovery_interface.py index 23a82bdde..94bd3168b 100644 --- a/scripts/build_recovery_interface.py +++ b/scripts/build_recovery_interface.py @@ -1,4 +1,5 @@ -# Builds the Recovery WebUI into recovery/WWWData.h when it's missing or older than its sources +# Builds the Recovery WebUI into recovery/WWWData.h when it's missing or older than its sources. +# In the build-recovery environment it only builds the WebUI, always. from pathlib import Path import os import shutil @@ -52,7 +53,29 @@ def build(): env.Exit(1) -if is_stale(): +if env.get("PIOENV") == "build-recovery": + # the WebUI is only built by the "build" target. Skip any other, e.g. `pio run -t upload` also runs this environment + # when it's one of the default_envs, and it has nothing to upload + if "build" not in [t.strip() for t in COMMAND_LINE_TARGETS]: + print("Skipping the Recovery WebUI build, the target is not 'build'") + env.Exit(0) + + # `pio run -e build-recovery` only builds the WebUI, always, and exits without building any firmware + def build_recovery_webUI(*args, **kwargs): + print("Building Recovery WebUI...") + build() + print("Recovery WebUI build completed successfully!") + env.Exit(0) + + env.AddCustomTarget( + name="build", + dependencies=None, + actions=[build_recovery_webUI], + title="build recovery web interface", + description="installs pnpm packages and builds the recovery web UI into recovery/WWWData.h", + always_build=True, + ) +elif is_stale(): print("Building Recovery WebUI...") build() else: diff --git a/src/core/system.cpp b/src/core/system.cpp index e1cffb6bb..d880cc9ef 100644 --- a/src/core/system.cpp +++ b/src/core/system.cpp @@ -568,7 +568,8 @@ void System::get_partition_info() { void System::recovery_installed() { #ifdef EMSESP_HAS_RECOVERY - // drop the version info of an EMS-ESP firmware that was in the factory partition before + // drop the version info of an EMS-ESP firmware that was in the factory partition before. The recovery reports its + // own version, but the install date is only known here, as it's not set when flashed over serial const esp_partition_t * factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr); if (factory != nullptr) { char c[20]; @@ -576,8 +577,11 @@ void System::recovery_installed() { if (EMSESP::nvs_.isKey(factory->label)) { EMSESP::nvs_.remove(factory->label); } - if (EMSESP::nvs_.isKey(c)) { - EMSESP::nvs_.remove(c); + const time_t now = time(nullptr); + if (now > 1500000000L) { + EMSESP::nvs_.putULong(c, now); // UTC epoch, read by the recovery app. Only if the clock is set (NTP) + } else if (EMSESP::nvs_.isKey(c)) { + EMSESP::nvs_.remove(c); // clock not set, the date of the previous firmware is wrong } partition_info_.erase(factory->label); // on the AsyncTCP task, same as the web handlers reading it } @@ -585,6 +589,19 @@ void System::recovery_installed() { #endif } +// the partition info is read by the web handlers on the AsyncTCP task, so the update after a recovery firmware +// download (which runs on the main loop) is deferred to be done there, on the next systemStatus request +void System::recovery_installed_deferred() { + recovery_installed_pending_ = true; +} + +void System::recovery_installed_apply() { + if (recovery_installed_pending_) { + recovery_installed_pending_ = false; + recovery_installed(); + } +} + std::string System::recovery_version() const { #ifdef EMSESP_HAS_RECOVERY esp_app_desc_t desc; @@ -3511,21 +3528,44 @@ bool System::uploadFirmwareURL(const char * url) { break; } + // the recovery firmware is recognized by its filename. It's written to the factory partition and + // EMS-ESP keeps running, there is no restart +#ifdef EMSESP_HAS_RECOVERY + String file_name = saved_url.substring(saved_url.lastIndexOf('/') + 1); + file_name.toLowerCase(); + const bool is_recovery = file_name.indexOf("recovery") >= 0; +#else + const bool is_recovery = false; +#endif + OtaUpdater ota; // only used for the recovery firmware + // check we have a valid size - if (firmware_size < (int)MIN_FIRMWARE_SIZE) { + if (firmware_size < (int)(is_recovery ? OtaUpdater::MIN_RECOVERY_FIRMWARE_SIZE : MIN_FIRMWARE_SIZE)) { LOG_ERROR("Firmware upload failed - invalid size"); return false; // error } OtaUpdater::confirmRunningApp(); - // check we have enough space for the upload in the ota partition - if (!Update.begin(firmware_size)) { + if (is_recovery) { +#ifdef EMSESP_HAS_RECOVERY + const esp_partition_t * factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, nullptr); + if (factory == nullptr || factory == esp_ota_get_running_partition()) { + LOG_ERROR("The recovery firmware can't be installed on this board"); + return false; // error + } + if (!ota.beginFirmware(firmware_size, factory, false)) { + LOG_ERROR("Recovery firmware upload failed - %s", ota.errorString()); + return false; // error + } +#endif + } else if (!Update.begin(firmware_size)) { + // check we have enough space for the upload in the ota partition LOG_ERROR("Firmware upload failed - no space"); return false; // error } - LOG_INFO("Firmware uploading (size: %dKB) over %s. Please wait...", firmware_size / 1024, is_https ? "HTTPS" : "HTTP"); + LOG_INFO("%s uploading (size: %dKB) over %s. Please wait...", is_recovery ? "Recovery firmware" : "Firmware", firmware_size / 1024, is_https ? "HTTPS" : "HTTP"); Shell::loop_all(); // flush log buffers so latest messages are shown in console @@ -3543,7 +3583,11 @@ bool System::uploadFirmwareURL(const char * url) { // via the systemStatus action, which runs on the AsyncTCP task while we're blocked here if (EMSESP::system_.systemStatus() < SYSTEM_STATUS::SYSTEM_STATUS_UPLOADING) { LOG_WARNING("Firmware upload cancelled at %u of %d bytes", (unsigned)total_read, firmware_size); - Update.abort(); // release the OTA partition handle so a later attempt can start cleanly + if (is_recovery) { + ota.abort(); + } else { + Update.abort(); // release the OTA partition handle so a later attempt can start cleanly + } ssl_client.stop(); // drop the connection saved_url.clear(); // prevent it from downloading again EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_NORMAL); @@ -3573,6 +3617,9 @@ bool System::uploadFirmwareURL(const char * url) { continue; } LOG_ERROR("Firmware upload failed - read stalled at %u of %d bytes", (unsigned)total_read, firmware_size); + if (is_recovery) { + ota.abort(); + } EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); return false; } @@ -3585,6 +3632,9 @@ bool System::uploadFirmwareURL(const char * url) { size_t n = stream->readBytes(buf, want); if (n == 0) { LOG_ERROR("Firmware upload failed - read returned 0 at %u of %d bytes", (unsigned)total_read, firmware_size); + if (is_recovery) { + ota.abort(); + } EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); return false; } @@ -3594,14 +3644,33 @@ bool System::uploadFirmwareURL(const char * url) { if (!magic_ok) { if (buf[0] != 0xE9) { LOG_ERROR("Firmware upload failed - bad magic byte 0x%02X (expected 0xE9, not an ESP32 firmware image?)", buf[0]); + if (is_recovery) { + ota.abort(); + } + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); + return false; + } + // the recovery firmware must be for this chip, and really be the recovery + if (is_recovery && !(OtaUpdater::isCompatibleFirmware(buf, n) && OtaUpdater::isRecoveryFirmware(buf, n))) { + LOG_ERROR("Recovery firmware upload failed - not a recovery firmware image for this chip"); + if (is_recovery) { + ota.abort(); + } EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); return false; } magic_ok = true; } - if (Update.write(buf, n) != n) { - LOG_ERROR("Firmware upload failed - flash write error at %u of %d bytes: %s", (unsigned)total_read, firmware_size, Update.errorString()); + const bool written = is_recovery ? ota.write(buf, n) : (Update.write(buf, n) == n); + if (!written) { + LOG_ERROR("Firmware upload failed - flash write error at %u of %d bytes: %s", + (unsigned)total_read, + firmware_size, + is_recovery ? ota.errorString() : Update.errorString()); + if (is_recovery) { + ota.abort(); + } EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); return false; } @@ -3618,13 +3687,22 @@ bool System::uploadFirmwareURL(const char * url) { yield(); } - if (!Update.end(true)) { - LOG_ERROR("Firmware upload failed - %s", Update.errorString()); + if (!(is_recovery ? ota.end() : Update.end(true))) { + LOG_ERROR("Firmware upload failed - %s", is_recovery ? ota.errorString() : Update.errorString()); EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_ERROR_UPLOAD); return false; // error } saved_url.clear(); // prevent from downloading again + + if (is_recovery) { + // no restart. The partition info is updated later on the AsyncTCP task, see recovery_installed_apply() + EMSESP::system_.recovery_installed_deferred(); + LOG_INFO("Recovery firmware uploaded successfully"); + EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_NORMAL); + return true; + } + LOG_INFO("Firmware uploaded successfully. Restarting..."); EMSESP::system_.systemStatus(SYSTEM_STATUS::SYSTEM_STATUS_PENDING_RESTART); #endif diff --git a/src/core/system.h b/src/core/system.h index 04da48ec2..af83e3324 100644 --- a/src/core/system.h +++ b/src/core/system.h @@ -84,6 +84,8 @@ class System { static void check_crash_loop(); // call at boot once NVS is open, may restart into the factory partition void recovery_installed(); // called on the AsyncTCP task after the recovery firmware was written + void recovery_installed_deferred(); // called from the main loop after the recovery firmware was downloaded from a URL + void recovery_installed_apply(); // called on the AsyncTCP task, handles the deferred call std::string recovery_version() const; // empty if there's no recovery firmware // commands @@ -391,6 +393,8 @@ class System { void set_partition_install_date(); + volatile bool recovery_installed_pending_ = false; // set by the main loop, handled on the AsyncTCP task + // firmware is confirmed to the bootloader and the crash counter cleared once it has been up this long static constexpr uint32_t FIRMWARE_HEALTHY_UPTIME = 120; // seconds // consecutive crashes before falling back to the recovery firmware in the factory partition diff --git a/src/web/WebStatusService.cpp b/src/web/WebStatusService.cpp index d012d6308..681a965ee 100644 --- a/src/web/WebStatusService.cpp +++ b/src/web/WebStatusService.cpp @@ -45,7 +45,8 @@ WebStatusService::WebStatusService(AsyncWebServer * server, SecurityManager * se // This contains both system & hardware Status to avoid having multiple costly endpoints // This is also used for polling during the SystemMonitor to see if EMS-ESP is alive void WebStatusService::systemStatus(AsyncWebServerRequest * request) { - EMSESP::system_.refreshHeapMem(); // refresh free heap and max alloc heap + EMSESP::system_.refreshHeapMem(); // refresh free heap and max alloc heap + EMSESP::system_.recovery_installed_apply(); // update the partition info if the recovery firmware was just downloaded auto * response = new PsramAsyncJsonResponse(false); JsonObject root = response->getRoot();