From cd5fb061aa6dea24c4ffdc70eb6c720d4ef26418 Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 13:17:01 +0200 Subject: [PATCH 1/9] added comments --- mock-api/mockServer.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mock-api/mockServer.js b/mock-api/mockServer.js index b75a49bf6..0ff53bca4 100644 --- a/mock-api/mockServer.js +++ b/mock-api/mockServer.js @@ -34,9 +34,10 @@ export default () => { server.middlewares.use(async (req, res, next) => { // catch any file uploads if (req.url.startsWith('/rest/uploadFile')) { + // show progress let progress = 0; const file_size = req.headers['content-length']; - + console.log('File size: ' + file_size); req.on('data', async (chunk) => { progress += chunk.length; const percentage = (progress / file_size) * 100; @@ -50,7 +51,7 @@ export default () => { try { [fields, files] = await form.parse(req); } catch (err) { - console.error(err); + console.error('Not json form content'); res.writeHead(err.httpCode || 400, { 'Content-Type': 'text/plain' }); From 47dc7346dc44b7308001d87a07ed99939544a3ed Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 13:17:13 +0200 Subject: [PATCH 2/9] package update --- interface/package.json | 2 +- interface/yarn.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/interface/package.json b/interface/package.json index 252358b2d..3b82bb022 100644 --- a/interface/package.json +++ b/interface/package.json @@ -21,7 +21,7 @@ "lint": "eslint . --fix" }, "dependencies": { - "@alova/adapter-xhr": "2.0.4", + "@alova/adapter-xhr": "2.0.5", "@emotion/react": "^11.13.0", "@emotion/styled": "^11.13.0", "@mui/icons-material": "^5.16.7", diff --git a/interface/yarn.lock b/interface/yarn.lock index 994d6833a..238314ab4 100644 --- a/interface/yarn.lock +++ b/interface/yarn.lock @@ -5,14 +5,14 @@ __metadata: version: 8 cacheKey: 10c0 -"@alova/adapter-xhr@npm:2.0.4": - version: 2.0.4 - resolution: "@alova/adapter-xhr@npm:2.0.4" +"@alova/adapter-xhr@npm:2.0.5": + version: 2.0.5 + resolution: "@alova/adapter-xhr@npm:2.0.5" dependencies: "@alova/shared": "npm:^1.0.4" peerDependencies: - alova: ^3.0.5 - checksum: 10c0/c68f51b83c75844cf2cddc47d6e08fa7a6f0af83cc6213b60d0d8ac32988f8e7b44f321b1f926fa282a2cda02757ce3cbfe94e86f140fd025ab22f0dfee9bed9 + alova: ^3.0.9 + checksum: 10c0/9c48689e7dd081843726c4d8a93abbb7aca8b4654c606e67b81ab23de42542d5c17f2fbff7b65d4b75c7d484b42a74e76ba77e61446021620a256051eb82496c languageName: node linkType: hard @@ -1664,7 +1664,7 @@ __metadata: version: 0.0.0-use.local resolution: "EMS-ESP@workspace:." dependencies: - "@alova/adapter-xhr": "npm:2.0.4" + "@alova/adapter-xhr": "npm:2.0.5" "@babel/core": "npm:^7.25.2" "@emotion/react": "npm:^11.13.0" "@emotion/styled": "npm:^11.13.0" From d9d854e456082add0fec16ffdd6488d65984688b Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 13:17:23 +0200 Subject: [PATCH 3/9] formatting --- interface/src/app/main/Help.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/app/main/Help.tsx b/interface/src/app/main/Help.tsx index dbc939e3e..fa4411e21 100644 --- a/interface/src/app/main/Help.tsx +++ b/interface/src/app/main/Help.tsx @@ -113,7 +113,7 @@ const Help = () => { color="primary" onClick={() => callAPI('system', 'allvalues')} > - {LL.ALLVALUES(0)} + {LL.ALLVALUES(0)} From 92a8a268a7fd546fd38391ec1e6d6fc912439536 Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 13:18:07 +0200 Subject: [PATCH 4/9] update firmware automatically - #1920 --- interface/src/api/system.ts | 3 + interface/src/app/settings/DownloadUpload.tsx | 91 ++++++++++++++++-- lib/framework/UploadFileService.cpp | 47 +++++++++- lib/framework/UploadFileService.h | 6 +- mock-api/rest_server.ts | 11 ++- src/system.cpp | 94 ++++++++++++++++++- src/system.h | 2 + src/test/test.cpp | 13 ++- src/test/test.h | 2 + src/version.h | 2 +- 10 files changed, 252 insertions(+), 19 deletions(-) diff --git a/interface/src/api/system.ts b/interface/src/api/system.ts index ff9007b31..ce380e8b8 100644 --- a/interface/src/api/system.ts +++ b/interface/src/api/system.ts @@ -45,3 +45,6 @@ export const uploadFile = (file: File) => { timeout: 60000 // override timeout for uploading firmware - 1 minute }); }; + +export const uploadURL = (data: { url: string }) => + alovaInstance.Post('/rest/uploadURL', data); diff --git a/interface/src/app/settings/DownloadUpload.tsx b/interface/src/app/settings/DownloadUpload.tsx index c63138d2e..a58816c73 100644 --- a/interface/src/app/settings/DownloadUpload.tsx +++ b/interface/src/app/settings/DownloadUpload.tsx @@ -3,7 +3,18 @@ import { toast } from 'react-toastify'; import DownloadIcon from '@mui/icons-material/GetApp'; import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew'; -import { Box, Button, Divider, Link, Typography } from '@mui/material'; +import WarningIcon from '@mui/icons-material/Warning'; +import { + Box, + Button, + CircularProgress, + Dialog, + DialogContent, + DialogTitle, + Divider, + Link, + Typography +} from '@mui/material'; import * as SystemApi from 'api/system'; import { @@ -15,6 +26,7 @@ import { } from 'api/app'; import { getDevVersion, getStableVersion } from 'api/system'; +import { dialogStyle } from 'CustomTheme'; import { useRequest } from 'alova/client'; import type { APIcall } from 'app/main/types'; import RestartMonitor from 'app/status/RestartMonitor'; @@ -32,6 +44,7 @@ const DownloadUpload = () => { const [restarting, setRestarting] = useState(false); const [restartNeeded, setRestartNeeded] = useState(false); + const [showWaiting, setShowWaiting] = useState(false); const { send: sendSettings } = useRequest(getSettings(), { immediate: false @@ -72,6 +85,13 @@ const DownloadUpload = () => { error } = useRequest(SystemApi.readHardwareStatus); + const { send: sendUploadURL } = useRequest( + (data: { url: string }) => SystemApi.uploadURL(data), + { + immediate: false + } + ); + const { send: restartCommand } = useRequest(SystemApi.restart(), { immediate: false }); @@ -87,14 +107,19 @@ const DownloadUpload = () => { }; // called immediately to get the latest version, on page load - // set immediate to false to avoid calling the API on page load and GH blocking while testing! const { data: latestVersion } = useRequest(getStableVersion, { - immediate: true - // immediate: false + // immediate: true + // uncomment for testing + // https://github.com/emsesp/EMS-ESP32/releases/download/v3.6.5/EMS-ESP-3_6_5-ESP32-16MB+.bin + immediate: false, + initialData: '3.6.5' }); const { data: latestDevVersion } = useRequest(getDevVersion, { - immediate: true - // immediate: false + // immediate: true + // uncomment for testing + // https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32-16MB+.bin + immediate: false, + initialData: '3.7.0-dev.31' }); const STABLE_URL = 'https://github.com/emsesp/EMS-ESP32/releases/download/'; @@ -115,6 +140,13 @@ const DownloadUpload = () => { ); }; + const installFirmwareURL = async (url: string) => { + setShowWaiting(true); + await sendUploadURL({ url: url }).catch((error: Error) => { + toast.error(error.message); + }); + }; + const saveFile = (json: unknown, filename: string) => { const anchor = document.createElement('a'); anchor.href = URL.createObjectURL( @@ -275,6 +307,20 @@ const DownloadUpload = () => { {LL.DOWNLOAD(1)} ) + )} {latestDevVersion && ( @@ -295,10 +341,43 @@ const DownloadUpload = () => { {LL.DOWNLOAD(1)} ) + )} + + {/* TODO translate all this text*/} + Uploading + + + Please wait while the firmware is being uploaded and installed. This + can take a few minutes. EMS-ESP will automatically restart when + completed. + + + + + + + {LL.UPLOAD()} diff --git a/lib/framework/UploadFileService.cpp b/lib/framework/UploadFileService.cpp index 3ab3428c1..8df1a8ae2 100644 --- a/lib/framework/UploadFileService.cpp +++ b/lib/framework/UploadFileService.cpp @@ -16,6 +16,7 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * : _securityManager(securityManager) , _is_firmware(false) , _md5() { + // end-points server->on( UPLOAD_FILE_PATH, HTTP_POST, @@ -23,6 +24,10 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * [this](AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) { handleUpload(request, filename, index, data, len, final); }); + + server->on(UPLOAD_URL_PATH, + securityManager->wrapCallback([this](AsyncWebServerRequest * request, JsonVariant json) { uploadURL(request, json); }, + AuthenticationPredicates::IS_AUTHENTICATED)); } void UploadFileService::handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) { @@ -113,7 +118,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri } void UploadFileService::uploadComplete(AsyncWebServerRequest * request) { - // did we complete uploading a json file? + // did we just complete uploading a json file? if (request->_tempFile) { request->_tempFile.close(); // close the file handle as the upload is now done emsesp::EMSESP::system_.store_nvs_values(); @@ -166,4 +171,42 @@ void UploadFileService::handleError(AsyncWebServerRequest * request, int code) { void UploadFileService::handleEarlyDisconnect() { _is_firmware = false; Update.abort(); -} \ No newline at end of file +} + +// upload firmware from a URL, like GitHub Release assets, Cloudflare R2 or Amazon S3 +void UploadFileService::uploadURL(AsyncWebServerRequest * request, JsonVariant json) { + if (json.is()) { + String url = json["url"].as(); + + // TODO fix this from WDT crashing + // calling from "test upload" in a console, it works + // but via the web it crashes with the error message: + // E (253289) task_wdt: Task watchdog got triggered. The following tasks did not reset the watchdog in time: + // E (253289) task_wdt: - async_tcp (CPU 0/1) + // E (253289) task_wdt: Tasks currently running: + // E (253289) task_wdt: CPU 0: ipc0 + // E (253289) task_wdt: CPU 1: loopTask + // E (253289) task_wdt: Aborting. + // + // I think we need to stop all async services before uploading the firmware. Like MQTT? + + // force close the connection + request->client()->close(true); + + // start the upload + if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) { + emsesp::EMSESP::system_.upload_status(false); // tell ems-esp we're not uploading anymore + } + + /* + if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) { + emsesp::EMSESP::system_.upload_status(false); + handleError(request, 500); // internal error, failed + } else { + request->onDisconnect(RestartService::restartNow); + AsyncWebServerResponse * response = request->beginResponse(200); + request->send(response); + } + */ + } +} diff --git a/lib/framework/UploadFileService.h b/lib/framework/UploadFileService.h index 0885805fa..067ad491e 100644 --- a/lib/framework/UploadFileService.h +++ b/lib/framework/UploadFileService.h @@ -13,7 +13,9 @@ #include #define UPLOAD_FILE_PATH "/rest/uploadFile" -#define TEMP_FILENAME_PATH "/tmp_upload" +#define UPLOAD_URL_PATH "/rest/uploadURL" + +#define TEMP_FILENAME_PATH "/tmp_upload" // for uploaded json files class UploadFileService { public: @@ -27,6 +29,8 @@ class UploadFileService { void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final); void uploadComplete(AsyncWebServerRequest * request); void handleError(AsyncWebServerRequest * request, int code); + void uploadURL(AsyncWebServerRequest * request, JsonVariant json); + void handleEarlyDisconnect(); }; diff --git a/mock-api/rest_server.ts b/mock-api/rest_server.ts index 8f97f42d1..5162afc3c 100644 --- a/mock-api/rest_server.ts +++ b/mock-api/rest_server.ts @@ -4666,10 +4666,15 @@ router .get(EMSESP_GET_SETTINGS_ENDPOINT, () => emsesp_info) .get(EMSESP_GET_CUSTOMIZATIONS_ENDPOINT, () => emsesp_deviceentities_1) .get(EMSESP_GET_ENTITIES_ENDPOINT, () => emsesp_customentities) - .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule); + .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule) -// API which are usually POST for security -router + // upload URL + .post('/rest/uploadURL', () => { + console.log('upload File from URL'); + return status(200); + }) + + // API which are usually POST for security .post(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info) .get(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info) .post(API_ENDPOINT_ROOT, async (request: any) => { diff --git a/src/system.cpp b/src/system.cpp index a805c25fb..5e834e7fb 100644 --- a/src/system.cpp +++ b/src/system.cpp @@ -48,6 +48,8 @@ #include #endif +#include + namespace emsesp { // Languages supported. Note: the order is important and must match locale_translations.h @@ -504,11 +506,12 @@ void System::start() { // button single click void System::button_OnClick(PButton & b) { - LOG_NOTICE("Button pressed - single click - show settings folders"); + LOG_NOTICE("Button pressed - single click"); #if defined(EMSESP_TEST) #ifndef EMSESP_STANDALONE - Test::listDir(LittleFS, FS_CONFIG_DIRECTORY, 3); + // show filesystem + Test::listDir(LittleFS, "/", 3); #endif #endif } @@ -1150,7 +1153,7 @@ bool System::check_restore() { LOG_ERROR("Unrecognized file uploaded"); } } else { - LOG_ERROR("Unrecognized file uploaded, not json"); + LOG_ERROR("Unrecognized file uploaded, not json. Will be removed."); } // close (just in case) and remove the temp file @@ -1809,6 +1812,7 @@ bool System::ntp_connected() { return ntp_connected_; } +// see if its a BBQKees Gateway by checking the nvs values String System::getBBQKeesGatewayDetails() { #ifndef EMSESP_STANDALONE if (!EMSESP::nvs_.isKey("mfg")) { @@ -1829,4 +1833,88 @@ String System::getBBQKeesGatewayDetails() { #endif } +// Stream from an URL and send straight to OTA uploader +bool System::uploadFirmwareURL(const char * url) { +#ifndef EMSESP_STANDALONE + // configure temporary server and url + HTTPClient http; + http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub + http.useHTTP10(true); + http.begin(String(url)); + + // start connection + int httpCode = http.GET(); + if (httpCode != HTTP_CODE_OK) { + LOG_ERROR("Firmware upload failed - HTTP code %u", httpCode); + return false; + } + + // check we have enough space + int firmware_size = http.getSize(); + LOG_INFO("Firmware uploading (file: %s, size: %d bytes)", url, firmware_size); + if (!Update.begin(firmware_size)) { + LOG_ERROR("Firmware upload failed - no space"); + return false; + } + + EMSESP::system_.upload_status(true); // tell EMS-ESP we're uploading + + // get tcp stream and send it to Updater + WiFiClient * stream = http.getStreamPtr(); + if (Update.writeStream(*stream) != firmware_size) { + LOG_ERROR("Firmware upload failed - size differences"); + return false; + } + + if (!Update.end(true)) { + LOG_ERROR("Firmware upload error"); + return false; + } + + http.end(); + + LOG_INFO("Firmware uploaded successfully. Restarting..."); + + restart_requested(true); // not sure this is needed? + +#endif + + return true; // OK + + /* + TODO backup code to save firmware to LittleFS first, in case of slow networks + + // create buffer for reading in 128 byte chunks + const size_t buffer_size = 1024; + uint8_t buff[buffer_size] = {0}; + + File file = LittleFS.open("/new_firmware", "w"); // TODO find new name + if (!file) { + Serial.println("Failed to open file for writing"); + return false; + } + + // read all data from server + while (http.connected() && (firmware_size > 0 || firmware_size == -1)) { + // get available data size + size_t size = stream->available(); + + if (size) { + // read up to 128 byte + int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size)); + + file.write(buff, c); + if (firmware_size > 0) { + firmware_size -= c; + } + } + yield(); + // delay(1); // so not to hurt WTD or timeout + } + + file.close(); + + */ +} + } // namespace emsesp diff --git a/src/system.h b/src/system.h index 18b4e1563..edc966768 100644 --- a/src/system.h +++ b/src/system.h @@ -100,6 +100,8 @@ class System { String getBBQKeesGatewayDetails(); + static bool uploadFirmwareURL(const char * url); + void led_init(bool refresh); void network_init(bool refresh); void button_init(bool refresh); diff --git a/src/test/test.cpp b/src/test/test.cpp index b6cfc4e28..c13c8c349 100644 --- a/src/test/test.cpp +++ b/src/test/test.cpp @@ -408,11 +408,16 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const ok = true; } -// THESE ONLY WORK WITH AN ESP32, not in standalone mode +// THESE ONLY WORK WITH AN ESP32, not in standalone/native mode #ifndef EMSESP_STANDALONE if (command == "ls") { listDir(LittleFS, "/", 3); - Serial.println(); + ok = true; + } + + if (command == "upload") { + // S3 has 16MB flash + EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32S3-16MB+.bin"); // TODO remove ok = true; } #endif @@ -2278,7 +2283,9 @@ void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) { Serial.print(" DIR: "); Serial.println(file.name()); if (levels) { - listDir(fs, file.name(), levels - 1); + // prefix a / to the name to make it a full path + listDir(fs, ("/" + String(file.name())).c_str(), levels - 1); + // listDir(fs, file.name(), levels - 1); } Serial.println(); } else { diff --git a/src/test/test.h b/src/test/test.h index 5f672358f..d44a9b9c7 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -58,6 +58,8 @@ namespace emsesp { // #define EMSESP_DEBUG_DEFAULT "custom" // #define EMSESP_DEBUG_DEFAULT "scheduler" // #define EMSESP_DEBUG_DEFAULT "heat_exchange" +// #define EMSESP_DEBUG_DEFAULT "ls" +#define EMSESP_DEBUG_DEFAULT "upload" #ifndef EMSESP_DEBUG_DEFAULT #define EMSESP_DEBUG_DEFAULT "general" diff --git a/src/version.h b/src/version.h index ef0dce791..a993524cf 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define EMSESP_APP_VERSION "3.7.0-dev.31" +#define EMSESP_APP_VERSION "3.7.0-dev.32" From 119dcaa7fc915ee8f744353132fb261bfff4a242 Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 16:17:03 +0200 Subject: [PATCH 5/9] make it work from web (thanks michael) --- .gitignore | 3 - interface/src/app/settings/DownloadUpload.tsx | 44 ++-------- interface/src/app/status/RestartMonitor.tsx | 14 ++-- lib/framework/UploadFileService.cpp | 36 ++------ src/emsesp.cpp | 4 +- src/system.cpp | 82 ++++++++----------- src/system.h | 2 +- src/web/WebStatusService.cpp | 6 ++ 8 files changed, 68 insertions(+), 123 deletions(-) diff --git a/.gitignore b/.gitignore index 2eb46fa12..8eef96223 100644 --- a/.gitignore +++ b/.gitignore @@ -47,9 +47,6 @@ interface/src/i18n/i18n-types.ts interface/src/i18n/i18n-util.ts interface/src/i18n/i18n-util.sync.ts interface/src/i18n/i18n-util.async.ts -# mock-api uploads -*/uploads/* -!uploads/README.md # scripts test.sh diff --git a/interface/src/app/settings/DownloadUpload.tsx b/interface/src/app/settings/DownloadUpload.tsx index a58816c73..5dc302285 100644 --- a/interface/src/app/settings/DownloadUpload.tsx +++ b/interface/src/app/settings/DownloadUpload.tsx @@ -4,17 +4,7 @@ import { toast } from 'react-toastify'; import DownloadIcon from '@mui/icons-material/GetApp'; import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew'; import WarningIcon from '@mui/icons-material/Warning'; -import { - Box, - Button, - CircularProgress, - Dialog, - DialogContent, - DialogTitle, - Divider, - Link, - Typography -} from '@mui/material'; +import { Box, Button, Divider, Link, Typography } from '@mui/material'; import * as SystemApi from 'api/system'; import { @@ -44,7 +34,6 @@ const DownloadUpload = () => { const [restarting, setRestarting] = useState(false); const [restartNeeded, setRestartNeeded] = useState(false); - const [showWaiting, setShowWaiting] = useState(false); const { send: sendSettings } = useRequest(getSettings(), { immediate: false @@ -141,10 +130,10 @@ const DownloadUpload = () => { }; const installFirmwareURL = async (url: string) => { - setShowWaiting(true); await sendUploadURL({ url: url }).catch((error: Error) => { toast.error(error.message); }); + setRestarting(true); }; const saveFile = (json: unknown, filename: string) => { @@ -357,27 +346,6 @@ const DownloadUpload = () => { )} - - {/* TODO translate all this text*/} - Uploading - - - Please wait while the firmware is being uploaded and installed. This - can take a few minutes. EMS-ESP will automatically restart when - completed. - - - - - - - {LL.UPLOAD()} @@ -405,7 +373,13 @@ const DownloadUpload = () => { }; return ( - {restarting ? : content()} + + {restarting ? ( + + ) : ( + content() + )} + ); }; diff --git a/interface/src/app/status/RestartMonitor.tsx b/interface/src/app/status/RestartMonitor.tsx index ba358af4a..685cac3e8 100644 --- a/interface/src/app/status/RestartMonitor.tsx +++ b/interface/src/app/status/RestartMonitor.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react'; +import { type FC, useEffect, useRef, useState } from 'react'; import * as SystemApi from 'api/system'; @@ -6,10 +6,14 @@ import { useRequest } from 'alova/client'; import { FormLoader } from 'components'; import { useI18nContext } from 'i18n/i18n-react'; -const RESTART_TIMEOUT = 2 * 60 * 1000; -const POLL_INTERVAL = 1000; +const RESTART_TIMEOUT = 2 * 60 * 1000; // 2 minutes +const POLL_INTERVAL = 1000; // every 1 second -const RestartMonitor = () => { +export interface RestartMonitorProps { + message?: string; +} + +const RestartMonitor: FC = ({ message }) => { const [failed, setFailed] = useState(false); const [timeoutId, setTimeoutId] = useState(); const { LL } = useI18nContext(); @@ -38,7 +42,7 @@ const RestartMonitor = () => { return ( ); diff --git a/lib/framework/UploadFileService.cpp b/lib/framework/UploadFileService.cpp index 8df1a8ae2..84f34e5cf 100644 --- a/lib/framework/UploadFileService.cpp +++ b/lib/framework/UploadFileService.cpp @@ -176,37 +176,11 @@ void UploadFileService::handleEarlyDisconnect() { // upload firmware from a URL, like GitHub Release assets, Cloudflare R2 or Amazon S3 void UploadFileService::uploadURL(AsyncWebServerRequest * request, JsonVariant json) { if (json.is()) { - String url = json["url"].as(); + // this will keep a copy of the URL, but won't initiate the download yet + emsesp::EMSESP::system_.uploadFirmwareURL(json["url"].as()); - // TODO fix this from WDT crashing - // calling from "test upload" in a console, it works - // but via the web it crashes with the error message: - // E (253289) task_wdt: Task watchdog got triggered. The following tasks did not reset the watchdog in time: - // E (253289) task_wdt: - async_tcp (CPU 0/1) - // E (253289) task_wdt: Tasks currently running: - // E (253289) task_wdt: CPU 0: ipc0 - // E (253289) task_wdt: CPU 1: loopTask - // E (253289) task_wdt: Aborting. - // - // I think we need to stop all async services before uploading the firmware. Like MQTT? - - // force close the connection - request->client()->close(true); - - // start the upload - if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) { - emsesp::EMSESP::system_.upload_status(false); // tell ems-esp we're not uploading anymore - } - - /* - if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) { - emsesp::EMSESP::system_.upload_status(false); - handleError(request, 500); // internal error, failed - } else { - request->onDisconnect(RestartService::restartNow); - AsyncWebServerResponse * response = request->beginResponse(200); - request->send(response); - } - */ + // end the connection + AsyncWebServerResponse * response = request->beginResponse(200); + request->send(response); } } diff --git a/src/emsesp.cpp b/src/emsesp.cpp index 6de204f29..8b6a976b8 100644 --- a/src/emsesp.cpp +++ b/src/emsesp.cpp @@ -1671,11 +1671,13 @@ void EMSESP::loop() { mqtt_.loop(); // sends out anything in the MQTT queue webModulesService.loop(); // loop through the external library modules if (system_.PSram() == 0) { - webSchedulerService.loop(); + webSchedulerService.loop(); // run non-async if there is no PSRAM available } // force a query on the EMS devices to fetch latest data at a set interval (1 min) scheduled_fetch_values(); + } else { + emsesp::EMSESP::system_.uploadFirmwareURL(); // start an upload from a URL. This is blocking. } uuid::loop(); diff --git a/src/system.cpp b/src/system.cpp index 5e834e7fb..55b45ee4d 100644 --- a/src/system.cpp +++ b/src/system.cpp @@ -1833,31 +1833,51 @@ String System::getBBQKeesGatewayDetails() { #endif } -// Stream from an URL and send straight to OTA uploader +// Stream from an URL and send straight to OTA uploader service. +// +// This function needs to be called twice, once with a url to persist it, and second with no arguments to start the upload +// This is to avoid timeouts in callback functions, like calling from a web hook. bool System::uploadFirmwareURL(const char * url) { #ifndef EMSESP_STANDALONE - // configure temporary server and url - HTTPClient http; - http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub - http.useHTTP10(true); - http.begin(String(url)); - // start connection + static String saved_url; + + // if the URL is not empty, save it for later + if (url && strlen(url) > 0) { + saved_url = url; + EMSESP::system_.upload_status(true); // tell EMS-ESP we're ready to start the uploading process + return true; + } + + // make sure we have a valid URL + if (saved_url.isEmpty()) { + LOG_ERROR("Firmware upload failed - no URL"); + return false; + } + + // Configure temporary client + HTTPClient http; + http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub 302's + http.useHTTP10(true); + http.begin(saved_url); + + // start a connection int httpCode = http.GET(); if (httpCode != HTTP_CODE_OK) { LOG_ERROR("Firmware upload failed - HTTP code %u", httpCode); return false; } - // check we have enough space + // check we have enough space for the upload in the ota partition int firmware_size = http.getSize(); - LOG_INFO("Firmware uploading (file: %s, size: %d bytes)", url, firmware_size); + LOG_INFO("Firmware uploading (file: %s, size: %d bytes). Please wait...", saved_url.c_str(), firmware_size); if (!Update.begin(firmware_size)) { LOG_ERROR("Firmware upload failed - no space"); return false; } - EMSESP::system_.upload_status(true); // tell EMS-ESP we're uploading + // flush buffers so latest log messages are shown + Shell::loop_all(); // get tcp stream and send it to Updater WiFiClient * stream = http.getStreamPtr(); @@ -1867,54 +1887,22 @@ bool System::uploadFirmwareURL(const char * url) { } if (!Update.end(true)) { - LOG_ERROR("Firmware upload error"); + LOG_ERROR("Firmware upload failed - general error"); return false; } http.end(); + EMSESP::system_.upload_status(false); + saved_url.clear(); // prevent from downloading again + LOG_INFO("Firmware uploaded successfully. Restarting..."); - restart_requested(true); // not sure this is needed? + restart_requested(true); #endif return true; // OK - - /* - TODO backup code to save firmware to LittleFS first, in case of slow networks - - // create buffer for reading in 128 byte chunks - const size_t buffer_size = 1024; - uint8_t buff[buffer_size] = {0}; - - File file = LittleFS.open("/new_firmware", "w"); // TODO find new name - if (!file) { - Serial.println("Failed to open file for writing"); - return false; - } - - // read all data from server - while (http.connected() && (firmware_size > 0 || firmware_size == -1)) { - // get available data size - size_t size = stream->available(); - - if (size) { - // read up to 128 byte - int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size)); - - file.write(buff, c); - if (firmware_size > 0) { - firmware_size -= c; - } - } - yield(); - // delay(1); // so not to hurt WTD or timeout - } - - file.close(); - - */ } } // namespace emsesp diff --git a/src/system.h b/src/system.h index edc966768..1a9f570b5 100644 --- a/src/system.h +++ b/src/system.h @@ -100,7 +100,7 @@ class System { String getBBQKeesGatewayDetails(); - static bool uploadFirmwareURL(const char * url); + static bool uploadFirmwareURL(const char * url = nullptr); void led_init(bool refresh); void network_init(bool refresh); diff --git a/src/web/WebStatusService.cpp b/src/web/WebStatusService.cpp index d2eb47648..a9d56cedc 100644 --- a/src/web/WebStatusService.cpp +++ b/src/web/WebStatusService.cpp @@ -31,6 +31,12 @@ WebStatusService::WebStatusService(AsyncWebServer * server, SecurityManager * se } void WebStatusService::systemStatus(AsyncWebServerRequest * request) { + // This is a litle trick for the OTA upload. We don't want the React RestartService to think we're finished + // with the upload so we fake it and pretent the /rest/systemStatus is not available. That way the spinner keeps spinning. + if (EMSESP::system_.upload_status()) { + return; // ignore endpoint + } + EMSESP::system_.refreshHeapMem(); // refresh free heap and max alloc heap auto * response = new AsyncJsonResponse(false); From a7a93eb4f5ac49d121dc31728bb2f4a955eccfda Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 16:18:52 +0200 Subject: [PATCH 6/9] revert back hardcoded GH calls --- interface/src/app/settings/DownloadUpload.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/interface/src/app/settings/DownloadUpload.tsx b/interface/src/app/settings/DownloadUpload.tsx index 5dc302285..cae8044cd 100644 --- a/interface/src/app/settings/DownloadUpload.tsx +++ b/interface/src/app/settings/DownloadUpload.tsx @@ -97,18 +97,18 @@ const DownloadUpload = () => { // called immediately to get the latest version, on page load const { data: latestVersion } = useRequest(getStableVersion, { - // immediate: true + immediate: true // uncomment for testing // https://github.com/emsesp/EMS-ESP32/releases/download/v3.6.5/EMS-ESP-3_6_5-ESP32-16MB+.bin - immediate: false, - initialData: '3.6.5' + // immediate: false, + // initialData: '3.6.5' }); const { data: latestDevVersion } = useRequest(getDevVersion, { - // immediate: true + immediate: true // uncomment for testing // https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32-16MB+.bin - immediate: false, - initialData: '3.7.0-dev.31' + // immediate: false, + // initialData: '3.7.0-dev.31' }); const STABLE_URL = 'https://github.com/emsesp/EMS-ESP32/releases/download/'; From 405e23561fd70855444eb060ede5e6df3aa3c003 Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 16:39:43 +0200 Subject: [PATCH 7/9] remove space --- interface/src/types/system.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/types/system.ts b/interface/src/types/system.ts index 778951444..2d59ca9d4 100644 --- a/interface/src/types/system.ts +++ b/interface/src/types/system.ts @@ -24,7 +24,6 @@ export interface HardwareStatus { psram: boolean; psram_size?: number; free_psram?: number; - free_caps: number; model: string; } From 87ee50708b2001112ad1e4d10b4947a2d992e00c Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 16:39:59 +0200 Subject: [PATCH 8/9] prevent annoying screen jump when rendering --- interface/src/app/status/Status.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/app/status/Status.tsx b/interface/src/app/status/Status.tsx index e591b8d8b..cca48dbee 100644 --- a/interface/src/app/status/Status.tsx +++ b/interface/src/app/status/Status.tsx @@ -306,7 +306,7 @@ const SystemStatus = () => { ); const content = () => { - if (!data) { + if (!data || !LL) { return ; } @@ -321,7 +321,7 @@ const SystemStatus = () => { {me.admin && (