From 92a8a268a7fd546fd38391ec1e6d6fc912439536 Mon Sep 17 00:00:00 2001 From: proddy Date: Sun, 18 Aug 2024 13:18:07 +0200 Subject: [PATCH] update firmware automatically - #1920 --- interface/src/api/system.ts | 3 + interface/src/app/settings/DownloadUpload.tsx | 91 ++++++++++++++++-- lib/framework/UploadFileService.cpp | 47 +++++++++- lib/framework/UploadFileService.h | 6 +- mock-api/rest_server.ts | 11 ++- src/system.cpp | 94 ++++++++++++++++++- src/system.h | 2 + src/test/test.cpp | 13 ++- src/test/test.h | 2 + src/version.h | 2 +- 10 files changed, 252 insertions(+), 19 deletions(-) diff --git a/interface/src/api/system.ts b/interface/src/api/system.ts index ff9007b31..ce380e8b8 100644 --- a/interface/src/api/system.ts +++ b/interface/src/api/system.ts @@ -45,3 +45,6 @@ export const uploadFile = (file: File) => { timeout: 60000 // override timeout for uploading firmware - 1 minute }); }; + +export const uploadURL = (data: { url: string }) => + alovaInstance.Post('/rest/uploadURL', data); diff --git a/interface/src/app/settings/DownloadUpload.tsx b/interface/src/app/settings/DownloadUpload.tsx index c63138d2e..a58816c73 100644 --- a/interface/src/app/settings/DownloadUpload.tsx +++ b/interface/src/app/settings/DownloadUpload.tsx @@ -3,7 +3,18 @@ import { toast } from 'react-toastify'; import DownloadIcon from '@mui/icons-material/GetApp'; import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew'; -import { Box, Button, Divider, Link, Typography } from '@mui/material'; +import WarningIcon from '@mui/icons-material/Warning'; +import { + Box, + Button, + CircularProgress, + Dialog, + DialogContent, + DialogTitle, + Divider, + Link, + Typography +} from '@mui/material'; import * as SystemApi from 'api/system'; import { @@ -15,6 +26,7 @@ import { } from 'api/app'; import { getDevVersion, getStableVersion } from 'api/system'; +import { dialogStyle } from 'CustomTheme'; import { useRequest } from 'alova/client'; import type { APIcall } from 'app/main/types'; import RestartMonitor from 'app/status/RestartMonitor'; @@ -32,6 +44,7 @@ const DownloadUpload = () => { const [restarting, setRestarting] = useState(false); const [restartNeeded, setRestartNeeded] = useState(false); + const [showWaiting, setShowWaiting] = useState(false); const { send: sendSettings } = useRequest(getSettings(), { immediate: false @@ -72,6 +85,13 @@ const DownloadUpload = () => { error } = useRequest(SystemApi.readHardwareStatus); + const { send: sendUploadURL } = useRequest( + (data: { url: string }) => SystemApi.uploadURL(data), + { + immediate: false + } + ); + const { send: restartCommand } = useRequest(SystemApi.restart(), { immediate: false }); @@ -87,14 +107,19 @@ const DownloadUpload = () => { }; // called immediately to get the latest version, on page load - // set immediate to false to avoid calling the API on page load and GH blocking while testing! const { data: latestVersion } = useRequest(getStableVersion, { - immediate: true - // immediate: false + // immediate: true + // uncomment for testing + // https://github.com/emsesp/EMS-ESP32/releases/download/v3.6.5/EMS-ESP-3_6_5-ESP32-16MB+.bin + immediate: false, + initialData: '3.6.5' }); const { data: latestDevVersion } = useRequest(getDevVersion, { - immediate: true - // immediate: false + // immediate: true + // uncomment for testing + // https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32-16MB+.bin + immediate: false, + initialData: '3.7.0-dev.31' }); const STABLE_URL = 'https://github.com/emsesp/EMS-ESP32/releases/download/'; @@ -115,6 +140,13 @@ const DownloadUpload = () => { ); }; + const installFirmwareURL = async (url: string) => { + setShowWaiting(true); + await sendUploadURL({ url: url }).catch((error: Error) => { + toast.error(error.message); + }); + }; + const saveFile = (json: unknown, filename: string) => { const anchor = document.createElement('a'); anchor.href = URL.createObjectURL( @@ -275,6 +307,20 @@ const DownloadUpload = () => { {LL.DOWNLOAD(1)} ) + )} {latestDevVersion && ( @@ -295,10 +341,43 @@ const DownloadUpload = () => { {LL.DOWNLOAD(1)} ) + )} + + {/* TODO translate all this text*/} + Uploading + + + Please wait while the firmware is being uploaded and installed. This + can take a few minutes. EMS-ESP will automatically restart when + completed. + + + + + + + {LL.UPLOAD()} diff --git a/lib/framework/UploadFileService.cpp b/lib/framework/UploadFileService.cpp index 3ab3428c1..8df1a8ae2 100644 --- a/lib/framework/UploadFileService.cpp +++ b/lib/framework/UploadFileService.cpp @@ -16,6 +16,7 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * : _securityManager(securityManager) , _is_firmware(false) , _md5() { + // end-points server->on( UPLOAD_FILE_PATH, HTTP_POST, @@ -23,6 +24,10 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * [this](AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) { handleUpload(request, filename, index, data, len, final); }); + + server->on(UPLOAD_URL_PATH, + securityManager->wrapCallback([this](AsyncWebServerRequest * request, JsonVariant json) { uploadURL(request, json); }, + AuthenticationPredicates::IS_AUTHENTICATED)); } void UploadFileService::handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) { @@ -113,7 +118,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri } void UploadFileService::uploadComplete(AsyncWebServerRequest * request) { - // did we complete uploading a json file? + // did we just complete uploading a json file? if (request->_tempFile) { request->_tempFile.close(); // close the file handle as the upload is now done emsesp::EMSESP::system_.store_nvs_values(); @@ -166,4 +171,42 @@ void UploadFileService::handleError(AsyncWebServerRequest * request, int code) { void UploadFileService::handleEarlyDisconnect() { _is_firmware = false; Update.abort(); -} \ No newline at end of file +} + +// upload firmware from a URL, like GitHub Release assets, Cloudflare R2 or Amazon S3 +void UploadFileService::uploadURL(AsyncWebServerRequest * request, JsonVariant json) { + if (json.is()) { + String url = json["url"].as(); + + // TODO fix this from WDT crashing + // calling from "test upload" in a console, it works + // but via the web it crashes with the error message: + // E (253289) task_wdt: Task watchdog got triggered. The following tasks did not reset the watchdog in time: + // E (253289) task_wdt: - async_tcp (CPU 0/1) + // E (253289) task_wdt: Tasks currently running: + // E (253289) task_wdt: CPU 0: ipc0 + // E (253289) task_wdt: CPU 1: loopTask + // E (253289) task_wdt: Aborting. + // + // I think we need to stop all async services before uploading the firmware. Like MQTT? + + // force close the connection + request->client()->close(true); + + // start the upload + if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) { + emsesp::EMSESP::system_.upload_status(false); // tell ems-esp we're not uploading anymore + } + + /* + if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) { + emsesp::EMSESP::system_.upload_status(false); + handleError(request, 500); // internal error, failed + } else { + request->onDisconnect(RestartService::restartNow); + AsyncWebServerResponse * response = request->beginResponse(200); + request->send(response); + } + */ + } +} diff --git a/lib/framework/UploadFileService.h b/lib/framework/UploadFileService.h index 0885805fa..067ad491e 100644 --- a/lib/framework/UploadFileService.h +++ b/lib/framework/UploadFileService.h @@ -13,7 +13,9 @@ #include #define UPLOAD_FILE_PATH "/rest/uploadFile" -#define TEMP_FILENAME_PATH "/tmp_upload" +#define UPLOAD_URL_PATH "/rest/uploadURL" + +#define TEMP_FILENAME_PATH "/tmp_upload" // for uploaded json files class UploadFileService { public: @@ -27,6 +29,8 @@ class UploadFileService { void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final); void uploadComplete(AsyncWebServerRequest * request); void handleError(AsyncWebServerRequest * request, int code); + void uploadURL(AsyncWebServerRequest * request, JsonVariant json); + void handleEarlyDisconnect(); }; diff --git a/mock-api/rest_server.ts b/mock-api/rest_server.ts index 8f97f42d1..5162afc3c 100644 --- a/mock-api/rest_server.ts +++ b/mock-api/rest_server.ts @@ -4666,10 +4666,15 @@ router .get(EMSESP_GET_SETTINGS_ENDPOINT, () => emsesp_info) .get(EMSESP_GET_CUSTOMIZATIONS_ENDPOINT, () => emsesp_deviceentities_1) .get(EMSESP_GET_ENTITIES_ENDPOINT, () => emsesp_customentities) - .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule); + .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule) -// API which are usually POST for security -router + // upload URL + .post('/rest/uploadURL', () => { + console.log('upload File from URL'); + return status(200); + }) + + // API which are usually POST for security .post(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info) .get(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info) .post(API_ENDPOINT_ROOT, async (request: any) => { diff --git a/src/system.cpp b/src/system.cpp index a805c25fb..5e834e7fb 100644 --- a/src/system.cpp +++ b/src/system.cpp @@ -48,6 +48,8 @@ #include #endif +#include + namespace emsesp { // Languages supported. Note: the order is important and must match locale_translations.h @@ -504,11 +506,12 @@ void System::start() { // button single click void System::button_OnClick(PButton & b) { - LOG_NOTICE("Button pressed - single click - show settings folders"); + LOG_NOTICE("Button pressed - single click"); #if defined(EMSESP_TEST) #ifndef EMSESP_STANDALONE - Test::listDir(LittleFS, FS_CONFIG_DIRECTORY, 3); + // show filesystem + Test::listDir(LittleFS, "/", 3); #endif #endif } @@ -1150,7 +1153,7 @@ bool System::check_restore() { LOG_ERROR("Unrecognized file uploaded"); } } else { - LOG_ERROR("Unrecognized file uploaded, not json"); + LOG_ERROR("Unrecognized file uploaded, not json. Will be removed."); } // close (just in case) and remove the temp file @@ -1809,6 +1812,7 @@ bool System::ntp_connected() { return ntp_connected_; } +// see if its a BBQKees Gateway by checking the nvs values String System::getBBQKeesGatewayDetails() { #ifndef EMSESP_STANDALONE if (!EMSESP::nvs_.isKey("mfg")) { @@ -1829,4 +1833,88 @@ String System::getBBQKeesGatewayDetails() { #endif } +// Stream from an URL and send straight to OTA uploader +bool System::uploadFirmwareURL(const char * url) { +#ifndef EMSESP_STANDALONE + // configure temporary server and url + HTTPClient http; + http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub + http.useHTTP10(true); + http.begin(String(url)); + + // start connection + int httpCode = http.GET(); + if (httpCode != HTTP_CODE_OK) { + LOG_ERROR("Firmware upload failed - HTTP code %u", httpCode); + return false; + } + + // check we have enough space + int firmware_size = http.getSize(); + LOG_INFO("Firmware uploading (file: %s, size: %d bytes)", url, firmware_size); + if (!Update.begin(firmware_size)) { + LOG_ERROR("Firmware upload failed - no space"); + return false; + } + + EMSESP::system_.upload_status(true); // tell EMS-ESP we're uploading + + // get tcp stream and send it to Updater + WiFiClient * stream = http.getStreamPtr(); + if (Update.writeStream(*stream) != firmware_size) { + LOG_ERROR("Firmware upload failed - size differences"); + return false; + } + + if (!Update.end(true)) { + LOG_ERROR("Firmware upload error"); + return false; + } + + http.end(); + + LOG_INFO("Firmware uploaded successfully. Restarting..."); + + restart_requested(true); // not sure this is needed? + +#endif + + return true; // OK + + /* + TODO backup code to save firmware to LittleFS first, in case of slow networks + + // create buffer for reading in 128 byte chunks + const size_t buffer_size = 1024; + uint8_t buff[buffer_size] = {0}; + + File file = LittleFS.open("/new_firmware", "w"); // TODO find new name + if (!file) { + Serial.println("Failed to open file for writing"); + return false; + } + + // read all data from server + while (http.connected() && (firmware_size > 0 || firmware_size == -1)) { + // get available data size + size_t size = stream->available(); + + if (size) { + // read up to 128 byte + int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size)); + + file.write(buff, c); + if (firmware_size > 0) { + firmware_size -= c; + } + } + yield(); + // delay(1); // so not to hurt WTD or timeout + } + + file.close(); + + */ +} + } // namespace emsesp diff --git a/src/system.h b/src/system.h index 18b4e1563..edc966768 100644 --- a/src/system.h +++ b/src/system.h @@ -100,6 +100,8 @@ class System { String getBBQKeesGatewayDetails(); + static bool uploadFirmwareURL(const char * url); + void led_init(bool refresh); void network_init(bool refresh); void button_init(bool refresh); diff --git a/src/test/test.cpp b/src/test/test.cpp index b6cfc4e28..c13c8c349 100644 --- a/src/test/test.cpp +++ b/src/test/test.cpp @@ -408,11 +408,16 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const ok = true; } -// THESE ONLY WORK WITH AN ESP32, not in standalone mode +// THESE ONLY WORK WITH AN ESP32, not in standalone/native mode #ifndef EMSESP_STANDALONE if (command == "ls") { listDir(LittleFS, "/", 3); - Serial.println(); + ok = true; + } + + if (command == "upload") { + // S3 has 16MB flash + EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32S3-16MB+.bin"); // TODO remove ok = true; } #endif @@ -2278,7 +2283,9 @@ void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) { Serial.print(" DIR: "); Serial.println(file.name()); if (levels) { - listDir(fs, file.name(), levels - 1); + // prefix a / to the name to make it a full path + listDir(fs, ("/" + String(file.name())).c_str(), levels - 1); + // listDir(fs, file.name(), levels - 1); } Serial.println(); } else { diff --git a/src/test/test.h b/src/test/test.h index 5f672358f..d44a9b9c7 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -58,6 +58,8 @@ namespace emsesp { // #define EMSESP_DEBUG_DEFAULT "custom" // #define EMSESP_DEBUG_DEFAULT "scheduler" // #define EMSESP_DEBUG_DEFAULT "heat_exchange" +// #define EMSESP_DEBUG_DEFAULT "ls" +#define EMSESP_DEBUG_DEFAULT "upload" #ifndef EMSESP_DEBUG_DEFAULT #define EMSESP_DEBUG_DEFAULT "general" diff --git a/src/version.h b/src/version.h index ef0dce791..a993524cf 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define EMSESP_APP_VERSION "3.7.0-dev.31" +#define EMSESP_APP_VERSION "3.7.0-dev.32"