update firmware automatically - #1920

interface/src/api/system.ts

@@ -45,3 +45,6 @@ export const uploadFile = (file: File) => {
     timeout: 60000 // override timeout for uploading firmware - 1 minute
   });
 };
+
+export const uploadURL = (data: { url: string }) =>
+  alovaInstance.Post('/rest/uploadURL', data);

interface/src/app/settings/DownloadUpload.tsx

@@ -3,7 +3,18 @@ import { toast } from 'react-toastify';
 
 import DownloadIcon from '@mui/icons-material/GetApp';
 import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew';
-import { Box, Button, Divider, Link, Typography } from '@mui/material';
+import WarningIcon from '@mui/icons-material/Warning';
+import {
+  Box,
+  Button,
+  CircularProgress,
+  Dialog,
+  DialogContent,
+  DialogTitle,
+  Divider,
+  Link,
+  Typography
+} from '@mui/material';
 
 import * as SystemApi from 'api/system';
 import {
@@ -15,6 +26,7 @@ import {
 } from 'api/app';
 import { getDevVersion, getStableVersion } from 'api/system';
 
+import { dialogStyle } from 'CustomTheme';
 import { useRequest } from 'alova/client';
 import type { APIcall } from 'app/main/types';
 import RestartMonitor from 'app/status/RestartMonitor';
@@ -32,6 +44,7 @@ const DownloadUpload = () => {
 
   const [restarting, setRestarting] = useState<boolean>(false);
   const [restartNeeded, setRestartNeeded] = useState<boolean>(false);
+  const [showWaiting, setShowWaiting] = useState<boolean>(false);
 
   const { send: sendSettings } = useRequest(getSettings(), {
     immediate: false
@@ -72,6 +85,13 @@ const DownloadUpload = () => {
     error
   } = useRequest(SystemApi.readHardwareStatus);
 
+  const { send: sendUploadURL } = useRequest(
+    (data: { url: string }) => SystemApi.uploadURL(data),
+    {
+      immediate: false
+    }
+  );
+
   const { send: restartCommand } = useRequest(SystemApi.restart(), {
     immediate: false
   });
@@ -87,14 +107,19 @@ const DownloadUpload = () => {
   };
 
   // called immediately to get the latest version, on page load
-  // set immediate to false to avoid calling the API on page load and GH blocking while testing!
   const { data: latestVersion } = useRequest(getStableVersion, {
-    immediate: true
+    // immediate: true
-    // immediate: false
+    // uncomment for testing
+    // https://github.com/emsesp/EMS-ESP32/releases/download/v3.6.5/EMS-ESP-3_6_5-ESP32-16MB+.bin
+    immediate: false,
+    initialData: '3.6.5'
   });
   const { data: latestDevVersion } = useRequest(getDevVersion, {
-    immediate: true
+    // immediate: true
-    // immediate: false
+    // uncomment for testing
+    // https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32-16MB+.bin
+    immediate: false,
+    initialData: '3.7.0-dev.31'
   });
 
   const STABLE_URL = 'https://github.com/emsesp/EMS-ESP32/releases/download/';
@@ -115,6 +140,13 @@ const DownloadUpload = () => {
     );
   };
 
+  const installFirmwareURL = async (url: string) => {
+    setShowWaiting(true);
+    await sendUploadURL({ url: url }).catch((error: Error) => {
+      toast.error(error.message);
+    });
+  };
+
   const saveFile = (json: unknown, filename: string) => {
     const anchor = document.createElement('a');
     anchor.href = URL.createObjectURL(
@@ -275,6 +307,20 @@ const DownloadUpload = () => {
                 {LL.DOWNLOAD(1)}
               </Link>
               )
+              <Button
+                sx={{ ml: 2 }}
+                size="small"
+                startIcon={<WarningIcon color="warning" />}
+                variant="outlined"
+                color="primary"
+                onClick={() =>
+                  installFirmwareURL(
+                    STABLE_URL + 'v' + latestVersion + '/' + getBinURL(latestVersion)
+                  )
+                }
+              >
+                Install
+              </Button>
             </Box>
           )}
           {latestDevVersion && (
@@ -295,10 +341,43 @@ const DownloadUpload = () => {
                 {LL.DOWNLOAD(1)}
               </Link>
               )
+              <Button
+                sx={{ ml: 2 }}
+                size="small"
+                startIcon={<WarningIcon color="warning" />}
+                variant="outlined"
+                color="primary"
+                onClick={() =>
+                  installFirmwareURL(DEV_URL + getBinURL(latestDevVersion))
+                }
+              >
+                Install
+              </Button>
             </Box>
           )}
         </Box>
 
+        <Dialog sx={dialogStyle} open={showWaiting}>
+          {/* TODO translate all this text*/}
+          <DialogTitle>Uploading</DialogTitle>
+          <DialogContent dividers>
+            <Typography sx={{ ml: 2, flexGrow: 1 }} color="warning.main">
+              Please wait while the firmware is being uploaded and installed. This
+              can take a few minutes. EMS-ESP will automatically restart when
+              completed.
+            </Typography>
+            <Box
+              display="flex"
+              alignItems="center"
+              justifyContent="center"
+              flexDirection="column"
+              padding={2}
+            >
+              <CircularProgress sx={{ margin: 4 }} size={36} />
+            </Box>
+          </DialogContent>
+        </Dialog>
+
         <Typography sx={{ pt: 2, pb: 2 }} variant="h6" color="primary">
           {LL.UPLOAD()}
         </Typography>

lib/framework/UploadFileService.cpp

@@ -16,6 +16,7 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager *
     : _securityManager(securityManager)
     , _is_firmware(false)
     , _md5() {
+    // end-points
     server->on(
         UPLOAD_FILE_PATH,
         HTTP_POST,
@@ -23,6 +24,10 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager *
         [this](AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) {
             handleUpload(request, filename, index, data, len, final);
         });
+
+    server->on(UPLOAD_URL_PATH,
+               securityManager->wrapCallback([this](AsyncWebServerRequest * request, JsonVariant json) { uploadURL(request, json); },
+                                             AuthenticationPredicates::IS_AUTHENTICATED));
 }
 
 void UploadFileService::handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) {
@@ -113,7 +118,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
 }
 
 void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
-    // did we complete uploading a json file?
+    // did we just complete uploading a json file?
     if (request->_tempFile) {
         request->_tempFile.close(); // close the file handle as the upload is now done
         emsesp::EMSESP::system_.store_nvs_values();
@@ -166,4 +171,42 @@ void UploadFileService::handleError(AsyncWebServerRequest * request, int code) {
 void UploadFileService::handleEarlyDisconnect() {
     _is_firmware = false;
     Update.abort();
-}
+}
+
+// upload firmware from a URL, like GitHub Release assets, Cloudflare R2 or Amazon S3
+void UploadFileService::uploadURL(AsyncWebServerRequest * request, JsonVariant json) {
+    if (json.is<JsonObject>()) {
+        String url = json["url"].as<String>();
+
+        // TODO fix this from WDT crashing
+        // calling from "test upload" in a console, it works
+        // but via the web it crashes with the error message:
+        //  E (253289) task_wdt: Task watchdog got triggered. The following tasks did not reset the watchdog in time:
+        //  E (253289) task_wdt:  - async_tcp (CPU 0/1)
+        //  E (253289) task_wdt: Tasks currently running:
+        //  E (253289) task_wdt: CPU 0: ipc0
+        //  E (253289) task_wdt: CPU 1: loopTask
+        //  E (253289) task_wdt: Aborting.
+        //
+        // I think we need to stop all async services before uploading the firmware. Like MQTT?
+
+        // force close the connection
+        request->client()->close(true);
+
+        // start the upload
+        if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) {
+            emsesp::EMSESP::system_.upload_status(false); // tell ems-esp we're not uploading anymore
+        }
+
+        /*
+        if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) {
+            emsesp::EMSESP::system_.upload_status(false);
+            handleError(request, 500); // internal error, failed
+        } else {
+            request->onDisconnect(RestartService::restartNow);
+            AsyncWebServerResponse * response = request->beginResponse(200);
+            request->send(response);
+        }
+        */
+    }
+}

lib/framework/UploadFileService.h

@@ -13,7 +13,9 @@
 #include <array>
 
 #define UPLOAD_FILE_PATH "/rest/uploadFile"
-#define TEMP_FILENAME_PATH "/tmp_upload"
+#define UPLOAD_URL_PATH "/rest/uploadURL"
+
+#define TEMP_FILENAME_PATH "/tmp_upload" // for uploaded json files
 
 class UploadFileService {
   public:
@@ -27,6 +29,8 @@ class UploadFileService {
     void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final);
     void uploadComplete(AsyncWebServerRequest * request);
     void handleError(AsyncWebServerRequest * request, int code);
+    void uploadURL(AsyncWebServerRequest * request, JsonVariant json);
+
     void handleEarlyDisconnect();
 };
 

mock-api/rest_server.ts

@@ -4666,10 +4666,15 @@ router
   .get(EMSESP_GET_SETTINGS_ENDPOINT, () => emsesp_info)
   .get(EMSESP_GET_CUSTOMIZATIONS_ENDPOINT, () => emsesp_deviceentities_1)
   .get(EMSESP_GET_ENTITIES_ENDPOINT, () => emsesp_customentities)
-  .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule);
+  .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule)
 
-// API which are usually POST for security
+  // upload URL
-router
+  .post('/rest/uploadURL', () => {
+    console.log('upload File from URL');
+    return status(200);
+  })
+
+  // API which are usually POST for security
   .post(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info)
   .get(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info)
   .post(API_ENDPOINT_ROOT, async (request: any) => {

src/system.cpp

@@ -48,6 +48,8 @@
 #include <esp_mac.h>
 #endif
 
+#include <HTTPClient.h>
+
 namespace emsesp {
 
 // Languages supported. Note: the order is important and must match locale_translations.h
@@ -504,11 +506,12 @@ void System::start() {
 
 // button single click
 void System::button_OnClick(PButton & b) {
-    LOG_NOTICE("Button pressed - single click - show settings folders");
+    LOG_NOTICE("Button pressed - single click");
 
 #if defined(EMSESP_TEST)
 #ifndef EMSESP_STANDALONE
-    Test::listDir(LittleFS, FS_CONFIG_DIRECTORY, 3);
+    // show filesystem
+    Test::listDir(LittleFS, "/", 3);
 #endif
 #endif
 }
@@ -1150,7 +1153,7 @@ bool System::check_restore() {
                 LOG_ERROR("Unrecognized file uploaded");
             }
         } else {
-            LOG_ERROR("Unrecognized file uploaded, not json");
+            LOG_ERROR("Unrecognized file uploaded, not json. Will be removed.");
         }
 
         // close (just in case) and remove the temp file
@@ -1809,6 +1812,7 @@ bool System::ntp_connected() {
     return ntp_connected_;
 }
 
+// see if its a BBQKees Gateway by checking the nvs values
 String System::getBBQKeesGatewayDetails() {
 #ifndef EMSESP_STANDALONE
     if (!EMSESP::nvs_.isKey("mfg")) {
@@ -1829,4 +1833,88 @@ String System::getBBQKeesGatewayDetails() {
 #endif
 }
 
+// Stream from an URL and send straight to OTA uploader
+bool System::uploadFirmwareURL(const char * url) {
+#ifndef EMSESP_STANDALONE
+    // configure temporary server and url
+    HTTPClient http;
+    http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub
+    http.useHTTP10(true);
+    http.begin(String(url));
+
+    // start connection
+    int httpCode = http.GET();
+    if (httpCode != HTTP_CODE_OK) {
+        LOG_ERROR("Firmware upload failed - HTTP code %u", httpCode);
+        return false;
+    }
+
+    // check we have enough space
+    int firmware_size = http.getSize();
+    LOG_INFO("Firmware uploading (file: %s, size: %d bytes)", url, firmware_size);
+    if (!Update.begin(firmware_size)) {
+        LOG_ERROR("Firmware upload failed - no space");
+        return false;
+    }
+
+    EMSESP::system_.upload_status(true); // tell EMS-ESP we're uploading
+
+    // get tcp stream and send it to Updater
+    WiFiClient * stream = http.getStreamPtr();
+    if (Update.writeStream(*stream) != firmware_size) {
+        LOG_ERROR("Firmware upload failed - size differences");
+        return false;
+    }
+
+    if (!Update.end(true)) {
+        LOG_ERROR("Firmware upload error");
+        return false;
+    }
+
+    http.end();
+
+    LOG_INFO("Firmware uploaded successfully. Restarting...");
+
+    restart_requested(true); // not sure this is needed?
+
+#endif
+
+    return true; // OK
+
+    /*
+    TODO backup code to save firmware to LittleFS first, in case of slow networks
+
+    // create buffer for reading in 128 byte chunks
+    const size_t buffer_size = 1024;
+    uint8_t buff[buffer_size] = {0};
+
+    File file = LittleFS.open("/new_firmware", "w"); // TODO find new name
+    if (!file) {
+        Serial.println("Failed to open file for writing");
+    return false;
+    }
+
+    // read all data from server
+    while (http.connected() && (firmware_size > 0 || firmware_size == -1)) {
+        // get available data size
+        size_t size = stream->available();
+
+        if (size) {
+            // read up to 128 byte
+            int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size));
+
+            file.write(buff, c);
+            if (firmware_size > 0) {
+                firmware_size -= c;
+            }
+        }
+        yield();
+        // delay(1); // so not to hurt WTD or timeout
+    }
+
+    file.close();
+
+    */
+}
+
 } // namespace emsesp

src/system.h

@@ -100,6 +100,8 @@ class System {
 
     String getBBQKeesGatewayDetails();
 
+    static bool uploadFirmwareURL(const char * url);
+
     void led_init(bool refresh);
     void network_init(bool refresh);
     void button_init(bool refresh);

src/test/test.cpp

@@ -408,11 +408,16 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
         ok = true;
     }
 
-// THESE ONLY WORK WITH AN ESP32, not in standalone mode
+// THESE ONLY WORK WITH AN ESP32, not in standalone/native mode
 #ifndef EMSESP_STANDALONE
     if (command == "ls") {
         listDir(LittleFS, "/", 3);
-        Serial.println();
+        ok = true;
+    }
+
+    if (command == "upload") {
+        // S3 has 16MB flash
+        EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32S3-16MB+.bin"); // TODO remove
         ok = true;
     }
 #endif
@@ -2278,7 +2283,9 @@ void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) {
             Serial.print(" DIR: ");
             Serial.println(file.name());
             if (levels) {
-                listDir(fs, file.name(), levels - 1);
+                // prefix a / to the name to make it a full path
+                listDir(fs, ("/" + String(file.name())).c_str(), levels - 1);
+                // listDir(fs, file.name(), levels - 1);
             }
             Serial.println();
         } else {

src/test/test.h

@@ -58,6 +58,8 @@ namespace emsesp {
 // #define EMSESP_DEBUG_DEFAULT "custom"
 // #define EMSESP_DEBUG_DEFAULT "scheduler"
 // #define EMSESP_DEBUG_DEFAULT "heat_exchange"
+// #define EMSESP_DEBUG_DEFAULT "ls"
+#define EMSESP_DEBUG_DEFAULT "upload"
 
 #ifndef EMSESP_DEBUG_DEFAULT
 #define EMSESP_DEBUG_DEFAULT "general"

src/version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.7.0-dev.31"
+#define EMSESP_APP_VERSION "3.7.0-dev.32"