make it work from web (thanks michael)

.gitignore

@@ -47,9 +47,6 @@ interface/src/i18n/i18n-types.ts
 interface/src/i18n/i18n-util.ts
 interface/src/i18n/i18n-util.sync.ts
 interface/src/i18n/i18n-util.async.ts
-# mock-api uploads
-*/uploads/*
-!uploads/README.md
 
 # scripts
 test.sh

interface/src/app/settings/DownloadUpload.tsx

@@ -4,17 +4,7 @@ import { toast } from 'react-toastify';
 import DownloadIcon from '@mui/icons-material/GetApp';
 import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew';
 import WarningIcon from '@mui/icons-material/Warning';
-import {
+import { Box, Button, Divider, Link, Typography } from '@mui/material';
-  Box,
-  Button,
-  CircularProgress,
-  Dialog,
-  DialogContent,
-  DialogTitle,
-  Divider,
-  Link,
-  Typography
-} from '@mui/material';
 
 import * as SystemApi from 'api/system';
 import {
@@ -44,7 +34,6 @@ const DownloadUpload = () => {
 
   const [restarting, setRestarting] = useState<boolean>(false);
   const [restartNeeded, setRestartNeeded] = useState<boolean>(false);
-  const [showWaiting, setShowWaiting] = useState<boolean>(false);
 
   const { send: sendSettings } = useRequest(getSettings(), {
     immediate: false
@@ -141,10 +130,10 @@ const DownloadUpload = () => {
   };
 
   const installFirmwareURL = async (url: string) => {
-    setShowWaiting(true);
     await sendUploadURL({ url: url }).catch((error: Error) => {
       toast.error(error.message);
     });
+    setRestarting(true);
   };
 
   const saveFile = (json: unknown, filename: string) => {
@@ -357,27 +346,6 @@ const DownloadUpload = () => {
           )}
         </Box>
 
-        <Dialog sx={dialogStyle} open={showWaiting}>
-          {/* TODO translate all this text*/}
-          <DialogTitle>Uploading</DialogTitle>
-          <DialogContent dividers>
-            <Typography sx={{ ml: 2, flexGrow: 1 }} color="warning.main">
-              Please wait while the firmware is being uploaded and installed. This
-              can take a few minutes. EMS-ESP will automatically restart when
-              completed.
-            </Typography>
-            <Box
-              display="flex"
-              alignItems="center"
-              justifyContent="center"
-              flexDirection="column"
-              padding={2}
-            >
-              <CircularProgress sx={{ margin: 4 }} size={36} />
-            </Box>
-          </DialogContent>
-        </Dialog>
-
         <Typography sx={{ pt: 2, pb: 2 }} variant="h6" color="primary">
           {LL.UPLOAD()}
         </Typography>
@@ -405,7 +373,13 @@ const DownloadUpload = () => {
   };
 
   return (
-    <SectionContent>{restarting ? <RestartMonitor /> : content()}</SectionContent>
+    <SectionContent>
+      {restarting ? (
+        <RestartMonitor message="Please wait while the firmware is being uploaded and installed. This can take a few minutes. EMS-ESP will automatically restart when completed." />
+      ) : (
+        content()
+      )}
+    </SectionContent>
   );
 };
 

interface/src/app/status/RestartMonitor.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from 'react';
+import { type FC, useEffect, useRef, useState } from 'react';
 
 import * as SystemApi from 'api/system';
 
@@ -6,10 +6,14 @@ import { useRequest } from 'alova/client';
 import { FormLoader } from 'components';
 import { useI18nContext } from 'i18n/i18n-react';
 
-const RESTART_TIMEOUT = 2 * 60 * 1000;
+const RESTART_TIMEOUT = 2 * 60 * 1000; // 2 minutes
-const POLL_INTERVAL = 1000;
+const POLL_INTERVAL = 1000; // every 1 second
 
-const RestartMonitor = () => {
+export interface RestartMonitorProps {
+  message?: string;
+}
+
+const RestartMonitor: FC<RestartMonitorProps> = ({ message }) => {
   const [failed, setFailed] = useState<boolean>(false);
   const [timeoutId, setTimeoutId] = useState<NodeJS.Timeout>();
   const { LL } = useI18nContext();
@@ -38,7 +42,7 @@ const RestartMonitor = () => {
 
   return (
     <FormLoader
-      message={LL.APPLICATION_RESTARTING() + '...'}
+      message={message ? message : LL.APPLICATION_RESTARTING() + '...'}
       errorMessage={failed ? 'Timed out' : undefined}
     />
   );

lib/framework/UploadFileService.cpp

@@ -176,37 +176,11 @@ void UploadFileService::handleEarlyDisconnect() {
 // upload firmware from a URL, like GitHub Release assets, Cloudflare R2 or Amazon S3
 void UploadFileService::uploadURL(AsyncWebServerRequest * request, JsonVariant json) {
     if (json.is<JsonObject>()) {
-        String url = json["url"].as<String>();
+        // this will keep a copy of the URL, but won't initiate the download yet
+        emsesp::EMSESP::system_.uploadFirmwareURL(json["url"].as<const char *>());
 
-        // TODO fix this from WDT crashing
+        // end the connection
-        // calling from "test upload" in a console, it works
+        AsyncWebServerResponse * response = request->beginResponse(200);
-        // but via the web it crashes with the error message:
+        request->send(response);
-        //  E (253289) task_wdt: Task watchdog got triggered. The following tasks did not reset the watchdog in time:
-        //  E (253289) task_wdt:  - async_tcp (CPU 0/1)
-        //  E (253289) task_wdt: Tasks currently running:
-        //  E (253289) task_wdt: CPU 0: ipc0
-        //  E (253289) task_wdt: CPU 1: loopTask
-        //  E (253289) task_wdt: Aborting.
-        //
-        // I think we need to stop all async services before uploading the firmware. Like MQTT?
-
-        // force close the connection
-        request->client()->close(true);
-
-        // start the upload
-        if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) {
-            emsesp::EMSESP::system_.upload_status(false); // tell ems-esp we're not uploading anymore
-        }
-
-        /*
-        if (!emsesp::EMSESP::system_.uploadFirmwareURL(url.c_str())) {
-            emsesp::EMSESP::system_.upload_status(false);
-            handleError(request, 500); // internal error, failed
-        } else {
-            request->onDisconnect(RestartService::restartNow);
-            AsyncWebServerResponse * response = request->beginResponse(200);
-            request->send(response);
-        }
-        */
     }
 }

src/emsesp.cpp

@@ -1671,11 +1671,13 @@ void EMSESP::loop() {
         mqtt_.loop();              // sends out anything in the MQTT queue
         webModulesService.loop();  // loop through the external library modules
         if (system_.PSram() == 0) {
-            webSchedulerService.loop();
+            webSchedulerService.loop(); // run non-async if there is no PSRAM available
         }
 
         // force a query on the EMS devices to fetch latest data at a set interval (1 min)
         scheduled_fetch_values();
+    } else {
+        emsesp::EMSESP::system_.uploadFirmwareURL(); // start an upload from a URL. This is blocking.
     }
 
     uuid::loop();

src/system.cpp

@@ -1833,31 +1833,51 @@ String System::getBBQKeesGatewayDetails() {
 #endif
 }
 
-// Stream from an URL and send straight to OTA uploader
+// Stream from an URL and send straight to OTA uploader service.
+//
+// This function needs to be called twice, once with a url to persist it, and second with no arguments to start the upload
+// This is to avoid timeouts in callback functions, like calling from a web hook.
 bool System::uploadFirmwareURL(const char * url) {
 #ifndef EMSESP_STANDALONE
-    // configure temporary server and url
-    HTTPClient http;
-    http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub
-    http.useHTTP10(true);
-    http.begin(String(url));
 
-    // start connection
+    static String saved_url;
+
+    // if the URL is not empty, save it for later
+    if (url && strlen(url) > 0) {
+        saved_url = url;
+        EMSESP::system_.upload_status(true); // tell EMS-ESP we're ready to start the uploading process
+        return true;
+    }
+
+    // make sure we have a valid URL
+    if (saved_url.isEmpty()) {
+        LOG_ERROR("Firmware upload failed - no URL");
+        return false;
+    }
+
+    // Configure temporary client
+    HTTPClient http;
+    http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub 302's
+    http.useHTTP10(true);
+    http.begin(saved_url);
+
+    // start a connection
     int httpCode = http.GET();
     if (httpCode != HTTP_CODE_OK) {
         LOG_ERROR("Firmware upload failed - HTTP code %u", httpCode);
         return false;
     }
 
-    // check we have enough space
+    // check we have enough space for the upload in the ota partition
     int firmware_size = http.getSize();
-    LOG_INFO("Firmware uploading (file: %s, size: %d bytes)", url, firmware_size);
+    LOG_INFO("Firmware uploading (file: %s, size: %d bytes). Please wait...", saved_url.c_str(), firmware_size);
     if (!Update.begin(firmware_size)) {
         LOG_ERROR("Firmware upload failed - no space");
         return false;
     }
 
-    EMSESP::system_.upload_status(true); // tell EMS-ESP we're uploading
+    // flush buffers so latest log messages are shown
+    Shell::loop_all();
 
     // get tcp stream and send it to Updater
     WiFiClient * stream = http.getStreamPtr();
@@ -1867,54 +1887,22 @@ bool System::uploadFirmwareURL(const char * url) {
     }
 
     if (!Update.end(true)) {
-        LOG_ERROR("Firmware upload error");
+        LOG_ERROR("Firmware upload failed - general error");
         return false;
     }
 
     http.end();
 
+    EMSESP::system_.upload_status(false);
+    saved_url.clear(); // prevent from downloading again
+
     LOG_INFO("Firmware uploaded successfully. Restarting...");
 
-    restart_requested(true); // not sure this is needed?
+    restart_requested(true);
 
 #endif
 
     return true; // OK
-
-    /*
-    TODO backup code to save firmware to LittleFS first, in case of slow networks
-
-    // create buffer for reading in 128 byte chunks
-    const size_t buffer_size = 1024;
-    uint8_t buff[buffer_size] = {0};
-
-    File file = LittleFS.open("/new_firmware", "w"); // TODO find new name
-    if (!file) {
-        Serial.println("Failed to open file for writing");
-    return false;
-    }
-
-    // read all data from server
-    while (http.connected() && (firmware_size > 0 || firmware_size == -1)) {
-        // get available data size
-        size_t size = stream->available();
-
-        if (size) {
-            // read up to 128 byte
-            int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size));
-
-            file.write(buff, c);
-            if (firmware_size > 0) {
-                firmware_size -= c;
-            }
-        }
-        yield();
-        // delay(1); // so not to hurt WTD or timeout
-    }
-
-    file.close();
-
-    */
 }
 
 } // namespace emsesp

src/system.h

@@ -100,7 +100,7 @@ class System {
 
     String getBBQKeesGatewayDetails();
 
-    static bool uploadFirmwareURL(const char * url);
+    static bool uploadFirmwareURL(const char * url = nullptr);
 
     void led_init(bool refresh);
     void network_init(bool refresh);

src/web/WebStatusService.cpp

@@ -31,6 +31,12 @@ WebStatusService::WebStatusService(AsyncWebServer * server, SecurityManager * se
 }
 
 void WebStatusService::systemStatus(AsyncWebServerRequest * request) {
+    // This is a litle trick for the OTA upload. We don't want the React RestartService to think we're finished
+    // with the upload so we fake it and pretent the /rest/systemStatus is not available. That way the spinner keeps spinning.
+    if (EMSESP::system_.upload_status()) {
+        return; // ignore endpoint
+    }
+
     EMSESP::system_.refreshHeapMem(); // refresh free heap and max alloc heap
 
     auto *     response = new AsyncJsonResponse(false);