Merge branch 'dev' of https://github.com/emsesp/EMS-ESP32 into dev

.gitignore

@@ -47,9 +47,6 @@ interface/src/i18n/i18n-types.ts
 interface/src/i18n/i18n-util.ts
 interface/src/i18n/i18n-util.sync.ts
 interface/src/i18n/i18n-util.async.ts
-# mock-api uploads
-*/uploads/*
-!uploads/README.md
 
 # scripts
 test.sh

interface/package.json

@@ -21,7 +21,7 @@
     "lint": "eslint . --fix"
   },
   "dependencies": {
-    "@alova/adapter-xhr": "2.0.4",
+    "@alova/adapter-xhr": "2.0.5",
     "@emotion/react": "^11.13.0",
     "@emotion/styled": "^11.13.0",
     "@mui/icons-material": "^5.16.7",

interface/src/api/system.ts

@@ -45,3 +45,6 @@ export const uploadFile = (file: File) => {
     timeout: 60000 // override timeout for uploading firmware - 1 minute
   });
 };
+
+export const uploadURL = (data: { url: string }) =>
+  alovaInstance.Post('/rest/uploadURL', data);

interface/src/app/main/Sensors.tsx

@@ -59,7 +59,9 @@ const Sensors = () => {
   const [analogDialogOpen, setAnalogDialogOpen] = useState<boolean>(false);
   const [creating, setCreating] = useState<boolean>(false);
 
-  const { data: sensorData } = useAutoRequest(() => readSensorData(), {
+  const { data: sensorData, send: fetchSensorData } = useAutoRequest(
+    () => readSensorData(),
+    {
       initialData: {
         ts: [],
         as: [],
@@ -67,7 +69,8 @@ const Sensors = () => {
         platform: 'ESP32'
       },
       pollingTime: 2000
-  });
+    }
+  );
 
   const { send: sendTemperatureSensor } = useRequest(
     (data: WriteTemperatureSensor) => writeTemperatureSensor(data),
@@ -256,6 +259,7 @@ const Sensors = () => {
 
   const onTemperatureDialogClose = () => {
     setTemperatureDialogOpen(false);
+    void fetchSensorData();
   };
 
   const onTemperatureDialogSave = async (ts: TemperatureSensor) => {
@@ -269,6 +273,7 @@ const Sensors = () => {
       .finally(() => {
         setTemperatureDialogOpen(false);
         setSelectedTemperatureSensor(undefined);
+        void fetchSensorData();
       });
   };
 
@@ -283,6 +288,7 @@ const Sensors = () => {
 
   const onAnalogDialogClose = () => {
     setAnalogDialogOpen(false);
+    void fetchSensorData();
   };
 
   const addAnalogSensor = () => {
@@ -322,6 +328,7 @@ const Sensors = () => {
       .finally(() => {
         setAnalogDialogOpen(false);
         setSelectedAnalogSensor(undefined);
+        void fetchSensorData();
       });
   };
 

interface/src/app/settings/DownloadUpload.tsx

@@ -3,6 +3,7 @@ import { toast } from 'react-toastify';
 
 import DownloadIcon from '@mui/icons-material/GetApp';
 import PowerSettingsNewIcon from '@mui/icons-material/PowerSettingsNew';
+import WarningIcon from '@mui/icons-material/Warning';
 import { Box, Button, Divider, Link, Typography } from '@mui/material';
 
 import * as SystemApi from 'api/system';
@@ -15,6 +16,7 @@ import {
 } from 'api/app';
 import { getDevVersion, getStableVersion } from 'api/system';
 
+import { dialogStyle } from 'CustomTheme';
 import { useRequest } from 'alova/client';
 import type { APIcall } from 'app/main/types';
 import RestartMonitor from 'app/status/RestartMonitor';
@@ -72,6 +74,13 @@ const DownloadUpload = () => {
     error
   } = useRequest(SystemApi.readHardwareStatus);
 
+  const { send: sendUploadURL } = useRequest(
+    (data: { url: string }) => SystemApi.uploadURL(data),
+    {
+      immediate: false
+    }
+  );
+
   const { send: restartCommand } = useRequest(SystemApi.restart(), {
     immediate: false
   });
@@ -87,14 +96,19 @@ const DownloadUpload = () => {
   };
 
   // called immediately to get the latest version, on page load
-  // set immediate to false to avoid calling the API on page load and GH blocking while testing!
   const { data: latestVersion } = useRequest(getStableVersion, {
     immediate: true
-    // immediate: false
+    // uncomment for testing
+    // https://github.com/emsesp/EMS-ESP32/releases/download/v3.6.5/EMS-ESP-3_6_5-ESP32-16MB+.bin
+    // immediate: false,
+    // initialData: '3.6.5'
   });
   const { data: latestDevVersion } = useRequest(getDevVersion, {
     immediate: true
-    // immediate: false
+    // uncomment for testing
+    // https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32-16MB+.bin
+    // immediate: false,
+    // initialData: '3.7.0-dev.31'
   });
 
   const STABLE_URL = 'https://github.com/emsesp/EMS-ESP32/releases/download/';
@@ -115,6 +129,13 @@ const DownloadUpload = () => {
     );
   };
 
+  const installFirmwareURL = async (url: string) => {
+    await sendUploadURL({ url: url }).catch((error: Error) => {
+      toast.error(error.message);
+    });
+    setRestarting(true);
+  };
+
   const saveFile = (json: unknown, filename: string) => {
     const anchor = document.createElement('a');
     anchor.href = URL.createObjectURL(
@@ -275,6 +296,20 @@ const DownloadUpload = () => {
                 {LL.DOWNLOAD(1)}
               </Link>
               )
+              <Button
+                sx={{ ml: 2 }}
+                size="small"
+                startIcon={<WarningIcon color="warning" />}
+                variant="outlined"
+                color="primary"
+                onClick={() =>
+                  installFirmwareURL(
+                    STABLE_URL + 'v' + latestVersion + '/' + getBinURL(latestVersion)
+                  )
+                }
+              >
+                Install
+              </Button>
             </Box>
           )}
           {latestDevVersion && (
@@ -295,6 +330,18 @@ const DownloadUpload = () => {
                 {LL.DOWNLOAD(1)}
               </Link>
               )
+              <Button
+                sx={{ ml: 2 }}
+                size="small"
+                startIcon={<WarningIcon color="warning" />}
+                variant="outlined"
+                color="primary"
+                onClick={() =>
+                  installFirmwareURL(DEV_URL + getBinURL(latestDevVersion))
+                }
+              >
+                Install
+              </Button>
             </Box>
           )}
         </Box>
@@ -326,7 +373,13 @@ const DownloadUpload = () => {
   };
 
   return (
-    <SectionContent>{restarting ? <RestartMonitor /> : content()}</SectionContent>
+    <SectionContent>
+      {restarting ? (
+        <RestartMonitor message="Please wait while the firmware is being uploaded and installed. This can take a few minutes. EMS-ESP will automatically restart when completed." />
+      ) : (
+        content()
+      )}
+    </SectionContent>
   );
 };
 

interface/src/app/status/RestartMonitor.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from 'react';
+import { type FC, useEffect, useRef, useState } from 'react';
 
 import * as SystemApi from 'api/system';
 
@@ -6,10 +6,14 @@ import { useRequest } from 'alova/client';
 import { FormLoader } from 'components';
 import { useI18nContext } from 'i18n/i18n-react';
 
-const RESTART_TIMEOUT = 2 * 60 * 1000;
+const RESTART_TIMEOUT = 2 * 60 * 1000; // 2 minutes
-const POLL_INTERVAL = 1000;
+const POLL_INTERVAL = 1000; // every 1 second
 
-const RestartMonitor = () => {
+export interface RestartMonitorProps {
+  message?: string;
+}
+
+const RestartMonitor: FC<RestartMonitorProps> = ({ message }) => {
   const [failed, setFailed] = useState<boolean>(false);
   const [timeoutId, setTimeoutId] = useState<NodeJS.Timeout>();
   const { LL } = useI18nContext();
@@ -38,7 +42,7 @@ const RestartMonitor = () => {
 
   return (
     <FormLoader
-      message={LL.APPLICATION_RESTARTING() + '...'}
+      message={message ? message : LL.APPLICATION_RESTARTING() + '...'}
       errorMessage={failed ? 'Timed out' : undefined}
     />
   );

interface/src/app/status/Status.tsx

@@ -306,7 +306,7 @@ const SystemStatus = () => {
   );
 
   const content = () => {
-    if (!data) {
+    if (!data || !LL) {
       return <FormLoader onRetry={loadData} errorMessage={error?.message} />;
     }
 
@@ -321,7 +321,7 @@ const SystemStatus = () => {
             </ListItemAvatar>
             <ListItemText
               primary={LL.EMS_ESP_VER()}
-              secondary={data.emsesp_version}
+              secondary={'' + data.emsesp_version}
             />
             {me.admin && (
               <Button

interface/src/types/system.ts

@@ -24,7 +24,6 @@ export interface HardwareStatus {
   psram: boolean;
   psram_size?: number;
   free_psram?: number;
-
   free_caps: number;
   model: string;
 }

interface/yarn.lock

@@ -5,14 +5,14 @@ __metadata:
   version: 8
   cacheKey: 10c0
 
-"@alova/adapter-xhr@npm:2.0.4":
+"@alova/adapter-xhr@npm:2.0.5":
-  version: 2.0.4
+  version: 2.0.5
-  resolution: "@alova/adapter-xhr@npm:2.0.4"
+  resolution: "@alova/adapter-xhr@npm:2.0.5"
   dependencies:
     "@alova/shared": "npm:^1.0.4"
   peerDependencies:
-    alova: ^3.0.5
+    alova: ^3.0.9
-  checksum: 10c0/c68f51b83c75844cf2cddc47d6e08fa7a6f0af83cc6213b60d0d8ac32988f8e7b44f321b1f926fa282a2cda02757ce3cbfe94e86f140fd025ab22f0dfee9bed9
+  checksum: 10c0/9c48689e7dd081843726c4d8a93abbb7aca8b4654c606e67b81ab23de42542d5c17f2fbff7b65d4b75c7d484b42a74e76ba77e61446021620a256051eb82496c
   languageName: node
   linkType: hard
 
@@ -1664,7 +1664,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "EMS-ESP@workspace:."
   dependencies:
-    "@alova/adapter-xhr": "npm:2.0.4"
+    "@alova/adapter-xhr": "npm:2.0.5"
     "@babel/core": "npm:^7.25.2"
     "@emotion/react": "npm:^11.13.0"
     "@emotion/styled": "npm:^11.13.0"

lib/framework/UploadFileService.cpp

@@ -16,6 +16,7 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager *
     : _securityManager(securityManager)
     , _is_firmware(false)
     , _md5() {
+    // end-points
     server->on(
         UPLOAD_FILE_PATH,
         HTTP_POST,
@@ -23,6 +24,10 @@ UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager *
         [this](AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) {
             handleUpload(request, filename, index, data, len, final);
         });
+
+    server->on(UPLOAD_URL_PATH,
+               securityManager->wrapCallback([this](AsyncWebServerRequest * request, JsonVariant json) { uploadURL(request, json); },
+                                             AuthenticationPredicates::IS_AUTHENTICATED));
 }
 
 void UploadFileService::handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) {
@@ -113,7 +118,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri
 }
 
 void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
-    // did we complete uploading a json file?
+    // did we just complete uploading a json file?
     if (request->_tempFile) {
         request->_tempFile.close(); // close the file handle as the upload is now done
         emsesp::EMSESP::system_.store_nvs_values();
@@ -167,3 +172,15 @@ void UploadFileService::handleEarlyDisconnect() {
     _is_firmware = false;
     Update.abort();
 }
+
+// upload firmware from a URL, like GitHub Release assets, Cloudflare R2 or Amazon S3
+void UploadFileService::uploadURL(AsyncWebServerRequest * request, JsonVariant json) {
+    if (json.is<JsonObject>()) {
+        // this will keep a copy of the URL, but won't initiate the download yet
+        emsesp::EMSESP::system_.uploadFirmwareURL(json["url"].as<const char *>());
+
+        // end the connection
+        AsyncWebServerResponse * response = request->beginResponse(200);
+        request->send(response);
+    }
+}

lib/framework/UploadFileService.h

@@ -13,7 +13,9 @@
 #include <array>
 
 #define UPLOAD_FILE_PATH "/rest/uploadFile"
-#define TEMP_FILENAME_PATH "/tmp_upload"
+#define UPLOAD_URL_PATH "/rest/uploadURL"
+
+#define TEMP_FILENAME_PATH "/tmp_upload" // for uploaded json files
 
 class UploadFileService {
   public:
@@ -27,6 +29,8 @@ class UploadFileService {
     void handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final);
     void uploadComplete(AsyncWebServerRequest * request);
     void handleError(AsyncWebServerRequest * request, int code);
+    void uploadURL(AsyncWebServerRequest * request, JsonVariant json);
+
     void handleEarlyDisconnect();
 };
 

mock-api/mockServer.js

@@ -34,9 +34,10 @@ export default () => {
       server.middlewares.use(async (req, res, next) => {
         // catch any file uploads
         if (req.url.startsWith('/rest/uploadFile')) {
+          // show progress
           let progress = 0;
           const file_size = req.headers['content-length'];
-
+          console.log('File size: ' + file_size);
           req.on('data', async (chunk) => {
             progress += chunk.length;
             const percentage = (progress / file_size) * 100;
@@ -50,7 +51,7 @@ export default () => {
           try {
             [fields, files] = await form.parse(req);
           } catch (err) {
-            console.error(err);
+            console.error('Not json form content');
             res.writeHead(err.httpCode || 400, {
               'Content-Type': 'text/plain'
             });

mock-api/rest_server.ts

@@ -4666,10 +4666,15 @@ router
   .get(EMSESP_GET_SETTINGS_ENDPOINT, () => emsesp_info)
   .get(EMSESP_GET_CUSTOMIZATIONS_ENDPOINT, () => emsesp_deviceentities_1)
   .get(EMSESP_GET_ENTITIES_ENDPOINT, () => emsesp_customentities)
-  .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule);
+  .get(EMSESP_GET_SCHEDULE_ENDPOINT, () => emsesp_schedule)
+
+  // upload URL
+  .post('/rest/uploadURL', () => {
+    console.log('upload File from URL');
+    return status(200);
+  })
 
   // API which are usually POST for security
-router
   .post(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info)
   .get(EMSESP_SYSTEM_INFO_ENDPOINT, () => emsesp_info)
   .post(API_ENDPOINT_ROOT, async (request: any) => {

src/emsesp.cpp

@@ -1671,11 +1671,13 @@ void EMSESP::loop() {
         mqtt_.loop();              // sends out anything in the MQTT queue
         webModulesService.loop();  // loop through the external library modules
         if (system_.PSram() == 0) {
-            webSchedulerService.loop();
+            webSchedulerService.loop(); // run non-async if there is no PSRAM available
         }
 
         // force a query on the EMS devices to fetch latest data at a set interval (1 min)
         scheduled_fetch_values();
+    } else {
+        emsesp::EMSESP::system_.uploadFirmwareURL(); // start an upload from a URL. This is blocking.
     }
 
     uuid::loop();

src/system.cpp

@@ -48,6 +48,8 @@
 #include <esp_mac.h>
 #endif
 
+#include <HTTPClient.h>
+
 namespace emsesp {
 
 // Languages supported. Note: the order is important and must match locale_translations.h
@@ -504,11 +506,12 @@ void System::start() {
 
 // button single click
 void System::button_OnClick(PButton & b) {
-    LOG_NOTICE("Button pressed - single click - show settings folders");
+    LOG_NOTICE("Button pressed - single click");
 
 #if defined(EMSESP_TEST)
 #ifndef EMSESP_STANDALONE
-    Test::listDir(LittleFS, FS_CONFIG_DIRECTORY, 3);
+    // show filesystem
+    Test::listDir(LittleFS, "/", 3);
 #endif
 #endif
 }
@@ -1150,7 +1153,7 @@ bool System::check_restore() {
                 LOG_ERROR("Unrecognized file uploaded");
             }
         } else {
-            LOG_ERROR("Unrecognized file uploaded, not json");
+            LOG_ERROR("Unrecognized file uploaded, not json. Will be removed.");
         }
 
         // close (just in case) and remove the temp file
@@ -1809,6 +1812,7 @@ bool System::ntp_connected() {
     return ntp_connected_;
 }
 
+// see if its a BBQKees Gateway by checking the nvs values
 String System::getBBQKeesGatewayDetails() {
 #ifndef EMSESP_STANDALONE
     if (!EMSESP::nvs_.isKey("mfg")) {
@@ -1829,4 +1833,76 @@ String System::getBBQKeesGatewayDetails() {
 #endif
 }
 
+// Stream from an URL and send straight to OTA uploader service.
+//
+// This function needs to be called twice, once with a url to persist it, and second with no arguments to start the upload
+// This is to avoid timeouts in callback functions, like calling from a web hook.
+bool System::uploadFirmwareURL(const char * url) {
+#ifndef EMSESP_STANDALONE
+
+    static String saved_url;
+
+    // if the URL is not empty, save it for later
+    if (url && strlen(url) > 0) {
+        saved_url = url;
+        EMSESP::system_.upload_status(true); // tell EMS-ESP we're ready to start the uploading process
+        return true;
+    }
+
+    // make sure we have a valid URL
+    if (saved_url.isEmpty()) {
+        LOG_ERROR("Firmware upload failed - no URL");
+        return false;
+    }
+
+    // Configure temporary client
+    HTTPClient http;
+    http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); // important for GitHub 302's
+    http.useHTTP10(true);
+    http.begin(saved_url);
+
+    // start a connection
+    int httpCode = http.GET();
+    if (httpCode != HTTP_CODE_OK) {
+        LOG_ERROR("Firmware upload failed - HTTP code %u", httpCode);
+        return false;
+    }
+
+    // check we have enough space for the upload in the ota partition
+    int firmware_size = http.getSize();
+    LOG_INFO("Firmware uploading (file: %s, size: %d bytes). Please wait...", saved_url.c_str(), firmware_size);
+    if (!Update.begin(firmware_size)) {
+        LOG_ERROR("Firmware upload failed - no space");
+        return false;
+    }
+
+    // flush buffers so latest log messages are shown
+    Shell::loop_all();
+
+    // get tcp stream and send it to Updater
+    WiFiClient * stream = http.getStreamPtr();
+    if (Update.writeStream(*stream) != firmware_size) {
+        LOG_ERROR("Firmware upload failed - size differences");
+        return false;
+    }
+
+    if (!Update.end(true)) {
+        LOG_ERROR("Firmware upload failed - general error");
+        return false;
+    }
+
+    http.end();
+
+    EMSESP::system_.upload_status(false);
+    saved_url.clear(); // prevent from downloading again
+
+    LOG_INFO("Firmware uploaded successfully. Restarting...");
+
+    restart_requested(true);
+
+#endif
+
+    return true; // OK
+}
+
 } // namespace emsesp

src/system.h

@@ -100,6 +100,8 @@ class System {
 
     String getBBQKeesGatewayDetails();
 
+    static bool uploadFirmwareURL(const char * url = nullptr);
+
     void led_init(bool refresh);
     void network_init(bool refresh);
     void button_init(bool refresh);

src/test/test.cpp

@@ -408,11 +408,16 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
         ok = true;
     }
 
-// THESE ONLY WORK WITH AN ESP32, not in standalone mode
+// THESE ONLY WORK WITH AN ESP32, not in standalone/native mode
 #ifndef EMSESP_STANDALONE
     if (command == "ls") {
         listDir(LittleFS, "/", 3);
-        Serial.println();
+        ok = true;
+    }
+
+    if (command == "upload") {
+        // S3 has 16MB flash
+        EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_31-ESP32S3-16MB+.bin"); // TODO remove
         ok = true;
     }
 #endif
@@ -2278,7 +2283,9 @@ void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) {
             Serial.print(" DIR: ");
             Serial.println(file.name());
             if (levels) {
-                listDir(fs, file.name(), levels - 1);
+                // prefix a / to the name to make it a full path
+                listDir(fs, ("/" + String(file.name())).c_str(), levels - 1);
+                // listDir(fs, file.name(), levels - 1);
             }
             Serial.println();
         } else {

src/test/test.h

@@ -58,6 +58,8 @@ namespace emsesp {
 // #define EMSESP_DEBUG_DEFAULT "custom"
 // #define EMSESP_DEBUG_DEFAULT "scheduler"
 // #define EMSESP_DEBUG_DEFAULT "heat_exchange"
+// #define EMSESP_DEBUG_DEFAULT "ls"
+#define EMSESP_DEBUG_DEFAULT "upload"
 
 #ifndef EMSESP_DEBUG_DEFAULT
 #define EMSESP_DEBUG_DEFAULT "general"

src/version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.7.0-dev.31"
+#define EMSESP_APP_VERSION "3.7.0-dev.32"

src/web/WebStatusService.cpp

@@ -31,6 +31,12 @@ WebStatusService::WebStatusService(AsyncWebServer * server, SecurityManager * se
 }
 
 void WebStatusService::systemStatus(AsyncWebServerRequest * request) {
+    // This is a litle trick for the OTA upload. We don't want the React RestartService to think we're finished
+    // with the upload so we fake it and pretent the /rest/systemStatus is not available. That way the spinner keeps spinning.
+    if (EMSESP::system_.upload_status()) {
+        return; // ignore endpoint
+    }
+
     EMSESP::system_.refreshHeapMem(); // refresh free heap and max alloc heap
 
     auto *     response = new AsyncJsonResponse(false);