Merge pull request #494 from proddy/dev

Feature: upload customization settings from a file #256

CHANGELOG_LATEST.md

@@ -40,6 +40,7 @@
 - API fetch individual attributes from an entity [#462](https://github.com/emsesp/EMS-ESP32/issues/462)
 - Option to disable mDNS
 - Option for rendering booleans on dashboard [#456](https://github.com/emsesp/EMS-ESP32/issues/456)
+- Upload customization settings from a file [#256](https://github.com/emsesp/EMS-ESP32/issues/256)
 
 ### Fixed
 

interface/package-lock.json

@@ -29,7 +29,7 @@
         "react": "^17.0.2",
         "react-app-rewired": "^2.2.1",
         "react-dom": "^17.0.2",
-        "react-dropzone": "^14.2.0",
+        "react-dropzone": "^14.2.1",
         "react-icons": "^4.3.1",
         "react-router-dom": "^6.3.0",
         "react-scripts": "5.0.1",
@@ -14437,9 +14437,9 @@
       }
     },
     "node_modules/react-dropzone": {
-      "version": "14.2.0",
+      "version": "14.2.1",
-      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.1.tgz",
-      "integrity": "sha512-D7AXPtRba8rd7DBOejh3W2v1Uax6i7XKPYPuMr13XFPfnDcPHHvlEfp3raVpdj3XMHlRfYuf2H5+m8p7mlgKdQ==",
+      "integrity": "sha512-jzX6wDtAjlfwZ+Fbg+G17EszWUkQVxhMTWMfAC9qSUq7II2pKglHA8aarbFKl0mLpRPDaNUcy+HD/Sf4gkf76Q==",
       "dependencies": {
         "attr-accept": "^2.2.2",
         "file-selector": "^0.6.0",
@@ -27793,9 +27793,9 @@
       }
     },
     "react-dropzone": {
-      "version": "14.2.0",
+      "version": "14.2.1",
-      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.1.tgz",
-      "integrity": "sha512-D7AXPtRba8rd7DBOejh3W2v1Uax6i7XKPYPuMr13XFPfnDcPHHvlEfp3raVpdj3XMHlRfYuf2H5+m8p7mlgKdQ==",
+      "integrity": "sha512-jzX6wDtAjlfwZ+Fbg+G17EszWUkQVxhMTWMfAC9qSUq7II2pKglHA8aarbFKl0mLpRPDaNUcy+HD/Sf4gkf76Q==",
       "requires": {
         "attr-accept": "^2.2.2",
         "file-selector": "^0.6.0",

interface/package.json

@@ -25,7 +25,7 @@
     "react": "^17.0.2",
     "react-app-rewired": "^2.2.1",
     "react-dom": "^17.0.2",
-    "react-dropzone": "^14.2.0",
+    "react-dropzone": "^14.2.1",
     "react-icons": "^4.3.1",
     "react-router-dom": "^6.3.0",
     "react-scripts": "5.0.1",

interface/src/AppRouting.tsx

@@ -47,10 +47,7 @@ const AppRouting: FC = () => {
       <RemoveTrailingSlashes />
       <Routes>
         <Route path="/unauthorized" element={<RootRedirect message="Please sign in to continue" signOut />} />
-        <Route
+        <Route path="/fileUpdated" element={<RootRedirect message="Upload successful" variant="success" />} />
-          path="/firmwareUpdated"
-          element={<RootRedirect message="Firmware update successful" variant="success" />}
-        />
         {features.security && (
           <Route
             path="/"

interface/src/api/endpoints.ts

@@ -92,7 +92,7 @@ export interface FileUploadConfig {
   onUploadProgress?: (progressEvent: ProgressEvent) => void;
 }
 
-export const uploadFile = (url: string, file: File, config?: FileUploadConfig): AxiosPromise<void> => {
+export const startUploadFile = (url: string, file: File, config?: FileUploadConfig): AxiosPromise<void> => {
   const formData = new FormData();
   formData.append('file', file);
 

interface/src/api/system.ts

@@ -2,7 +2,7 @@ import { AxiosPromise } from 'axios';
 
 import { OTASettings, SystemStatus, LogSettings, LogEntries } from '../types';
 
-import { AXIOS, AXIOS_BIN, FileUploadConfig, uploadFile } from './endpoints';
+import { AXIOS, AXIOS_BIN, FileUploadConfig, startUploadFile } from './endpoints';
 
 export function readSystemStatus(timeout?: number): AxiosPromise<SystemStatus> {
   return AXIOS.get('/systemStatus', { timeout });
@@ -24,8 +24,8 @@ export function updateOTASettings(otaSettings: OTASettings): AxiosPromise<OTASet
   return AXIOS.post('/otaSettings', otaSettings);
 }
 
-export const uploadFirmware = (file: File, config?: FileUploadConfig): AxiosPromise<void> =>
+export const uploadFile = (file: File, config?: FileUploadConfig): AxiosPromise<void> =>
-  uploadFile('/uploadFirmware', file, config);
+  startUploadFile('/uploadFile', file, config);
 
 export function readLogSettings(): AxiosPromise<LogSettings> {
   return AXIOS.get('/logSettings');

interface/src/components/upload/SingleUpload.tsx

@@ -32,7 +32,8 @@ const SingleUpload: FC<SingleUploadProps> = ({ onDrop, onCancel, uploading, prog
   const dropzoneState = useDropzone({
     onDrop,
     accept: {
-      'application/octet-stream': ['.bin']
+      'application/octet-stream': ['.bin'],
+      'application/json': ['.json']
     },
     disabled: uploading,
     multiple: false

interface/src/components/upload/useFileUpload.ts

@@ -42,7 +42,7 @@ const useFileUpload = ({ upload }: MediaUploadOptions) => {
         cancelToken: cancelToken.token
       });
       resetUploadingStates();
-      enqueueSnackbar('Upload successful', { variant: 'success' });
+      enqueueSnackbar('File uploaded', { variant: 'success' });
     } catch (error: unknown) {
       if (axios.isCancel(error)) {
         enqueueSnackbar('Upload aborted', { variant: 'warning' });

interface/src/framework/security/ManageUsersForm.tsx

@@ -83,14 +83,13 @@ const ManageUsersForm: FC = () => {
     const noAdminConfigured = () => !data.users.find((u) => u.admin);
 
     const removeUser = (toRemove: User) => {
-      const users = data.users.filter((u) => u.id !== toRemove.id);
+      const users = data.users.filter((u) => u.username !== toRemove.username);
       setData({ ...data, users });
     };
 
     const createUser = () => {
       setCreating(true);
       setUser({
-        id: '',
         username: '',
         password: '',
         admin: true
@@ -108,7 +107,7 @@ const ManageUsersForm: FC = () => {
 
     const doneEditingUser = () => {
       if (user) {
-        const users = [...data.users.filter((u) => u.id !== user.id), user];
+        const users = [...data.users.filter((u) => u.username !== user.username), user];
         setData({ ...data, users });
         setUser(undefined);
       }
@@ -118,8 +117,8 @@ const ManageUsersForm: FC = () => {
       setGeneratingToken(undefined);
     };
 
-    const generateToken = (id: string) => {
+    const generateToken = (username: string) => {
-      setGeneratingToken(id);
+      setGeneratingToken(username);
     };
 
     const onSubmit = async () => {
@@ -127,9 +126,11 @@ const ManageUsersForm: FC = () => {
       authenticatedContext.refresh();
     };
 
+    const user_table = data.users.map((u) => ({ ...u, id: u.username }));
+
     return (
       <>
-        <Table data={{ nodes: data.users }} theme={table_theme}>
+        <Table data={{ nodes: user_table }} theme={table_theme}>
           {(tableList: any) => (
             <>
               <Header>
@@ -140,16 +141,16 @@ const ManageUsersForm: FC = () => {
                 </HeaderRow>
               </Header>
               <Body>
-                {tableList.map((u: User, index: number) => (
+                {tableList.map((u: any) => (
                   <Row key={u.id} item={u}>
-                    <Cell>{u.id}</Cell>
+                    <Cell>{u.username}</Cell>
                     <Cell>{u.admin ? <CheckIcon /> : <CloseIcon />}</Cell>
                     <Cell>
                       <IconButton
                         size="small"
                         disabled={!authenticatedContext.me.admin}
                         aria-label="Generate Token"
-                        onClick={() => generateToken(u.id)}
+                        onClick={() => generateToken(u.username)}
                       >
                         <VpnKeyIcon />
                       </IconButton>

interface/src/framework/system/FirmwareFileUpload.tsx

@@ -1,33 +0,0 @@
-import { AxiosPromise } from 'axios';
-import { FC } from 'react';
-
-import { FileUploadConfig } from '../../api/endpoints';
-import { MessageBox, SingleUpload, useFileUpload } from '../../components';
-
-interface UploadFirmwareProps {
-  uploadFirmware: (file: File, config?: FileUploadConfig) => AxiosPromise<void>;
-}
-
-const FirmwareFileUpload: FC<UploadFirmwareProps> = ({ uploadFirmware }) => {
-  const [uploadFile, cancelUpload, uploading, uploadProgress] = useFileUpload({ upload: uploadFirmware });
-
-  return (
-    <>
-      {!uploading && (
-        <MessageBox
-          message="Upload a new firmware (.bin) file below to replace the existing firmware"
-          level="warning"
-          my={2}
-        />
-      )}
-      <SingleUpload
-        onDrop={uploadFile}
-        onCancel={cancelUpload}
-        uploading={uploading}
-        progress={uploadProgress}
-      />
-    </>
-  );
-};
-
-export default FirmwareFileUpload;

interface/src/framework/system/GeneralFileUpload.tsx

@@ -0,0 +1,28 @@
+import { AxiosPromise } from 'axios';
+import { FC } from 'react';
+
+import { FileUploadConfig } from '../../api/endpoints';
+import { MessageBox, SingleUpload, useFileUpload } from '../../components';
+
+interface UploadFileProps {
+  uploadGeneralFile: (file: File, config?: FileUploadConfig) => AxiosPromise<void>;
+}
+
+const GeneralFileUpload: FC<UploadFileProps> = ({ uploadGeneralFile }) => {
+  const [uploadFile, cancelUpload, uploading, uploadProgress] = useFileUpload({ upload: uploadGeneralFile });
+
+  return (
+    <>
+      {!uploading && (
+        <MessageBox
+          message="Upload a new firmware (.bin) file or an exported settings or customizations (.json) file below. EMS-ESP will restart afterwards to apply the new changes."
+          level="warning"
+          my={2}
+        />
+      )}
+      <SingleUpload onDrop={uploadFile} onCancel={cancelUpload} uploading={uploading} progress={uploadProgress} />
+    </>
+  );
+};
+
+export default GeneralFileUpload;

interface/src/framework/system/RestartMonitor.tsx

@@ -8,7 +8,7 @@ const RESTART_TIMEOUT = 2 * 60 * 1000;
 const POLL_TIMEOUT = 2000;
 const POLL_INTERVAL = 5000;
 
-const FirmwareRestartMonitor: FC = () => {
+const RestartMonitor: FC = () => {
   const [failed, setFailed] = useState<boolean>(false);
   const [timeoutId, setTimeoutId] = useState<NodeJS.Timeout>();
 
@@ -16,7 +16,7 @@ const FirmwareRestartMonitor: FC = () => {
   const poll = useRef(async () => {
     try {
       await SystemApi.readSystemStatus(POLL_TIMEOUT);
-      document.location.href = '/firmwareUpdated';
+      document.location.href = '/fileUpdated';
     } catch (error: unknown) {
       if (new Date().getTime() < timeoutAt.current) {
         setTimeoutId(setTimeout(poll.current, POLL_INTERVAL));
@@ -40,4 +40,4 @@ const FirmwareRestartMonitor: FC = () => {
   );
 };
 
-export default FirmwareRestartMonitor;
+export default RestartMonitor;

interface/src/framework/system/System.tsx

@@ -6,7 +6,7 @@ import { Tab } from '@mui/material';
 import { useRouterTab, RouterTabs, useLayoutTitle, RequireAdmin } from '../../components';
 import { AuthenticatedContext } from '../../contexts/authentication';
 import { FeaturesContext } from '../../contexts/features';
-import UploadFirmwareForm from './UploadFirmwareForm';
+import UploadFileForm from './UploadFileForm';
 import SystemStatusForm from './SystemStatusForm';
 import OTASettingsForm from './OTASettingsForm';
 
@@ -26,7 +26,7 @@ const System: FC = () => {
         <Tab value="log" label="System Log" />
 
         {features.ota && <Tab value="ota" label="OTA Settings" disabled={!me.admin} />}
-        {features.upload_firmware && <Tab value="upload" label="Upload Firmware" disabled={!me.admin} />}
+        {features.upload_firmware && <Tab value="upload" label="Upload" disabled={!me.admin} />}
       </RouterTabs>
       <Routes>
         <Route path="status" element={<SystemStatusForm />} />
@@ -46,7 +46,7 @@ const System: FC = () => {
             path="upload"
             element={
               <RequireAdmin>
-                <UploadFirmwareForm />
+                <UploadFileForm />
               </RequireAdmin>
             }
           />

interface/src/framework/system/SystemStatusForm.tsx

@@ -159,7 +159,7 @@ const SystemStatusForm: FC = () => {
             <Typography variant="body2">
               Use&nbsp;
               <Link target="_blank" href={uploadURL} color="primary">
-                {'UPLOAD FIRMWARE'}
+                {'UPLOAD'}
               </Link>
               &nbsp;to apply the new firmware
             </Typography>

interface/src/framework/system/UploadFileForm.tsx

@@ -0,0 +1,26 @@
+import { FC, useRef, useState } from 'react';
+
+import * as SystemApi from '../../api/system';
+import { SectionContent } from '../../components';
+import { FileUploadConfig } from '../../api/endpoints';
+
+import GeneralFileUpload from './GeneralFileUpload';
+import RestartMonitor from './RestartMonitor';
+
+const UploadFileForm: FC = () => {
+  const [restarting, setRestarting] = useState<boolean>();
+
+  const uploadFile = useRef(async (file: File, config?: FileUploadConfig) => {
+    const response = await SystemApi.uploadFile(file, config);
+    setRestarting(true);
+    return response;
+  });
+
+  return (
+    <SectionContent title="Upload File" titleGutter>
+      {restarting ? <RestartMonitor /> : <GeneralFileUpload uploadGeneralFile={uploadFile.current} />}
+    </SectionContent>
+  );
+};
+
+export default UploadFileForm;

interface/src/framework/system/UploadFirmwareForm.tsx

@@ -1,26 +0,0 @@
-import { FC, useRef, useState } from 'react';
-
-import * as SystemApi from '../../api/system';
-import { SectionContent } from '../../components';
-import { FileUploadConfig } from '../../api/endpoints';
-
-import FirmwareFileUpload from './FirmwareFileUpload';
-import FirmwareRestartMonitor from './FirmwareRestartMonitor';
-
-const UploadFirmwareForm: FC = () => {
-  const [restarting, setRestarting] = useState<boolean>();
-
-  const uploadFirmware = useRef(async (file: File, config?: FileUploadConfig) => {
-    const response = await SystemApi.uploadFirmware(file, config);
-    setRestarting(true);
-    return response;
-  });
-
-  return (
-    <SectionContent title="Upload Firmware" titleGutter>
-      {restarting ? <FirmwareRestartMonitor /> : <FirmwareFileUpload uploadFirmware={uploadFirmware.current} />}
-    </SectionContent>
-  );
-};
-
-export default UploadFirmwareForm;

interface/src/project/HelpInformation.tsx

@@ -2,7 +2,7 @@ import { FC, useContext } from 'react';
 
 import { Typography, Button, Box, List, ListItem, ListItemText, Link, ListItemAvatar } from '@mui/material';
 
-import { SectionContent, ButtonRow } from '../components';
+import { SectionContent, ButtonRow, MessageBox } from '../components';
 
 import { AuthenticatedContext } from '../contexts/authentication';
 
@@ -36,7 +36,7 @@ const HelpInformation: FC = () => {
       } else {
         const json = response.data;
         const a = document.createElement('a');
-        const filename = 'emsesp_' + endpoint + '.txt';
+        const filename = 'emsesp_' + endpoint + '.json';
         a.href = URL.createObjectURL(
           new Blob([JSON.stringify(json, null, 2)], {
             type: 'text/plain'
@@ -102,9 +102,13 @@ const HelpInformation: FC = () => {
             <GitHubIcon />
           </ListItemAvatar>
           <ListItemText>
-            To report an issue or request a feature, please do via&nbsp;
+            To report an issue or request a feature, please&nbsp;
+            <Link component="button" variant="body1" onClick={() => onDownload('info')}>
+              download
+            </Link>
+            &nbsp;the debug information and include in a new&nbsp;
             <Link target="_blank" href="https://github.com/emsesp/EMS-ESP32/issues/new/choose" color="primary">
-              {'GitHub'}
+              GitHub issue
             </Link>
           </ListItemText>
         </ListItem>
@@ -112,26 +116,17 @@ const HelpInformation: FC = () => {
 
       {me.admin && (
         <>
-          <Typography sx={{ pt: 2 }} variant="h6" color="primary">
+          <Typography sx={{ pt: 2, pb: 2 }} variant="h6" color="primary">
-            Export Data
+            Download Settings
           </Typography>
           <Box color="warning.main">
             <Typography variant="body2">
-              Download the current system information, application settings and any customizations using the buttons
+              Export the application settings and any customizations to a JSON file. These files can later be uploaded
-              below.
+              via System&rarr;Upload.
             </Typography>
           </Box>
-
           <Box sx={{ display: 'flex' }}>
             <ButtonRow>
-              <Button
-                startIcon={<DownloadIcon />}
-                variant="outlined"
-                color="secondary"
-                onClick={() => onDownload('info')}
-              >
-                system info
-              </Button>
               <Button
                 startIcon={<DownloadIcon />}
                 variant="outlined"
@@ -150,6 +145,12 @@ const HelpInformation: FC = () => {
               </Button>
             </ButtonRow>
           </Box>
+          <MessageBox
+            my={2}
+            level="warning"
+            message="Be careful when sharing your Settings as the file contains passwords and other sensitive system
+              information!"
+          />
         </>
       )}
 
@@ -157,7 +158,7 @@ const HelpInformation: FC = () => {
         <Typography align="center" variant="h6">
           EMS-ESP is a free and open-source project.
           <br></br>Please consider supporting us by giving it a&nbsp;
-          <StarIcon style={{ color: '#fdff3a' }} /> on&nbsp;
+          <StarIcon style={{ fontSize: 16, color: '#fdff3a', verticalAlign: 'middle' }} /> on&nbsp;
           <Link href="https://github.com/emsesp/EMS-ESP32" color="primary">
             {'GitHub'}
           </Link>

interface/src/types/security.ts

@@ -1,5 +1,4 @@
 export interface User {
-  id: string; // needed for Table
   username: string;
   password: string;
   admin: boolean;

interface/src/utils/endpoints.ts

@@ -2,9 +2,9 @@ import { AxiosError } from 'axios';
 
 export const extractErrorMessage = (error: unknown, defaultMessage: string) => {
   if (error instanceof AxiosError) {
-    return error.response && error.response.data && error?.response?.data?.message;
+    return defaultMessage + ' (' + error.request.statusText + ')';
   } else if (error instanceof Error) {
-    return error.message;
+    return defaultMessage + ' (' + error.message + ')';
   }
   return defaultMessage;
 };

lib/framework/ESP8266React.cpp

@@ -13,7 +13,7 @@ ESP8266React::ESP8266React(AsyncWebServer * server, FS * fs)
     , _ntpSettingsService(server, fs, &_securitySettingsService)
     , _ntpStatus(server, &_securitySettingsService)
     , _otaSettingsService(server, fs, &_securitySettingsService)
-    , _uploadFirmwareService(server, &_securitySettingsService)
+    , _uploadFileService(server, &_securitySettingsService)
     , _mqttSettingsService(server, fs, &_securitySettingsService)
     , _mqttStatus(server, &_mqttSettingsService, &_securitySettingsService)
     , _authenticationService(server, &_securitySettingsService)

lib/framework/ESP8266React.h

@@ -16,7 +16,7 @@
 #include <NTPSettingsService.h>
 #include <NTPStatus.h>
 #include <OTASettingsService.h>
-#include <UploadFirmwareService.h>
+#include <UploadFileService.h>
 #include <RestartService.h>
 #include <SecuritySettingsService.h>
 #include <SystemStatus.h>
@@ -78,7 +78,7 @@ class ESP8266React {
     NTPSettingsService      _ntpSettingsService;
     NTPStatus               _ntpStatus;
     OTASettingsService      _otaSettingsService;
-    UploadFirmwareService   _uploadFirmwareService;
+    UploadFileService       _uploadFileService;
     MqttSettingsService     _mqttSettingsService;
     MqttStatus              _mqttStatus;
     AuthenticationService   _authenticationService;

lib/framework/FSPersistence.h

@@ -30,12 +30,12 @@ class FSPersistence {
             DynamicJsonDocument  jsonDocument = DynamicJsonDocument(_bufferSize);
             DeserializationError error        = deserializeJson(jsonDocument, settingsFile);
             if (error == DeserializationError::Ok && jsonDocument.is<JsonObject>()) {
-                // jsonDocument.shrinkToFit(); // added by proddy
                 JsonObject jsonObject = jsonDocument.as<JsonObject>();
 
 // debug added by Proddy
 #if defined(EMSESP_DEBUG)
 #if defined(EMSESP_USE_SERIAL)
+                Serial.println();
                 Serial.printf("Reading file: %s: ", _filePath);
                 serializeJson(jsonDocument, Serial);
                 Serial.println();
@@ -49,9 +49,17 @@ class FSPersistence {
             settingsFile.close();
         }
 
-        // If we reach here we have not been successful in loading the config,
+// If we reach here we have not been successful in loading the config,
-        // hard-coded emergency defaults are now applied.
+// hard-coded emergency defaults are now applied.
+#if defined(EMSESP_DEBUG)
+#if defined(EMSESP_USE_SERIAL)
+        Serial.println();
+        Serial.printf("Applying defaults for %s: ", _filePath);
+        Serial.println();
+#endif
+#endif
         applyDefaults();
+        writeToFS(); // added to make sure the initial file is created
     }
 
     bool writeToFS() {

lib/framework/Features.h

@@ -30,7 +30,7 @@
 #define FT_OTA 1
 #endif
 
-// upload firmware feature on by default
+// upload firmware/file feature on by default
 #ifndef FT_UPLOAD_FIRMWARE
 #define FT_UPLOAD_FIRMWARE 1
 #endif

lib/framework/SecuritySettingsService.h

@@ -43,7 +43,6 @@ class SecuritySettings {
         JsonArray users = root.createNestedArray("users");
         for (User user : settings.users) {
             JsonObject userRoot  = users.createNestedObject();
-            userRoot["id"]       = user.username; // for React Table
             userRoot["username"] = user.username;
             userRoot["password"] = user.password;
             userRoot["admin"]    = user.admin;

lib/framework/UploadFileService.cpp

@@ -0,0 +1,122 @@
+#include <UploadFileService.h>
+
+using namespace std::placeholders; // for `_1` etc
+
+static bool is_firmware = false;
+
+UploadFileService::UploadFileService(AsyncWebServer * server, SecurityManager * securityManager)
+    : _securityManager(securityManager) {
+    server->on(UPLOAD_FILE_PATH,
+               HTTP_POST,
+               std::bind(&UploadFileService::uploadComplete, this, _1),
+               std::bind(&UploadFileService::handleUpload, this, _1, _2, _3, _4, _5, _6));
+}
+
+void UploadFileService::handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) {
+    // quit if not authorized
+    Authentication authentication = _securityManager->authenticateRequest(request);
+    if (!AuthenticationPredicates::IS_ADMIN(authentication)) {
+        handleError(request, 403); // send the forbidden response
+        return;
+    }
+
+    // at init
+    if (!index) {
+        // check details of the file, to see if its a valid bin or json file
+        std::string fname(filename.c_str());
+        auto        position  = fname.find_last_of(".");
+        std::string extension = fname.substr(position + 1);
+        size_t      fsize     = request->contentLength();
+
+        Serial.printf("Received filename: %s, len: %d, index: %d, ext: %s, fsize: %d", filename.c_str(), len, index, extension.c_str(), fsize);
+        Serial.println();
+
+        if ((extension == "bin") && (fsize > 1500000)) {
+            is_firmware = true;
+        } else if (extension == "json") {
+            is_firmware = false;
+        } else {
+            is_firmware = false;
+            return; // not support file type
+        }
+
+        if (is_firmware) {
+            // it's firmware - initialize the ArduinoOTA updater
+            if (Update.begin(fsize)) {
+                request->onDisconnect(UploadFileService::handleEarlyDisconnect); // success, let's make sure we end the update if the client hangs up
+            } else {
+#if defined(EMSESP_USE_SERIAL)
+                Update.printError(Serial);
+#endif
+                handleError(request, 500); // failed to begin, send an error response
+            }
+        } else {
+            // its a normal file, open a new temp file to write the contents too
+            request->_tempFile = LITTLEFS.open(TEMP_FILENAME_PATH, "w");
+        }
+    }
+
+    if (!is_firmware) {
+        if (len) {
+            request->_tempFile.write(data, len); // stream the incoming chunk to the opened file
+        }
+
+    } else {
+        // if we haven't delt with an error, continue with the firmware update
+        if (!request->_tempObject) {
+            if (Update.write(data, len) != len) {
+#if defined(EMSESP_USE_SERIAL)
+                Update.printError(Serial);
+#endif
+                handleError(request, 500);
+            }
+            if (final) {
+                if (!Update.end(true)) {
+#if defined(EMSESP_USE_SERIAL)
+                    Update.printError(Serial);
+#endif
+                    handleError(request, 500);
+                }
+            }
+        }
+    }
+}
+
+void UploadFileService::uploadComplete(AsyncWebServerRequest * request) {
+    // did we complete uploading a json file?
+    if (request->_tempFile) {
+        request->_tempFile.close(); // close the file handle as the upload is now done
+        request->onDisconnect(RestartService::restartNow);
+        AsyncWebServerResponse * response = request->beginResponse(200);
+        request->send(response);
+        return;
+    }
+
+    // check if it was a firmware upgrade
+    // if no error, send the success response
+    if (is_firmware && !request->_tempObject) {
+        request->onDisconnect(RestartService::restartNow);
+        AsyncWebServerResponse * response = request->beginResponse(200);
+        request->send(response);
+        return;
+    }
+
+    handleError(request, 403); // send the forbidden response
+}
+
+void UploadFileService::handleError(AsyncWebServerRequest * request, int code) {
+    // if we have had an error already, do nothing
+    if (request->_tempObject) {
+        return;
+    }
+
+    // send the error code to the client and record the error code in the temp object
+    request->_tempObject              = new int(code);
+    AsyncWebServerResponse * response = request->beginResponse(code);
+    request->send(response);
+}
+
+void UploadFileService::handleEarlyDisconnect() {
+    is_firmware = false;
+    Update.abort();
+}

lib/framework/UploadFileService.h

@@ -1,20 +1,23 @@
-#ifndef UploadFirmwareService_h
+#ifndef UploadFileService_h
-#define UploadFirmwareService_h
+#define UploadFileService_h
 
 #include <Arduino.h>
 
 #include <Update.h>
 #include <WiFi.h>
 
+#include <LITTLEFS.h>
+
 #include <ESPAsyncWebServer.h>
 #include <SecurityManager.h>
 #include <RestartService.h>
 
-#define UPLOAD_FIRMWARE_PATH "/rest/uploadFirmware"
+#define UPLOAD_FILE_PATH "/rest/uploadFile"
+#define TEMP_FILENAME_PATH "/tmp_upload"
 
-class UploadFirmwareService {
+class UploadFileService {
   public:
-    UploadFirmwareService(AsyncWebServer * server, SecurityManager * securityManager);
+    UploadFileService(AsyncWebServer * server, SecurityManager * securityManager);
 
   private:
     SecurityManager * _securityManager;

lib/framework/UploadFirmwareService.cpp

@@ -1,68 +0,0 @@
-#include <UploadFirmwareService.h>
-
-using namespace std::placeholders; // for `_1` etc
-
-UploadFirmwareService::UploadFirmwareService(AsyncWebServer * server, SecurityManager * securityManager)
-    : _securityManager(securityManager) {
-    server->on(UPLOAD_FIRMWARE_PATH,
-               HTTP_POST,
-               std::bind(&UploadFirmwareService::uploadComplete, this, _1),
-               std::bind(&UploadFirmwareService::handleUpload, this, _1, _2, _3, _4, _5, _6));
-}
-
-void UploadFirmwareService::handleUpload(AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t * data, size_t len, bool final) {
-    if (!index) {
-        Authentication authentication = _securityManager->authenticateRequest(request);
-        if (AuthenticationPredicates::IS_ADMIN(authentication)) {
-            if (Update.begin(request->contentLength())) {
-                // success, let's make sure we end the update if the client hangs up
-                request->onDisconnect(UploadFirmwareService::handleEarlyDisconnect);
-            } else {
-                // failed to begin, send an error response
-                Update.printError(Serial);
-                handleError(request, 500);
-            }
-        } else {
-            // send the forbidden response
-            handleError(request, 403);
-        }
-    }
-
-    // if we haven't delt with an error, continue with the update
-    if (!request->_tempObject) {
-        if (Update.write(data, len) != len) {
-            Update.printError(Serial);
-            handleError(request, 500);
-        }
-        if (final) {
-            if (!Update.end(true)) {
-                Update.printError(Serial);
-                handleError(request, 500);
-            }
-        }
-    }
-}
-
-void UploadFirmwareService::uploadComplete(AsyncWebServerRequest * request) {
-    // if no error, send the success response
-    if (!request->_tempObject) {
-        request->onDisconnect(RestartService::restartNow);
-        AsyncWebServerResponse * response = request->beginResponse(200);
-        request->send(response);
-    }
-}
-
-void UploadFirmwareService::handleError(AsyncWebServerRequest * request, int code) {
-    // if we have had an error already, do nothing
-    if (request->_tempObject) {
-        return;
-    }
-    // send the error code to the client and record the error code in the temp object
-    request->_tempObject              = new int(code);
-    AsyncWebServerResponse * response = request->beginResponse(code);
-    request->send(response);
-}
-
-void UploadFirmwareService::handleEarlyDisconnect() {
-    Update.abort();
-}

lib_standalone/ESP8266React.h

@@ -17,6 +17,13 @@
 #include <StatefulService.h>
 #include <Network.h>
 
+#define AP_SETTINGS_FILE "/config/apSettings.json"
+#define MQTT_SETTINGS_FILE "/config/mqttSettings.json"
+#define NETWORK_SETTINGS_FILE "/config/networkSettings.json"
+#define NTP_SETTINGS_FILE "/config/ntpSettings.json"
+#define EMSESP_SETTINGS_FILE "/config/emsespSettings.json"
+#define OTA_SETTINGS_FILE "/config/otaSettings.json"
+
 class DummySettings {
   public:
     uint8_t  tx_mode               = 1;

lib_standalone/Features.h

@@ -28,7 +28,7 @@
 #define FT_OTA 0
 #endif
 
-// upload firmware feature off by default
+// upload firmware/file feature off by default
 #ifndef FT_UPLOAD_FIRMWARE
 #define FT_UPLOAD_FIRMWARE 0
 #endif

mock-api/server.js

@@ -249,7 +249,7 @@ const SYSTEM_STATUS_ENDPOINT = REST_ENDPOINT_ROOT + 'systemStatus'
 const SECURITY_SETTINGS_ENDPOINT = REST_ENDPOINT_ROOT + 'securitySettings'
 const RESTART_ENDPOINT = REST_ENDPOINT_ROOT + 'restart'
 const FACTORY_RESET_ENDPOINT = REST_ENDPOINT_ROOT + 'factoryReset'
-const UPLOAD_FIRMWARE_ENDPOINT = REST_ENDPOINT_ROOT + 'uploadFirmware'
+const UPLOAD_FILE_ENDPOINT = REST_ENDPOINT_ROOT + 'uploadFile'
 const SIGN_IN_ENDPOINT = REST_ENDPOINT_ROOT + 'signIn'
 const GENERATE_TOKEN_ENDPOINT = REST_ENDPOINT_ROOT + 'generateToken'
 const system_status = {
@@ -270,8 +270,8 @@ const system_status = {
 security_settings = {
   jwt_secret: 'naughty!',
   users: [
-    { id: 'admin', username: 'admin', password: 'admin', admin: true },
+    { username: 'admin', password: 'admin', admin: true },
-    { id: 'guest', username: 'guest', password: 'guest', admin: false },
+    { username: 'guest', password: 'guest', admin: false },
   ],
 }
 const features = {
@@ -829,7 +829,7 @@ rest_server.post(RESTART_ENDPOINT, (req, res) => {
 rest_server.post(FACTORY_RESET_ENDPOINT, (req, res) => {
   res.sendStatus(200)
 })
-rest_server.post(UPLOAD_FIRMWARE_ENDPOINT, (req, res) => {
+rest_server.post(UPLOAD_FILE_ENDPOINT, (req, res) => {
   res.sendStatus(200)
 })
 rest_server.post(SIGN_IN_ENDPOINT, (req, res) => {

scripts/upload_fw.py

@@ -0,0 +1,10 @@
+# for calling dos upload from Window WSL2 Linux, because serial ports are not mapped yet
+# example file
+Import('env')
+from subprocess import call
+
+def upload(source, target, env):
+    print("bin file: " + str(target[0]))
+    call(["cmd.exe", "/c", "c:\\Users\\paul\\OneDrive\\Desktop\\ems-esp32.bat"])
+
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", [upload])

src/command.cpp

@@ -262,7 +262,7 @@ uint8_t Command::call(const uint8_t device_type, const char * cmd, const char *
             return CommandRet::NOT_ALLOWED; // command not allowed
         }
 
-        if (value == nullptr) {
+        if ((value == nullptr) || (strlen(value) == 0)) {
             if (EMSESP::system_.readonly_mode()) {
                 LOG_INFO(F("[readonly] Calling command '%s/%s' (%s)"), dname.c_str(), cmd, read_flash_string(cf->description_).c_str());
             } else {

src/console.cpp

@@ -661,7 +661,7 @@ void Console::load_system_commands(unsigned int context) {
                                                networkSettings.ssid = arguments.front().c_str();
                                                return StateUpdateResult::CHANGED;
                                            });
-                                           shell.println("Use `wifi reconnect` to save and apply the new settings");
+                                           shell.println("Use `wifi reconnect` to apply the new settings");
                                        });
 
     // added by mvdp

src/default_settings.h

@@ -93,11 +93,11 @@
 #endif
 
 #ifndef EMSESP_DEFAULT_ANALOG_ENABLED
-#define EMSESP_DEFAULT_ANALOG_ENABLED false
+#define EMSESP_DEFAULT_ANALOG_ENABLED true
 #endif
 
 #ifndef EMSESP_DEFAULT_TELNET_ENABLED
-#define EMSESP_DEFAULT_TELNET_ENABLED false
+#define EMSESP_DEFAULT_TELNET_ENABLED true
 #endif
 
 #ifndef EMSESP_DEFAULT_BOARD_PROFILE

src/emsesp.cpp

@@ -1374,7 +1374,7 @@ void EMSESP::start() {
 // start the file system
 #ifndef EMSESP_STANDALONE
     if (!LITTLEFS.begin(true)) {
-        Serial.println("LITTLEFS Mount Failed. EMS-ESP stopped.");
+        Serial.println("LITTLEFS Mount failed. EMS-ESP stopped.");
         return;
     }
 #endif
@@ -1383,6 +1383,12 @@ void EMSESP::start() {
     webLogService.begin(); // start web log service. now we can start capturing logs to the web log
     LOG_INFO(F("Last system reset reason Core0: %s, Core1: %s"), system_.reset_reason(0).c_str(), system_.reset_reason(1).c_str());
 
+    // do any system upgrades
+    if (system_.check_upgrade()) {
+        LOG_INFO(F("System will be restarted to apply upgrade"));
+        system_.system_restart();
+    };
+
     webSettingsService.begin();      // load EMS-ESP Application settings...
     system_.reload_settings();       // ... and store some of the settings locally
     webCustomizationService.begin(); // load the customizations
@@ -1392,8 +1398,6 @@ void EMSESP::start() {
         console_.start_telnet();
     }
 
-    system_.check_upgrade(); // do any system upgrades
-
     // start all the EMS-ESP services
     mqtt_.start();   // mqtt init
     system_.start(); // starts commands, led, adc, button, network, syslog & uart

src/system.cpp

@@ -689,7 +689,7 @@ void System::commands_init() {
 
     // these commands will return data in JSON format
     Command::add(EMSdevice::DeviceType::SYSTEM, F_(info), System::command_info, F("show system status"));
-    Command::add(EMSdevice::DeviceType::SYSTEM, F_(settings), System::command_settings, F("fetch system settings"));
+    Command::add(EMSdevice::DeviceType::SYSTEM, F_(settings), System::command_settings, F("fetch system settings"), CommandFlag::ADMIN_ONLY);
     Command::add(EMSdevice::DeviceType::SYSTEM, F_(customizations), System::command_customizations, F("fetch system customizations"));
     Command::add(EMSdevice::DeviceType::SYSTEM, F_(commands), System::command_commands, F("fetch system commands"));
 
@@ -889,10 +889,48 @@ void System::show_system(uuid::console::Shell & shell) {
 #endif
 }
 
-// upgrade from previous versions of EMS-ESP
+// handle upgrades from previous versions
-// returns true if an upgrade was done
+// or managing an uploaded files to replace settings files
+// returns true if we need a reboot
 bool System::check_upgrade() {
-    return false;
+    bool reboot_required = false;
+
+#ifndef EMSESP_STANDALONE
+    // see if we have a temp file, if so try and read it
+    File new_file = LITTLEFS.open(TEMP_FILENAME_PATH);
+    if (new_file) {
+        DynamicJsonDocument  jsonDocument = DynamicJsonDocument(FS_BUFFER_SIZE);
+        DeserializationError error        = deserializeJson(jsonDocument, new_file);
+        if (error == DeserializationError::Ok && jsonDocument.is<JsonObject>()) {
+            JsonObject input = jsonDocument.as<JsonObject>();
+            // see what type of file it is, either settings or customization. anything else is ignored
+            std::string settings_type = input["type"];
+            if (settings_type == "settings") {
+                // It's a settings file. Parse each section separately. If it's system related it will require a reboot
+                reboot_required = saveSettings(NETWORK_SETTINGS_FILE, "Network", input);
+                reboot_required |= saveSettings(AP_SETTINGS_FILE, "AP", input);
+                reboot_required |= saveSettings(MQTT_SETTINGS_FILE, "MQTT", input);
+                reboot_required |= saveSettings(NTP_SETTINGS_FILE, "NTP", input);
+                reboot_required |= saveSettings(SECURITY_SETTINGS_FILE, "Security", input);
+                reboot_required |= saveSettings(EMSESP_SETTINGS_FILE, "Settings", input);
+            } else if (settings_type == "customizations") {
+                // it's a customization file, just replace it and there's no need to reboot
+                saveSettings(EMSESP_CUSTOMIZATION_FILE, "Customizations", input);
+                return false; // no reboot required
+            } else {
+                LOG_ERROR(F("Unrecognized file uploaded"));
+            }
+        } else {
+            LOG_ERROR(F("Unrecognized file uploaded, not json"));
+        }
+
+        // close (just in case) and remove the temp file
+        new_file.close();
+        LITTLEFS.remove(TEMP_FILENAME_PATH);
+    }
+#endif
+
+    return reboot_required;
 }
 
 // list commands
@@ -900,191 +938,67 @@ bool System::command_commands(const char * value, const int8_t id, JsonObject &
     return Command::list(EMSdevice::DeviceType::SYSTEM, output);
 }
 
+// convert settings file into json object
+void System::extractSettings(const char * filename, const char * section, JsonObject & output) {
+#ifndef EMSESP_STANDALONE
+    File settingsFile = LITTLEFS.open(filename);
+    if (settingsFile) {
+        DynamicJsonDocument  jsonDocument = DynamicJsonDocument(EMSESP_JSON_SIZE_XLARGE_DYN);
+        DeserializationError error        = deserializeJson(jsonDocument, settingsFile);
+        if (error == DeserializationError::Ok && jsonDocument.is<JsonObject>()) {
+            JsonObject jsonObject = jsonDocument.as<JsonObject>();
+            JsonObject node       = output.createNestedObject(section);
+            for (JsonPair kvp : jsonObject) {
+                node[kvp.key()] = kvp.value();
+            }
+        }
+    }
+    settingsFile.close();
+#endif
+}
+
+// save settings file using input from a json object
+bool System::saveSettings(const char * filename, const char * section, JsonObject & input) {
+#ifndef EMSESP_STANDALONE
+    JsonObject section_json = input[section];
+    if (section_json) {
+        File section_file = LITTLEFS.open(filename, "w");
+        if (section_file) {
+            LOG_INFO(F("Applying new %s settings"), section);
+            serializeJson(section_json, section_file);
+            section_file.close();
+            return true; // reboot required
+        }
+    }
+#endif
+    return false; // not found
+}
+
 // export all settings to JSON text
+// we need to keep the original format so the import/upload works as we just replace files
 // http://ems-esp/api/system/settings
-// value and id are ignored
-// note: ssid and passwords are excluded
 bool System::command_settings(const char * value, const int8_t id, JsonObject & output) {
-    output["label"] = "settings";
+    output["type"] = "settings";
 
     JsonObject node = output.createNestedObject("System");
     node["version"] = EMSESP_APP_VERSION;
 
-    EMSESP::esp8266React.getNetworkSettingsService()->read([&](NetworkSettings & settings) {
+    extractSettings(NETWORK_SETTINGS_FILE, "Network", output);
-        node                     = output.createNestedObject("Network");
+    extractSettings(AP_SETTINGS_FILE, "AP", output);
-        node["hostname"]         = settings.hostname;
+    extractSettings(MQTT_SETTINGS_FILE, "MQTT", output);
-        node["static_ip_config"] = settings.staticIPConfig;
+    extractSettings(NTP_SETTINGS_FILE, "NTP", output);
-        node["enableIPv6"]       = settings.enableIPv6;
+    extractSettings(OTA_SETTINGS_FILE, "OTA", output);
-        node["low_bandwidth"]    = settings.bandwidth20;
+    extractSettings(SECURITY_SETTINGS_FILE, "Security", output);
-        node["disable_sleep"]    = settings.nosleep;
+    extractSettings(EMSESP_SETTINGS_FILE, "Settings", output);
-        JsonUtils::writeIP(node, "local_ip", settings.localIP);
-        JsonUtils::writeIP(node, "gateway_ip", settings.gatewayIP);
-        JsonUtils::writeIP(node, "subnet_mask", settings.subnetMask);
-        JsonUtils::writeIP(node, "dns_ip_1", settings.dnsIP1);
-        JsonUtils::writeIP(node, "dns_ip_2", settings.dnsIP2);
-    });
-
-#ifndef EMSESP_STANDALONE
-    EMSESP::esp8266React.getAPSettingsService()->read([&](APSettings & settings) {
-        node                   = output.createNestedObject("AP");
-        const char * pM[]      = {"always", "disconnected", "never"};
-        node["provision_mode"] = pM[settings.provisionMode];
-        node["security"]       = settings.password.length() ? "wpa2" : "open";
-        node["ssid"]           = settings.ssid;
-        node["local_ip"]       = settings.localIP.toString();
-        node["gateway_ip"]     = settings.gatewayIP.toString();
-        node["subnet_mask"]    = settings.subnetMask.toString();
-        node["channel"]        = settings.channel;
-        node["ssid_hidden"]    = settings.ssidHidden;
-        node["max_clients"]    = settings.maxClients;
-    });
-#endif
-
-    EMSESP::esp8266React.getMqttSettingsService()->read([&](MqttSettings & settings) {
-        node                            = output.createNestedObject("MQTT");
-        node["enabled"]                 = settings.enabled;
-        node["host"]                    = settings.host;
-        node["port"]                    = settings.port;
-        node["username"]                = settings.username;
-        node["client_id"]               = settings.clientId;
-        node["keep_alive"]              = settings.keepAlive;
-        node["clean_session"]           = settings.cleanSession;
-        node["base"]                    = settings.base;
-        node["discovery_prefix"]        = settings.discovery_prefix;
-        node["nested_format"]           = settings.nested_format;
-        node["ha_enabled"]              = settings.ha_enabled;
-        node["mqtt_qos"]                = settings.mqtt_qos;
-        node["mqtt_retain"]             = settings.mqtt_retain;
-        node["publish_time_boiler"]     = settings.publish_time_boiler;
-        node["publish_time_thermostat"] = settings.publish_time_thermostat;
-        node["publish_time_solar"]      = settings.publish_time_solar;
-        node["publish_time_mixer"]      = settings.publish_time_mixer;
-        node["publish_time_other"]      = settings.publish_time_other;
-        node["publish_time_sensor"]     = settings.publish_time_sensor;
-        node["publish_single"]          = settings.publish_single;
-        node["publish_2_command"]       = settings.publish_single2cmd;
-        node["send_response"]           = settings.send_response;
-    });
-
-#ifndef EMSESP_STANDALONE
-    EMSESP::esp8266React.getNTPSettingsService()->read([&](NTPSettings & settings) {
-        node              = output.createNestedObject("NTP");
-        node["enabled"]   = settings.enabled;
-        node["server"]    = settings.server;
-        node["tz_label"]  = settings.tzLabel;
-        node["tz_format"] = settings.tzFormat;
-    });
-
-    EMSESP::esp8266React.getOTASettingsService()->read([&](OTASettings & settings) {
-        node            = output.createNestedObject("OTA");
-        node["enabled"] = settings.enabled;
-        node["port"]    = settings.port;
-    });
-#endif
-
-    EMSESP::webSettingsService.read([&](WebSettings & settings) {
-        node = output.createNestedObject("Settings");
-
-        node["board_profile"] = settings.board_profile;
-        node["tx_mode"]       = settings.tx_mode;
-        node["ems_bus_id"]    = settings.ems_bus_id;
-
-        node["syslog_enabled"]       = settings.syslog_enabled;
-        node["syslog_level"]         = settings.syslog_level;
-        node["syslog_mark_interval"] = settings.syslog_mark_interval;
-        node["syslog_host"]          = settings.syslog_host;
-        node["syslog_port"]          = settings.syslog_port;
-
-        node["master_thermostat"] = settings.master_thermostat;
-
-        node["shower_timer"] = settings.shower_timer;
-        node["shower_alert"] = settings.shower_alert;
-        if (settings.shower_alert) {
-            node["shower_alert_coldshot"] = settings.shower_alert_coldshot; // seconds
-            node["shower_alert_trigger"]  = settings.shower_alert_trigger;  // minutes
-        }
-
-        node["rx_gpio"]      = settings.rx_gpio;
-        node["tx_gpio"]      = settings.tx_gpio;
-        node["dallas_gpio"]  = settings.dallas_gpio;
-        node["pbutton_gpio"] = settings.pbutton_gpio;
-        node["led_gpio"]     = settings.led_gpio;
-
-        node["hide_led"]      = settings.hide_led;
-        node["notoken_api"]   = settings.notoken_api;
-        node["readonly_mode"] = settings.readonly_mode;
-
-        node["fahrenheit"]      = settings.fahrenheit;
-        node["dallas_parasite"] = settings.dallas_parasite;
-        node["bool_format"]     = settings.bool_format;
-        node["bool_dashboard"]  = settings.bool_dashboard;
-        node["enum_format"]     = settings.enum_format;
-        node["analog_enabled"]  = settings.analog_enabled;
-        node["telnet_enabled"]  = settings.telnet_enabled;
-
-        node["phy_type"]       = settings.phy_type;
-        node["eth_power"]      = settings.eth_power;
-        node["eth_phy_addr"]   = settings.eth_phy_addr;
-        node["eth_clock_mode"] = settings.eth_clock_mode;
-    });
 
     return true;
 }
 
 // http://ems-esp/api/system/customizations
+// we need to keep the original format so the import/upload works as we just replace file
 bool System::command_customizations(const char * value, const int8_t id, JsonObject & output) {
-    output["label"] = "customizations";
+    output["type"] = "customizations";
-
+    extractSettings(EMSESP_CUSTOMIZATION_FILE, "Customizations", output);
-    JsonObject node = output.createNestedObject("Customizations");
-
-    EMSESP::webCustomizationService.read([&](WebCustomization & settings) {
-        // sensors
-        JsonArray sensorsJson = node.createNestedArray("sensors");
-        for (const auto & sensor : settings.sensorCustomizations) {
-            JsonObject sensorJson = sensorsJson.createNestedObject();
-            sensorJson["id"]      = sensor.id;     // key
-            sensorJson["name"]    = sensor.name;   // n
-            sensorJson["offset"]  = sensor.offset; // o
-        }
-
-        JsonArray analogJson = node.createNestedArray("analogs");
-        for (const AnalogCustomization & sensor : settings.analogCustomizations) {
-            JsonObject sensorJson = analogJson.createNestedObject();
-            sensorJson["gpio"]    = sensor.gpio;
-            sensorJson["name"]    = sensor.name;
-            if (EMSESP::system_.enum_format() == ENUM_FORMAT_INDEX) {
-                sensorJson["type"] = sensor.type;
-            } else {
-                sensorJson["type"] = FL_(enum_sensortype)[sensor.type];
-            }
-            if (sensor.type == AnalogSensor::AnalogType::ADC) {
-                sensorJson["offset"] = sensor.offset;
-                sensorJson["factor"] = sensor.factor;
-                sensorJson["uom"]    = EMSdevice::uom_to_string(sensor.uom);
-            } else if (sensor.type == AnalogSensor::AnalogType::COUNTER || sensor.type == AnalogSensor::AnalogType::TIMER
-                       || sensor.type == AnalogSensor::AnalogType::RATE) {
-                sensorJson["factor"] = sensor.factor;
-                sensorJson["uom"]    = EMSdevice::uom_to_string(sensor.uom);
-            } else if (sensor.type >= AnalogSensor::AnalogType::PWM_0) {
-                sensorJson["frequency"] = sensor.factor;
-                sensorJson["factor"]    = sensor.factor;
-            }
-        }
-
-        // masked entities
-        JsonArray mask_entitiesJson = node.createNestedArray("masked_entities");
-        for (const auto & entityCustomization : settings.entityCustomizations) {
-            JsonObject entityJson    = mask_entitiesJson.createNestedObject();
-            entityJson["product_id"] = entityCustomization.product_id;
-            entityJson["device_id"]  = entityCustomization.device_id;
-
-            JsonArray mask_entityJson = entityJson.createNestedArray("entities");
-            for (std::string entity_id : entityCustomization.entity_ids) {
-                mask_entityJson.add(entity_id);
-            }
-        }
-    });
-
     return true;
 }
 
@@ -1094,36 +1008,26 @@ bool System::command_info(const char * value, const int8_t id, JsonObject & outp
     JsonObject node;
 
     // System
-    node = output.createNestedObject("System");
+    node            = output.createNestedObject("System Status");
-
     node["version"] = EMSESP_APP_VERSION;
     node["uptime"]  = uuid::log::format_timestamp_ms(uuid::get_uptime_ms(), 3);
-    node["uptime (seconds)"] = uuid::get_uptime_sec();
+    // node["uptime (seconds)"] = uuid::get_uptime_sec();
-    node["network time"]     = EMSESP::system_.ntp_connected() ? "connected" : "disconnected";
-
 #ifndef EMSESP_STANDALONE
     node["freemem"] = ESP.getFreeHeap() / 1000L; // kilobytes
 #endif
     node["reset reason"] = EMSESP::system_.reset_reason(0) + " / " + EMSESP::system_.reset_reason(1);
 
-    if (EMSESP::dallas_enabled()) {
-        node["temperature sensors"] = EMSESP::dallassensor_.no_sensors();
-    }
-
-    if (EMSESP::analog_enabled()) {
-        node["analog sensors"] = EMSESP::analogsensor_.no_sensors();
-    }
 
 #ifndef EMSESP_STANDALONE
-    // Network
+    // Network Status
-    node = output.createNestedObject("Network");
+    node = output.createNestedObject("Network Status");
     if (WiFi.status() == WL_CONNECTED) {
         node["connection"] = F("WiFi");
         node["hostname"]   = WiFi.getHostname();
-        node["SSID"]            = WiFi.SSID();
+        // node["SSID"]            = WiFi.SSID();
-        node["BSSID"]           = WiFi.BSSIDstr();
+        // node["BSSID"]           = WiFi.BSSIDstr();
         node["RSSI"] = WiFi.RSSI();
-        node["MAC"]             = WiFi.macAddress();
+        // node["MAC"]             = WiFi.macAddress();
         node["IPv4 address"]    = uuid::printable_to_string(WiFi.localIP()) + "/" + uuid::printable_to_string(WiFi.subnetMask());
         node["IPv4 gateway"]    = uuid::printable_to_string(WiFi.gatewayIP());
         node["IPv4 nameserver"] = uuid::printable_to_string(WiFi.dnsIP());
@@ -1140,18 +1044,111 @@ bool System::command_info(const char * value, const int8_t id, JsonObject & outp
         if (ETH.localIPv6().toString() != "0000:0000:0000:0000:0000:0000:0000:0000") {
             node["IPv6 address"] = uuid::printable_to_string(ETH.localIPv6());
         }
+        EMSESP::webSettingsService.read([&](WebSettings & settings) {
+            node["phy type"]       = settings.phy_type;
+            node["eth power"]      = settings.eth_power;
+            node["eth phy addr"]   = settings.eth_phy_addr;
+            node["eth clock mode"] = settings.eth_clock_mode;
+        });
+    }
+#endif
+    EMSESP::esp8266React.getNetworkSettingsService()->read([&](NetworkSettings & settings) {
+        node["static ip config"] = settings.staticIPConfig;
+        node["enable IPv6"]      = settings.enableIPv6;
+        node["low bandwidth"]    = settings.bandwidth20;
+        node["disable sleep"]    = settings.nosleep;
+    });
+    EMSESP::esp8266React.getAPSettingsService()->read([&](APSettings & settings) {
+        const char * pM[]         = {"always", "disconnected", "never"};
+        node["AP provision mode"] = pM[settings.provisionMode];
+        node["AP security"]       = settings.password.length() ? "wpa2" : "open";
+        node["AP ssid"]           = settings.ssid;
+    });
+
+    // NTP status
+    node                 = output.createNestedObject("NTP Status");
+    node["network time"] = EMSESP::system_.ntp_connected() ? "connected" : "disconnected";
+    EMSESP::esp8266React.getNTPSettingsService()->read([&](NTPSettings & settings) {
+        node["enabled"]  = settings.enabled;
+        node["server"]   = settings.server;
+        node["tz label"] = settings.tzLabel;
+        // node["tz format"] = settings.tzFormat;
+    });
+
+    // OTA status
+    node = output.createNestedObject("OTA Status");
+    EMSESP::esp8266React.getOTASettingsService()->read([&](OTASettings & settings) {
+        node["enabled"] = settings.enabled;
+        node["port"]    = settings.port;
+    });
+
+    // MQTT Status
+    node                = output.createNestedObject("MQTT Status");
+    node["MQTT status"] = Mqtt::connected() ? F_(connected) : F_(disconnected);
+    if (Mqtt::enabled()) {
+        node["MQTT publishes"]     = Mqtt::publish_count();
+        node["MQTT publish fails"] = Mqtt::publish_fails();
+    }
+    EMSESP::esp8266React.getMqttSettingsService()->read([&](MqttSettings & settings) {
+        node["enabled"]                 = settings.enabled;
+        node["client_id"]               = settings.clientId;
+        node["keep alive"]              = settings.keepAlive;
+        node["clean session"]           = settings.cleanSession;
+        node["base"]                    = settings.base;
+        node["discovery prefix"]        = settings.discovery_prefix;
+        node["nested format"]           = settings.nested_format;
+        node["ha enabled"]              = settings.ha_enabled;
+        node["mqtt qos"]                = settings.mqtt_qos;
+        node["mqtt retain"]             = settings.mqtt_retain;
+        node["publish time boiler"]     = settings.publish_time_boiler;
+        node["publish time thermostat"] = settings.publish_time_thermostat;
+        node["publish time solar"]      = settings.publish_time_solar;
+        node["publish time mixer"]      = settings.publish_time_mixer;
+        node["publish time other"]      = settings.publish_time_other;
+        node["publish time sensor"]     = settings.publish_time_sensor;
+        node["publish single"]          = settings.publish_single;
+        node["publish2command"]         = settings.publish_single2cmd;
+        node["send response"]           = settings.send_response;
+    });
+
+    // Syslog Status
+    node            = output.createNestedObject("Syslog Status");
+    node["enabled"] = EMSESP::system_.syslog_enabled_;
+#ifndef EMSESP_STANDALONE
+    if (EMSESP::system_.syslog_enabled_) {
+        node["syslog started"] = syslog_.started();
+        node["syslog level"]   = FL_(enum_syslog_level)[syslog_.log_level() + 1];
+        node["syslog ip"]      = syslog_.ip();
+        node["syslog queue"]   = syslog_.queued();
     }
 #endif
 
-    // Status
+    // Sensor Status
-    node = output.createNestedObject("Status");
+    node = output.createNestedObject("Sensor Status");
+    if (EMSESP::dallas_enabled()) {
+        node["temperature sensors"]      = EMSESP::dallassensor_.no_sensors();
+        node["temperature sensor reads"] = EMSESP::dallassensor_.reads();
+        node["temperature sensor fails"] = EMSESP::dallassensor_.fails();
+    }
+    if (EMSESP::analog_enabled()) {
+        node["analog sensors"]      = EMSESP::analogsensor_.no_sensors();
+        node["analog sensor reads"] = EMSESP::analogsensor_.reads();
+        node["analog sensor fails"] = EMSESP::analogsensor_.fails();
+    }
 
+    // API Status
+    node              = output.createNestedObject("API Status");
+    node["API calls"] = WebAPIService::api_count();
+    node["API fails"] = WebAPIService::api_fails();
+
+    // EMS Bus Status
+    node = output.createNestedObject("Bus Status");
     switch (EMSESP::bus_status()) {
     case EMSESP::BUS_STATUS_OFFLINE:
         node["bus status"] = (F("disconnected"));
         break;
     case EMSESP::BUS_STATUS_TX_ERRORS:
-        node["bus status"] = (F("connected, tx issues - try a different tx-mode"));
+        node["bus status"] = (F("connected, tx issues - try a different Tx Mode"));
         break;
     case EMSESP::BUS_STATUS_CONNECTED:
         node["bus status"] = (F("connected"));
@@ -1160,7 +1157,6 @@ bool System::command_info(const char * value, const int8_t id, JsonObject & outp
         node["bus status"] = (F("unknown"));
         break;
     }
-
     if (EMSESP::bus_status() != EMSESP::BUS_STATUS_OFFLINE) {
         node["bus protocol"]                = EMSbus::is_ht3() ? F("HT3") : F("Buderus");
         node["bus telegrams received (rx)"] = EMSESP::rxservice_.telegram_count();
@@ -1171,33 +1167,36 @@ bool System::command_info(const char * value, const int8_t id, JsonObject & outp
         node["bus writes failed"]           = EMSESP::txservice_.telegram_write_fail_count();
         node["bus rx line quality"]         = EMSESP::rxservice_.quality();
         node["bus tx line quality"]         = (EMSESP::txservice_.read_quality() + EMSESP::txservice_.read_quality()) / 2;
-        if (Mqtt::enabled()) {
-            node["MQTT status"]        = Mqtt::connected() ? F_(connected) : F_(disconnected);
-            node["MQTT publishes"]     = Mqtt::publish_count();
-            node["MQTT publish fails"] = Mqtt::publish_fails();
     }
-        node["temperature sensors"] = EMSESP::dallassensor_.no_sensors();
-        if (EMSESP::dallas_enabled()) {
-            node["temperature sensor reads"] = EMSESP::dallassensor_.reads();
-            node["temperature sensor fails"] = EMSESP::dallassensor_.fails();
-        }
-        node["analog sensors"] = EMSESP::analogsensor_.no_sensors();
-        if (EMSESP::analog_enabled()) {
-            node["analog sensor reads"] = EMSESP::analogsensor_.reads();
-            node["analog sensor fails"] = EMSESP::analogsensor_.fails();
-        }
-        node["API calls"] = WebAPIService::api_count();
-        node["API fails"] = WebAPIService::api_fails();
 
-#ifndef EMSESP_STANDALONE
+    // Settings
-        if (EMSESP::system_.syslog_enabled_) {
+    node = output.createNestedObject("Settings");
-            node["syslog started"] = syslog_.started();
+    EMSESP::webSettingsService.read([&](WebSettings & settings) {
-            node["syslog level"]   = FL_(enum_syslog_level)[syslog_.log_level() + 1];
+        node["board profile"] = settings.board_profile;
-            node["syslog ip"]      = syslog_.ip();
+        node["tx mode"]       = settings.tx_mode;
-            node["syslog queue"]   = syslog_.queued();
+        node["ems bus id"]    = settings.ems_bus_id;
-        }
+        node["shower timer"]  = settings.shower_timer;
-#endif
+        node["shower alert"]  = settings.shower_alert;
+        if (settings.shower_alert) {
+            node["shower alert coldshot"] = settings.shower_alert_coldshot; // seconds
+            node["shower alert trigger"]  = settings.shower_alert_trigger;  // minutes
         }
+        node["rx gpio"]         = settings.rx_gpio;
+        node["tx gpio"]         = settings.tx_gpio;
+        node["dallas gpio"]     = settings.dallas_gpio;
+        node["pbutton gpio"]    = settings.pbutton_gpio;
+        node["led gpio"]        = settings.led_gpio;
+        node["hide led"]        = settings.hide_led;
+        node["notoken api"]     = settings.notoken_api;
+        node["readonly mode"]   = settings.readonly_mode;
+        node["fahrenheit"]      = settings.fahrenheit;
+        node["dallas parasite"] = settings.dallas_parasite;
+        node["bool format"]     = settings.bool_format;
+        node["bool dashboard"]  = settings.bool_dashboard;
+        node["enum format"]     = settings.enum_format;
+        node["analog enabled"]  = settings.analog_enabled;
+        node["telnet enabled"]  = settings.telnet_enabled;
+    });
 
     // Devices - show EMS devices
     JsonArray devices = output.createNestedArray("Devices");

src/system.h

@@ -84,6 +84,9 @@ class System {
     void button_init(bool refresh);
     void commands_init();
 
+    static void extractSettings(const char * filename, const char * section, JsonObject & output);
+    static bool saveSettings(const char * filename, const char * section, JsonObject & input);
+
     static bool is_valid_gpio(uint8_t pin);
     static bool load_board_profile(std::vector<int8_t> & data, const std::string & board_profile);
 

src/test/test.cpp

@@ -1614,6 +1614,7 @@ void Test::listDir(fs::FS & fs, const char * dirname, uint8_t levels) {
             if (levels) {
                 listDir(fs, file.name(), levels - 1);
             }
+            Serial.println();
         } else {
             Serial.print("  FILE: ");
             Serial.print(file.name());
@@ -1638,6 +1639,7 @@ void Test::debug(uuid::console::Shell & shell, const std::string & cmd) {
 #ifndef EMSESP_STANDALONE
     if (command == "ls") {
         listDir(LITTLEFS, "/", 3);
+        Serial.println();
     }
 #endif
 }

src/version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.4.0b16"
+#define EMSESP_APP_VERSION "3.4.0b17"