Merge branch 'origin/dev'

.vscode/extensions.json

@@ -2,7 +2,6 @@
     // See http://go.microsoft.com/fwlink/?LinkId=827846
     // for the documentation about the extensions.json format
     "recommendations": [
-        "arcanis.vscode-zipfs",
         "dbaeumer.vscode-eslint",
         "esbenp.prettier-vscode",
         "platformio.platformio-ide"

.vscode/settings.json

@@ -41,5 +41,6 @@
     "**/*.min.*",
     "**/*.map",
     "**/ArduinoJson/**"
-  ]
+  ],
+  "cSpell.enableFiletypes": ["!cpp"]
 }

CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+# [3.6.1] September 9 2023
+
+## **IMPORTANT! BREAKING CHANGES**
+
+- `shower_data` MQTT topic shows duration is seconds (was previously a full english sentence)
+
+## Added
+
+- show WiFi rssi in Network Status Page, show quality as color
+
+## Fixed
+
+- Issue in espMqttClient causing a memory leak when MQTT broker is disconnected due to network unavailability [#1264](https://github.com/emsesp/EMS-ESP32/issues/1264)
+- Using MQTT enum values correctly formatted in MQTT Discovery [#1280](https://github.com/emsesp/EMS-ESP32/issues/1280)
+
+## Changed
+
+- mqtt free mem check set to 60 kb
+- small cosmetic changes to Searching in Customization web page
+- updated to espressif32@6.4.0
+
 # [3.6.0] August 13 2023
 
 ## **IMPORTANT! BREAKING CHANGES**

CHANGELOG_LATEST.md

@@ -1,11 +1,22 @@
 # Changelog
 
-# [3.7.0]
+## [3.6.1]
 
 ## **IMPORTANT! BREAKING CHANGES**
 
+- `shower_data` MQTT topic shows duration is seconds (was previously a full english sentence)
+
 ## Added
 
+- show WiFi rssi in Network Status Page, show quality as color
+
 ## Fixed
 
+- Issue in espMqttClient causing a memory leak when MQTT broker is disconnected due to network unavailability [#1264](https://github.com/emsesp/EMS-ESP32/issues/1264)
+- Using MQTT enum values correctly formatted in MQTT Discovery [#1280](https://github.com/emsesp/EMS-ESP32/issues/1280)
+
 ## Changed
+
+- mqtt free mem check set to 60 kb
+- small cosmetic changes to Searching in Customization web page
+- updated to espressif32@6.4.0

Makefile

@@ -42,7 +42,7 @@ DEFINES += -DARDUINOJSON_ENABLE_STD_STRING=1 -DARDUINOJSON_ENABLE_PROGMEM=1 -DAR
 DEFINES += -DEMSESP_DEBUG -DEMSESP_STANDALONE -DEMSESP_TEST -D__linux__ -DEMC_RX_BUFFER_SIZE=1500
 DEFINES += $(ARGS)
 
-DEFAULTS = -DEMSESP_DEFAULT_LOCALE=\"en\" -DEMSESP_DEFAULT_TX_MODE=8 -DEMSESP_DEFAULT_VERSION=\"3.6.0-dev\" -DEMSESP_DEFAULT_BOARD_PROFILE=\"S32\"
+DEFAULTS = -DEMSESP_DEFAULT_LOCALE=\"en\" -DEMSESP_DEFAULT_TX_MODE=8 -DEMSESP_DEFAULT_VERSION=\"3.7.0-dev\" -DEMSESP_DEFAULT_BOARD_PROFILE=\"S32\"
 
 #----------------------------------------------------------------------
 # Sources & Files

interface/.typesafe-i18n.json

@@ -1,5 +1,5 @@
 {
    "adapter": "react",
    "baseLocale": "pl",
-   "$schema": "https://unpkg.com/typesafe-i18n@5.26.0/schema/typesafe-i18n.json"
+   "$schema": "https://unpkg.com/typesafe-i18n@5.26.2/schema/typesafe-i18n.json"
 }

interface/package.json

@@ -12,7 +12,7 @@
     "build-hosted": "vite build --mode hosted",
     "preview": "vite preview",
     "preview-standalone": "npm-run-all -p preview typesafe-i18n mock-api",
-    "mock-api": "nodemon --watch ../mock-api ../mock-api/server.js",
+    "mock-api": "node --watch ../mock-api ../mock-api/server.js",
     "standalone": "npm-run-all -p dev typesafe-i18n mock-api",
     "typesafe-i18n": "typesafe-i18n",
     "format": "prettier --write '**/*.{ts,tsx,js,css,json,md}'",
@@ -22,55 +22,54 @@
     "@alova/adapter-xhr": "^1.0.1",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
-    "@mui/icons-material": "^5.14.3",
+    "@mui/icons-material": "^5.14.8",
-    "@mui/material": "^5.14.4",
+    "@mui/material": "^5.14.8",
     "@preact/compat": "^17.1.2",
     "@prefresh/vite": "^2.4.1",
     "@table-library/react-table-library": "4.1.7",
-    "@types/lodash-es": "^4.17.8",
+    "@types/lodash-es": "^4.17.9",
-    "@types/node": "^20.4.10",
+    "@types/node": "^20.6.0",
-    "@types/react": "^18.2.20",
+    "@types/react": "^18.2.21",
     "@types/react-dom": "^18.2.7",
     "@types/react-router-dom": "^5.3.3",
-    "alova": "^2.10.0",
+    "alova": "^2.11.1",
     "async-validator": "^4.2.5",
     "history": "^5.3.0",
     "jwt-decode": "^3.1.2",
     "lodash-es": "^4.17.21",
     "mime-types": "^2.1.35",
-    "preact": "^10.16.0",
+    "preact": "^10.17.1",
     "react": "latest",
     "react-dom": "latest",
     "react-dropzone": "^14.2.3",
-    "react-icons": "^4.10.1",
+    "react-icons": "^4.11.0",
     "react-router-dom": "^6.15.0",
     "react-toastify": "^9.1.3",
     "sockette": "^2.0.6",
-    "typesafe-i18n": "^5.26.0",
+    "typesafe-i18n": "^5.26.2",
-    "typescript": "^5.1.6"
+    "typescript": "^5.2.2"
   },
   "devDependencies": {
-    "@babel/core": "^7.22.10",
+    "@babel/core": "^7.22.17",
     "@preact/preset-vite": "^2.5.0",
     "@types/babel__core": "^7",
-    "@typescript-eslint/eslint-plugin": "^6.3.0",
+    "@typescript-eslint/eslint-plugin": "^6.6.0",
-    "@typescript-eslint/parser": "^6.3.0",
+    "@typescript-eslint/parser": "^6.6.0",
-    "eslint": "^8.47.0",
+    "eslint": "^8.49.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^17.1.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-import-resolver-typescript": "^3.6.0",
     "eslint-plugin-autofix": "^1.1.0",
-    "eslint-plugin-import": "^2.28.0",
+    "eslint-plugin-import": "^2.28.1",
     "eslint-plugin-jsx-a11y": "^6.7.1",
     "eslint-plugin-prettier": "alpha",
-    "eslint-plugin-react": "^7.33.1",
+    "eslint-plugin-react": "^7.33.2",
     "eslint-plugin-react-hooks": "^4.6.0",
-    "nodemon": "^3.0.1",
     "npm-run-all": "^4.1.5",
-    "prettier": "^3.0.1",
+    "prettier": "^3.0.3",
     "rollup-plugin-visualizer": "^5.9.2",
-    "terser": "^5.19.2",
+    "terser": "^5.19.4",
     "vite": "^4.4.9",
     "vite-plugin-svgr": "^3.2.0",
     "vite-tsconfig-paths": "^4.2.0"

interface/src/framework/network/NetworkStatusForm.tsx

@@ -38,6 +38,15 @@ const networkStatusHighlight = ({ status }: NetworkStatus, theme: Theme) => {
   }
 };
 
+const networkQualityHighlight = ({ rssi }: NetworkStatus, theme: Theme) => {
+  if (rssi <= -85) {
+    return theme.palette.error.main;
+  } else if (rssi <= -75) {
+    return theme.palette.warning.main;
+  }
+  return theme.palette.success.main;
+};
+
 export const isWiFi = ({ status }: NetworkStatus) => status === NetworkConnectionStatus.WIFI_STATUS_CONNECTED;
 export const isEthernet = ({ status }: NetworkStatus) => status === NetworkConnectionStatus.ETHERNET_STATUS_CONNECTED;
 
@@ -110,11 +119,11 @@ const NetworkStatusForm: FC = () => {
             <>
               <ListItem>
                 <ListItemAvatar>
-                  <Avatar>
+                  <Avatar sx={{ bgcolor: networkQualityHighlight(data, theme) }}>
                     <SettingsInputAntennaIcon />
                   </Avatar>
                 </ListItemAvatar>
-                <ListItemText primary="SSID" secondary={data.ssid} />
+                <ListItemText primary="SSID (RSSI)" secondary={data.ssid + ' (' + data.rssi + ' dBm)'} />
               </ListItem>
               <Divider variant="inset" component="li" />
             </>

interface/src/framework/network/WiFiNetworkSelector.tsx

@@ -1,10 +1,11 @@
 import LockIcon from '@mui/icons-material/Lock';
 import LockOpenIcon from '@mui/icons-material/LockOpen';
 import WifiIcon from '@mui/icons-material/Wifi';
-import { Avatar, Badge, List, ListItem, ListItemAvatar, ListItemIcon, ListItemText } from '@mui/material';
+import { Avatar, Badge, List, ListItem, ListItemAvatar, ListItemIcon, ListItemText, useTheme } from '@mui/material';
 import { useContext } from 'react';
 
 import { WiFiConnectionContext } from './WiFiConnectionContext';
+import type { Theme } from '@mui/material';
 import type { FC } from 'react';
 import type { WiFiNetwork, WiFiNetworkList } from 'types';
 import { MessageBox } from 'components';
@@ -42,8 +43,18 @@ export const networkSecurityMode = ({ encryption_type }: WiFiNetwork) => {
   }
 };
 
+const networkQualityHighlight = ({ rssi }: WiFiNetwork, theme: Theme) => {
+  if (rssi <= -85) {
+    return theme.palette.error.main;
+  } else if (rssi <= -75) {
+    return theme.palette.warning.main;
+  }
+  return theme.palette.success.main;
+};
+
 const WiFiNetworkSelector: FC<WiFiNetworkSelectorProps> = ({ networkList }) => {
   const { LL } = useI18nContext();
+  const theme = useTheme();
 
   const wifiConnectionContext = useContext(WiFiConnectionContext);
 
@@ -57,8 +68,8 @@ const WiFiNetworkSelector: FC<WiFiNetworkSelectorProps> = ({ networkList }) => {
         secondary={'Security: ' + networkSecurityMode(network) + ', Ch: ' + network.channel}
       />
       <ListItemIcon>
-        <Badge badgeContent={network.rssi + 'db'}>
+        <Badge badgeContent={network.rssi + 'dBm'}>
-          <WifiIcon />
+          <WifiIcon sx={{ color: networkQualityHighlight(network, theme) }} />
         </Badge>
       </ListItemIcon>
     </ListItem>

interface/src/project/SettingsCustomization.tsx

@@ -86,7 +86,7 @@ const SettingsCustomization: FC = () => {
 
   const entities_theme = useTheme({
     Table: `
-      --data-table-library_grid-template-columns: 150px repeat(1, minmax(80px, 1fr)) 45px minmax(45px, auto) minmax(120px, auto);
+      --data-table-library_grid-template-columns: 156px repeat(1, minmax(80px, 1fr)) 45px minmax(45px, auto) minmax(120px, auto);
     `,
     BaseRow: `
       font-size: 14px;
@@ -192,17 +192,9 @@ const SettingsCustomization: FC = () => {
     return value;
   }
 
-  function formatName(de: DeviceEntity) {
+  const formatName = (de: DeviceEntity, withShortname: boolean) =>
-    return (
+    (de.n && de.n[0] === '!' ? LL.COMMAND(1) + ': ' + de.n.slice(1) : de.cn && de.cn !== '' ? de.cn : de.n) +
-      <>
+    (withShortname ? ' ' + de.id : '');
-        {de.n && (de.n[0] === '!' ? LL.COMMAND(1) + ': ' + de.n.slice(1) : de.cn && de.cn !== '' ? de.cn : de.n) + ' '}(
-        <Link target="_blank" href={APIURL + devices?.devices[selectedDevice].tn + '/' + de.id}>
-          {de.id}
-        </Link>
-        )
-      </>
-    );
-  }
 
   const getMaskNumber = (newMask: string[]) => {
     let new_mask = 0;
@@ -232,10 +224,13 @@ const SettingsCustomization: FC = () => {
     return new_masks;
   };
 
+  const filter_entity = (de: DeviceEntity) =>
+    (de.m & selectedFilters || !selectedFilters) && formatName(de, true).includes(search.toLocaleLowerCase());
+
   const maskDisabled = (set: boolean) => {
     setDeviceEntities(
       deviceEntities.map(function (de) {
-        if ((de.m & selectedFilters || !selectedFilters) && de.id.toLowerCase().includes(search.toLowerCase())) {
+        if (filter_entity(de)) {
           return {
             ...de,
             m: set
@@ -353,7 +348,7 @@ const SettingsCustomization: FC = () => {
         margin="normal"
         select
       >
-        <MenuItem disabled key={0} value={-1}>
+        <MenuItem disabled key={-1} value={-1}>
           {LL.SELECT_DEVICE()}...
         </MenuItem>
         {devices.devices.map((device: DeviceShort, index) => (
@@ -370,9 +365,7 @@ const SettingsCustomization: FC = () => {
       return;
     }
 
-    const shown_data = deviceEntities.filter(
+    const shown_data = deviceEntities.filter((de) => filter_entity(de));
-      (de) => (de.m & selectedFilters || !selectedFilters) && de.id.toLowerCase().includes(search.toLowerCase())
-    );
 
     return (
       <>
@@ -470,7 +463,13 @@ const SettingsCustomization: FC = () => {
                     <Cell stiff>
                       <EntityMaskToggle onUpdate={updateDeviceEntity} de={de} />
                     </Cell>
-                    <Cell>{formatName(de)}</Cell>
+                    <Cell>
+                      {formatName(de, false)}&nbsp;(
+                      <Link target="_blank" href={APIURL + devices?.devices[selectedDevice].tn + '/' + de.id}>
+                        {de.id}
+                      </Link>
+                      )
+                    </Cell>
                     <Cell>{!(de.m & DeviceEntityMask.DV_READONLY) && formatValue(de.mi)}</Cell>
                     <Cell>{!(de.m & DeviceEntityMask.DV_READONLY) && formatValue(de.ma)}</Cell>
                     <Cell>{formatValue(de.v)}</Cell>

interface/src/project/SettingsCustomizationDialog.tsx

@@ -1,4 +1,5 @@
 import CancelIcon from '@mui/icons-material/Cancel';
+import CloseIcon from '@mui/icons-material/Close';
 import DoneIcon from '@mui/icons-material/Done';
 
 import {
@@ -67,15 +68,34 @@ const SettingsCustomizationDialog = ({ open, onClose, onSave, selectedItem }: Se
     <Dialog sx={dialogStyle} open={open} onClose={close}>
       <DialogTitle>{LL.EDIT() + ' ' + LL.ENTITY()}</DialogTitle>
       <DialogContent dividers>
-        <Box color="warning.main">
+        <Grid container direction="row">
-          <Typography variant="body2">{editItem.id}</Typography>
+          <Typography variant="body2" color="warning.main">
-        </Box>
+            {LL.ENTITY() + ' ID'}:&nbsp;
-        <Box color="warning.main" mt={1} mb={2}>
-          <Typography variant="body2">
-            {LL.DEFAULT(1) + ' ' + LL.ENTITY_NAME(1)}:&nbsp;{editItem.n}
           </Typography>
-        </Box>
+          <Typography variant="body2">{editItem.id}</Typography>
-        <Box mb={3}>
+        </Grid>
+
+        <Grid container direction="row">
+          <Typography variant="body2" color="warning.main">
+            {LL.DEFAULT(1) + ' ' + LL.ENTITY_NAME(1)}:&nbsp;
+          </Typography>
+          <Typography variant="body2">{editItem.n}</Typography>
+        </Grid>
+
+        <Grid container direction="row">
+          <Typography variant="body2" color="warning.main">
+            {LL.WRITEABLE()}:&nbsp;
+          </Typography>
+          <Typography variant="body2">
+            {editItem.w ? (
+              <DoneIcon color="success" sx={{ fontSize: 16 }} />
+            ) : (
+              <CloseIcon color="error" sx={{ fontSize: 16 }} />
+            )}
+          </Typography>
+        </Grid>
+
+        <Box mt={1} mb={2}>
           <EntityMaskToggle onUpdate={updateDeviceEntity} de={editItem} />
         </Box>
         <Grid container spacing={1}>

interface/src/project/SettingsEntities.tsx

@@ -13,7 +13,7 @@ import { toast } from 'react-toastify';
 
 import SettingsEntitiesDialog from './SettingsEntitiesDialog';
 import * as EMSESP from './api';
-import { DeviceValueUOM_s } from './types';
+import { DeviceValueTypeNames, DeviceValueUOM_s } from './types';
 import { entityItemValidation } from './validators';
 import type { EntityItem } from './types';
 import type { FC } from 'react';
@@ -57,7 +57,7 @@ const SettingsEntities: FC = () => {
 
   const entity_theme = useTheme({
     Table: `
-      --data-table-library_grid-template-columns: repeat(1, minmax(60px, 1fr)) minmax(80px, auto) 80px 80px 80px;
+      --data-table-library_grid-template-columns: repeat(1, minmax(60px, 1fr)) minmax(80px, auto) 80px 80px 80px 90px;
     `,
     BaseRow: `
       font-size: 14px;
@@ -81,6 +81,9 @@ const SettingsEntities: FC = () => {
       &:nth-of-type(5) {
         text-align: center;
       }
+      &:nth-of-type(6) {
+        text-align: center;
+      }
     `,
     HeaderRow: `
       text-transform: uppercase;
@@ -208,8 +211,9 @@ const SettingsEntities: FC = () => {
                 <HeaderCell>{LL.NAME(0)}</HeaderCell>
                 <HeaderCell stiff>{LL.ID_OF(LL.DEVICE())}</HeaderCell>
                 <HeaderCell stiff>{LL.ID_OF(LL.TYPE(1))}</HeaderCell>
-                <HeaderCell stiff>Offset</HeaderCell>
+                <HeaderCell stiff>{LL.OFFSET()}</HeaderCell>
-                <HeaderCell stiff>{LL.VALUE(0)}</HeaderCell>
+                <HeaderCell stiff>{LL.VALUE(1) + ' ' + LL.TYPE(1)}</HeaderCell>
+                <HeaderCell stiff>{LL.VALUE(1)}</HeaderCell>
               </HeaderRow>
             </Header>
             <Body>
@@ -219,6 +223,7 @@ const SettingsEntities: FC = () => {
                   <Cell>{showHex(ei.device_id as number, 2)}</Cell>
                   <Cell>{showHex(ei.type_id as number, 3)}</Cell>
                   <Cell>{ei.offset}</Cell>
+                  <Cell>{DeviceValueTypeNames[ei.value_type]}</Cell>
                   <Cell>{formatValue(ei.value, ei.uom)}</Cell>
                 </Row>
               ))}

interface/src/project/SettingsEntitiesDialog.tsx

@@ -148,7 +148,7 @@ const SettingsEntitiesDialog = ({
             <ValidatedTextField
               fieldErrors={fieldErrors}
               name="offset"
-              label="Offset"
+              label={LL.OFFSET()}
               margin="normal"
               fullWidth
               type="number"
@@ -159,7 +159,7 @@ const SettingsEntitiesDialog = ({
           <Grid item xs={4}>
             <TextField
               name="value_type"
-              label="Value Type"
+              label={LL.VALUE(1) + ' ' + LL.TYPE(1)}
               value={editItem.value_type}
               variant="outlined"
               onChange={updateFormValue}

interface/src/project/SettingsScheduler.tsx

@@ -1,6 +1,6 @@
 import AddIcon from '@mui/icons-material/Add';
 import CancelIcon from '@mui/icons-material/Cancel';
-import CheckCircleIcon from '@mui/icons-material/CheckCircle';
+import CircleIcon from '@mui/icons-material/Circle';
 import WarningIcon from '@mui/icons-material/Warning';
 
 import { Box, Typography, Divider, Stack, Button } from '@mui/material';
@@ -216,7 +216,11 @@ const SettingsScheduler: FC = () => {
               {tableList.map((si: ScheduleItem) => (
                 <Row key={si.id} item={si} onClick={() => editScheduleItem(si)}>
                   <Cell stiff>
-                    {si.active && <CheckCircleIcon sx={{ color: '#79D200', fontSize: 16, verticalAlign: 'middle' }} />}
+                    {si.active ? (
+                      <CircleIcon color="success" sx={{ fontSize: 16, verticalAlign: 'middle' }} />
+                    ) : (
+                      <CircleIcon color="error" sx={{ fontSize: 16, verticalAlign: 'middle' }} />
+                    )}
                   </Cell>
                   <Cell stiff>
                     <Stack spacing={1} direction="row">

interface/src/project/SettingsSchedulerDialog.tsx

@@ -1,6 +1,5 @@
 import AddIcon from '@mui/icons-material/Add';
 import CancelIcon from '@mui/icons-material/Cancel';
-import CheckCircleIcon from '@mui/icons-material/CheckCircle';
 import DoneIcon from '@mui/icons-material/Done';
 import RemoveIcon from '@mui/icons-material/RemoveCircleOutline';
 
@@ -184,11 +183,6 @@ const SettingsSchedulerDialog = ({
             control={<Checkbox checked={editItem.active} onChange={updateFormValue} name="active" />}
             label={LL.ACTIVE()}
           />
-          {editItem.active && (
-            <Grid item sx={{ mt: 1 }}>
-              <CheckCircleIcon sx={{ color: '#79D200', fontSize: 16, verticalAlign: 'middle' }} />
-            </Grid>
-          )}
         </Grid>
         <Grid container>
           <TextField

interface/src/project/types.ts

@@ -379,3 +379,16 @@ export const enum DeviceValueType {
   STRING,
   CMD
 }
+
+export const DeviceValueTypeNames = [
+  'BOOL',
+  'INT',
+  'UINT',
+  'SHORT',
+  'USHORT',
+  'ULONG',
+  'TIME',
+  'ENUM',
+  'STRING',
+  'CMD'
+];

interface/vite.config.ts

@@ -50,9 +50,9 @@ export default defineConfig(({ command, mode }) => {
       server: {
         open: true,
         port: 3000,
-        watch: {
+        // watch: {
-          usePolling: true
+        //   usePolling: true
-        },
+        // },
         proxy: {
           '/rest': 'http://localhost:3080',
           '/api': {

lib/espMqttClient/src/MqttClient.cpp

@@ -103,7 +103,7 @@ bool MqttClient::disconnected() const {
 }
 
 bool MqttClient::connect() {
-    bool result = true;
+    bool result = false;
     if (_state == State::disconnected) {
         EMC_SEMAPHORE_TAKE();
         if (_addPacketFront(_cleanSession,
@@ -116,19 +116,21 @@ bool MqttClient::connect() {
                             _willPayloadLength,
                             (uint16_t)(_keepAlive / 1000), // 32b to 16b doesn't overflow because it comes from 16b orignally
                             _clientId)) {
+            result = true;
+            _state = State::connectingTcp1;
 #if defined(ARDUINO_ARCH_ESP32)
             if (_useInternalTask == espMqttClientTypes::UseInternalTask::YES) {
                 vTaskResume(_taskHandle);
             }
 #endif
-            _state = State::connectingTcp1;
         } else {
             EMC_SEMAPHORE_GIVE();
             emc_log_e("Could not create CONNECT packet");
             _onError(0, Error::OUT_OF_MEMORY);
-            result = false;
         }
         EMC_SEMAPHORE_GIVE();
+    } else if (_state <= State::connected) { // already connected or connecting
+        result = true;
     }
     return result;
 }
@@ -196,6 +198,14 @@ const char * MqttClient::getClientId() const {
     return _clientId;
 }
 
+size_t MqttClient::queueSize() {
+    size_t ret = 0;
+    EMC_SEMAPHORE_TAKE();
+    ret = _outbox.size();
+    EMC_SEMAPHORE_GIVE();
+    return ret;
+}
+
 void MqttClient::loop() {
     switch ((State)_state) { // modified by proddy for EMS-ESP compiling standalone
     case State::disconnected:
@@ -335,7 +345,6 @@ int MqttClient::_sendPacket() {
     size_t wantToWrite = 0;
     size_t written     = 0;
     if (packet && (wantToWrite == written)) {
-        // mixing signed with unsigned here but safe because of MQTT packet size limits
         wantToWrite = packet->packet.available(_bytesSent);
         if (wantToWrite == 0) {
             EMC_SEMAPHORE_GIVE();
@@ -630,9 +639,6 @@ void MqttClient::_onPubcomp() {
         // if it doesn't match the ID, return
         if ((it.get()->packet.packetType()) == PacketType.PUBREL) {
             if (it.get()->packet.packetId() == idToMatch) {
-                // if (!_addPacket(PacketType.PUBCOMP, idToMatch)) {
-                //     emc_log_e("Could not create PUBCOMP packet");
-                // }
                 callback = true;
                 _outbox.remove(it);
                 break;
@@ -697,20 +703,6 @@ void MqttClient::_onUnsuback() {
     }
 }
 
-uint16_t MqttClient::getQueue() const {
-    EMC_SEMAPHORE_TAKE();
-    espMqttClientInternals::Outbox<OutgoingPacket>::Iterator it    = _outbox.front();
-    uint16_t                                                 count = 0;
-    while (it) {
-        // if (it.get()->packet.packetType() == PacketType.PUBLISH) {
-        ++count;
-        // }
-        ++it;
-    }
-    EMC_SEMAPHORE_GIVE();
-    return count;
-}
-
 void MqttClient::_clearQueue(int clearData) {
     emc_log_i("clearing queue (clear session: %d)", clearData);
     EMC_SEMAPHORE_TAKE();

lib/espMqttClient/src/MqttClient.h

@@ -65,7 +65,7 @@ class MqttClient {
     uint16_t     publish(const char * topic, uint8_t qos, bool retain, espMqttClientTypes::PayloadCallback callback, size_t length);
     void         clearQueue(bool deleteSessionData = false); // Not MQTT compliant and may cause unpredictable results when `deleteSessionData` = true!
     const char * getClientId() const;
-    uint16_t     getQueue() const;
+    size_t       queueSize(); // No const because of mutex
     void         loop();
 
   protected:
@@ -131,8 +131,9 @@ class MqttClient {
         uint32_t                       timeSent;
         espMqttClientInternals::Packet packet;
         template <typename... Args>
-        OutgoingPacket(uint32_t t, espMqttClientTypes::Error error, Args &&... args)
+        OutgoingPacket(uint32_t t, espMqttClientTypes::Error & error, Args &&... args)
-            : timeSent(t)
+            : // NOLINT(runtime/references)
+            timeSent(t)
             , packet(error, std::forward<Args>(args)...) {
         }
     };
@@ -150,19 +151,27 @@ class MqttClient {
     bool _addPacket(Args &&... args) {
         espMqttClientTypes::Error                                error(espMqttClientTypes::Error::SUCCESS);
         espMqttClientInternals::Outbox<OutgoingPacket>::Iterator it = _outbox.emplace(0, error, std::forward<Args>(args)...);
-        if (it && error == espMqttClientTypes::Error::SUCCESS)
+        if (it && error == espMqttClientTypes::Error::SUCCESS) {
             return true;
+        } else {
+            if (it)
+                _outbox.remove(it);
             return false;
         }
+    }
 
     template <typename... Args>
     bool _addPacketFront(Args &&... args) {
         espMqttClientTypes::Error                                error(espMqttClientTypes::Error::SUCCESS);
         espMqttClientInternals::Outbox<OutgoingPacket>::Iterator it = _outbox.emplaceFront(0, error, std::forward<Args>(args)...);
-        if (it && error == espMqttClientTypes::Error::SUCCESS)
+        if (it && error == espMqttClientTypes::Error::SUCCESS) {
             return true;
+        } else {
+            if (it)
+                _outbox.remove(it);
             return false;
         }
+    }
 
     void _checkOutbox();
     int  _sendPacket();

lib/espMqttClient/src/Outbox.h

@@ -163,6 +163,16 @@ class Outbox {
     return false;
   }
 
+  size_t size() const {
+    Node* n = _first;
+    size_t count = 0;
+    while (n) {
+      n = n->next;
+      ++count;
+    }
+    return count;
+  }
+
  private:
   Node* _first;
   Node* _last;

lib/espMqttClient/src/Packets/Packet.cpp

@@ -100,7 +100,7 @@ Packet::Packet(espMqttClientTypes::Error& error,
   (password ? 2 + strlen(password) : 0);
 
   // allocate memory
-  if (!_allocate(remainingLength)) {
+  if (!_allocate(remainingLength, false)) {
     error = espMqttClientTypes::Error::OUT_OF_MEMORY;
     return;
   }
@@ -300,8 +300,8 @@ Packet::Packet(espMqttClientTypes::Error& error, MQTTPacketType type)
 }
 
 
-bool Packet::_allocate(size_t remainingLength) {
+bool Packet::_allocate(size_t remainingLength, bool check) {
-  if (EMC_GET_FREE_MEMORY() < EMC_MIN_FREE_MEMORY) {
+  if (check && EMC_GET_FREE_MEMORY() < EMC_MIN_FREE_MEMORY) {
     emc_log_w("Packet buffer not allocated: low memory");
     return false;
   }

lib/espMqttClient/src/Packets/Packet.h

@@ -133,7 +133,7 @@ class Packet {
 
  private:
   // pass remainingLength = total size - header - remainingLengthLength!
-  bool _allocate(size_t remainingLength);
+  bool _allocate(size_t remainingLength, bool check = true);
 
   // fills header and returns index of next available byte in buffer
   size_t _fillPublishHeader(uint16_t packetId,

lib/framework/NetworkSettingsService.cpp

@@ -6,6 +6,10 @@ NetworkSettingsService::NetworkSettingsService(AsyncWebServer * server, FS * fs,
     : _httpEndpoint(NetworkSettings::read, NetworkSettings::update, this, server, NETWORK_SETTINGS_SERVICE_PATH, securityManager)
     , _fsPersistence(NetworkSettings::read, NetworkSettings::update, this, fs, NETWORK_SETTINGS_FILE)
     , _lastConnectionAttempt(0) {
+    addUpdateHandler([&](const String & originId) { reconfigureWiFiConnection(); }, false);
+}
+
+void NetworkSettingsService::begin() {
     // We want the device to come up in opmode=0 (WIFI_OFF), when erasing the flash this is not the default.
     // If needed, we save opmode=0 before disabling persistence so the device boots with WiFi disabled in the future.
     if (WiFi.getMode() != WIFI_OFF) {
@@ -21,10 +25,6 @@ NetworkSettingsService::NetworkSettingsService(AsyncWebServer * server, FS * fs,
 
     WiFi.onEvent(std::bind(&NetworkSettingsService::WiFiEvent, this, _1));
 
-    addUpdateHandler([&](const String & originId) { reconfigureWiFiConnection(); }, false);
-}
-
-void NetworkSettingsService::begin() {
     _fsPersistence.readFromFS();
     reconfigureWiFiConnection();
 }

mock-api/package.json

@@ -5,15 +5,14 @@
   "author": "proddy",
   "main": "server.js",
   "scripts": {
-    "dev": "nodemon ./server.js localhost 3080",
+    "dev": "node --watch ./server.js localhost 3080",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "dependencies": {
     "@msgpack/msgpack": "^2.8.0",
     "compression": "^1.7.4",
     "express": "^4.18.2",
-    "multer": "^1.4.5-lts.1",
+    "multer": "^1.4.5-lts.1"
-    "nodemon": "^3.0.1"
   },
   "packageManager": "yarn@3.4.1"
 }

mock-api/server.js

@@ -299,7 +299,7 @@ const SIGN_IN_ENDPOINT = REST_ENDPOINT_ROOT + 'signIn';
 const GENERATE_TOKEN_ENDPOINT = REST_ENDPOINT_ROOT + 'generateToken';
 
 let system_status = {
-  emsesp_version: '3.6.0-demo',
+  emsesp_version: '3.7.0-demo',
   esp_platform: 'ESP32',
   max_alloc_heap: 89,
   psram_size: 0,
@@ -2627,7 +2627,7 @@ rest_server.get(EMSESP_BOARDPROFILE_ENDPOINT, (req, res) => {
 // EMS-ESP API specific
 const emsesp_info = {
   System: {
-    version: '3.6.0',
+    version: '3.7.0',
     uptime: '001+06:40:34.018',
     'uptime (seconds)': 110434,
     freemem: 131,

pio_local.ini_example

@@ -18,6 +18,7 @@ default_envs = esp32_4M
 ; default_envs = esp32_16M
 ; default_envs = lolin_s3
 ; default_envs = standalone
+; default_envs =  debug
 
 [env:esp32_4M]
 ; if using OTA enter your details below
@@ -33,6 +34,8 @@ upload_port = /dev/ttyUSB*
 ; override arduino espressif core
 platform = espressif32 ; take latest
 ; platform = espressif32@5.3.0
+; platform = espressif32@6.3.2
+; platform = espressif32@6.4.0
 
 extra_scripts =
   pre:scripts/build_interface.py ; comment out if you don't want to re-build the WebUI each time
@@ -52,13 +55,21 @@ extra_scripts =
 ; options for debugging are: EMSESP_DEBUG EMSESP_UART_DEBUG EMSESP_DEBUG_SENSOR
 [env:debug]
 board = esp32dev
+framework = arduino
 platform = espressif32
 board_build.partitions = esp32_partition_debug.csv
+; board_build.partitions = esp32_partition_4M.csv
+board_upload.flash_size = 4MB
+board_build.filesystem = littlefs
 upload_protocol = esptool
 build_type = debug
 monitor_raw = no
 monitor_filters = esp32_exception_decoder
 debug_tool = esp-prog
 debug_init_break = tbreak setup
-build_flags = ${factory_settings.build_flags} -DONEWIRE_CRC16=0 -DNO_GLOBAL_ARDUINOOTA -DARDUINOJSON_ENABLE_STD_STRING=1 -DESP32=1 -DARDUINO_ARCH_ESP32=1
+build_flags = ${factory_settings.build_flags} -DEMSESP_EN_ONLY -DCORE_DEBUG_LEVEL=5 -DONEWIRE_CRC16=0 -DNO_GLOBAL_ARDUINOOTA -DARDUINOJSON_ENABLE_STD_STRING=1 -DESP32=1 -DARDUINO_ARCH_ESP32=1
-extra_scripts = pre:scripts/build_interface.py
+upload_port = /dev/ttyUSB0
+extra_scripts =
+  ; pre:scripts/build_interface.py ; comment out if you don't want to re-build the WebUI each time
+  scripts/rename_fw.py
+  ; post:scripts/app-tls-size.py

platformio.ini

@@ -39,7 +39,7 @@ unbuild_flags =
   ${common.core_unbuild_flags}
 
 [espressi32_base]
-platform = espressif32@6.3.2
+platform = espressif32
 framework = arduino
 build_flags = ${common.build_flags}
 build_unflags = ${common.unbuild_flags}
@@ -65,6 +65,8 @@ check_flags =
 ; the Web interface is built seperately
 [env:ci]
 extends = espressi32_base
+; stay on platform 6.3.2 until heap issue is solved
+platform = espressif32@6.3.2
 extra_scripts = scripts/rename_fw.py
 board = esp32dev
 board_build.partitions = esp32_partition_4M.csv
@@ -85,6 +87,8 @@ build_unflags = ${common.unbuild_flags}
 
 [env:esp32_4M]
 extends = espressi32_base
+; stay on platform 6.3.2 until heap issue is solved
+platform = espressif32@6.3.2
 board = esp32dev
 board_upload.flash_size = 4MB
 board_build.partitions = esp32_partition_4M.csv
@@ -92,6 +96,8 @@ build_flags = ${common.build_flags} -Os
 
 [env:esp32_4Mplus]
 extends = espressi32_base
+; stay on platform 6.3.2 until heap issue is solved
+platform = espressif32@6.3.2
 board = esp32dev
 board_upload.flash_size = 4MB
 board_build.partitions = esp32_asym_partition_4M.csv
@@ -99,6 +105,8 @@ build_flags = ${common.build_flags}
 
 [env:esp32_16M]
 extends = espressi32_base
+; stay on platform 6.3.2 until heap issue is solved
+platform = espressif32@6.3.2
 board = esp32dev
 board_upload.flash_size = 16MB
 board_build.partitions = esp32_partition_16M.csv
@@ -143,7 +151,7 @@ platform = native
 build_flags =
   -DARDUINOJSON_ENABLE_STD_STRING=1 -DARDUINOJSON_ENABLE_PROGMEM=1 -DARDUINOJSON_ENABLE_ARDUINO_STRING -DARDUINOJSON_USE_DOUBLE=0
   -DEMSESP_DEBUG -DEMSESP_STANDALONE -DEMSESP_TEST -D__linux__
-  -DEMSESP_DEFAULT_LOCALE=\"en\" -DEMSESP_DEFAULT_TX_MODE=8 -DEMSESP_DEFAULT_VERSION=\"3.6.0-dev\" -DEMSESP_DEFAULT_BOARD_PROFILE=\"S32\"
+  -DEMSESP_DEFAULT_LOCALE=\"en\" -DEMSESP_DEFAULT_TX_MODE=8 -DEMSESP_DEFAULT_VERSION=\"3.7.0-dev\" -DEMSESP_DEFAULT_BOARD_PROFILE=\"S32\"
 	-lpthread
   -std=gnu++11 -Og -ggdb
 build_src_flags =

scripts/api_test.http

@@ -1,17 +1,21 @@
 # testing EMS-ESP API
-# use with https://marketplace.visualstudio.com/items?itemName=humao.rest-client
+# use with "REST Client" extension in Visual Studio Code
+# https://marketplace.visualstudio.com/items?itemName=humao.rest-client
+
+@host = http://ems-esp.local
+@host_dev = http://ems-esp2.local
 
 @token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwiYWRtaW4iOnRydWV9.2bHpWya2C7Q12WjNUBD6_7N3RCD7CMl-EGhyQVzFdDg
 
-GET http://ems-esp.local/api/system/info
+GET {{host}}/api/system/info
 
 ###
 
-GET http://ems-esp.local/api/thermostat/seltemp
+GET {{host}}/api/thermostat/seltemp
 
 ###
 
-POST http://ems-esp.local/api/thermostat/seltemp
+POST {{host}}/api/thermostat/seltemp
 Content-Type: application/json
 Authorization: Bearer {{token}}
 
@@ -21,7 +25,7 @@ Authorization: Bearer {{token}}
 
 ###
 
-POST http://ems-esp.local/api/thermostat
+POST {{host}}/api/thermostat
 Content-Type: application/json
 Authorization: Bearer {{token}}
 
@@ -32,12 +36,61 @@ Authorization: Bearer {{token}}
 
 ###
 
-POST http://10.10.10.134/api
+POST {{host}}/api
 Content-Type: application/json
-Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwiYWRtaW4iOnRydWV9.2bHpWya2C7Q12WjNUBD6_7N3RCD7CMl-EGhyQVzFdDg
+Authorization: Bearer {{token}}
 
 {
     "device" : "boiler",
     "entity" : "wwtapactivated",
     "value" : "on"
 }
+
+###
+
+GET {{host}}/api/system/restart
+Authorization: Bearer {{token}}
+
+###
+
+GET {{host}}/api/boiler/coldshot
+Authorization: Bearer {{token}}
+
+###
+
+GET {{host}}/api/temperaturesensor/info
+
+###
+
+#
+# Test on dev
+#
+
+GET {{host_dev}}/api/system/info
+
+# Run a test. EMS-ESP must be compiled with -DEMSESP_TEST
+# Use this to load up a dummy thermostat and boiler with data
+
+###
+
+GET {{host_dev}}/api?device=system&cmd=test&data=general
+
+###
+
+GET {{host_dev}}/api/boiler/info
+
+###
+GET {{host_dev}}/api/boiler/values
+
+###
+
+GET {{host_dev}}/api/analogsensor/info
+
+###
+
+GET {{host_dev}}/api/boiler/coldshot
+Authorization: Bearer {{token}}
+
+###
+GET {{host_dev}}/api/system/commands
+

sonar-project.properties

@@ -1,7 +1,7 @@
 sonar.organization=emsesp
 sonar.projectKey=emsesp_EMS-ESP32
 sonar.projectName=EMS-ESP32
-sonar.projectVersion=3.6.0
+sonar.projectVersion=3.7.0
 sonar.sources=./src
 sonar.cfamily.build-wrapper-output=build_wrapper_output_directory
 sonar.sourceEncoding=UTF-8

src/command.cpp

@@ -301,9 +301,14 @@ uint8_t Command::call(const uint8_t device_type, const char * cmd, const char *
     auto cf = find_command(device_type, device_id, cmd);
 
     // check if its a call to and end-point to a device
-    // except for system commands as this is a special device without any queryable entities (device values)
+    // this is used to fetch the attributes of the device entity, or call a command directly
-    if ((device_type > EMSdevice::DeviceType::SYSTEM) && (!value || !strlen(value))) {
+    bool single_command = (!value || !strlen(value));
-        if (!cf || !cf->cmdfunction_json_) {
+    if (single_command) {
+        // exception 1: anything that is from System
+        // exception 2: boiler coldshot command
+        bool get_attributes = (!cf || !cf->cmdfunction_json_) && (device_type > EMSdevice::DeviceType::SYSTEM) && (strcmp(cmd, F_(coldshot)) != 0);
+
+        if (get_attributes) {
             LOG_DEBUG("Calling %s command '%s' to retrieve attributes", dname, cmd);
             return EMSESP::get_device_value_info(output, cmd, id, device_type) ? CommandRet::OK : CommandRet::ERROR; // entity = cmd
         }
@@ -344,7 +349,7 @@ uint8_t Command::call(const uint8_t device_type, const char * cmd, const char *
 
         // check if read-only. This also checks for valid tags (e.g. heating circuits)
         if (cf->cmdfunction_) {
-            if (EMSESP::cmd_is_readonly(device_type, device_id, cmd, id)) {
+            if (!single_command && EMSESP::cmd_is_readonly(device_type, device_id, cmd, id)) {
                 return_code = CommandRet::INVALID; // readonly or invalid hc
             } else {
                 return_code = ((cf->cmdfunction_)(value, id)) ? CommandRet::OK : CommandRet::ERROR;
@@ -593,7 +598,7 @@ void Command::show_devices(uuid::console::Shell & shell) {
 // output list of all commands to console
 // calls show with verbose mode set
 void Command::show_all(uuid::console::Shell & shell) {
-    shell.println("Available commands (*=do not need authorization): ");
+    shell.println("Available commands (*=authorization not required): ");
 
     // show system first
     shell.print(COLOR_BOLD_ON);

src/console.cpp

@@ -137,6 +137,9 @@ static void setup_commands(std::shared_ptr<Commands> & commands) {
                                   Test::run_test(shell, arguments[0].c_str(), arguments[1].c_str());
                               }
                           });
+    commands->add_command(ShellContext::MAIN, CommandFlags::USER, string_vector{"t"}, [=](Shell & shell, const std::vector<std::string> & arguments) {
+        Test::run_test(shell, "default");
+    });
 #endif
 
     commands->add_command(ShellContext::MAIN, CommandFlags::USER, string_vector{F_(su)}, [=](Shell & shell, const std::vector<std::string> & arguments) {
@@ -419,7 +422,6 @@ static void setup_commands(std::shared_ptr<Commands> & commands) {
                                   to_app(shell).send_read_request(type_id, device_id, 0, EMS_MAX_TELEGRAM_LENGTH, true);
                               }
                               to_app(shell).set_read_id(type_id);
-
                           });
 
     commands->add_command(ShellContext::MAIN,

src/devices/boiler.cpp

@@ -2131,6 +2131,7 @@ bool Boiler::set_tapwarmwater_activated(const char * value, const int8_t id) {
         wwTapActivated_ = 1;
     }
 
+    // LOG_INFO("calling EMS_TYPE_UBAFunctionTest with %d", v);
     write_command(EMS_TYPE_UBAFunctionTest, 0, message_data, sizeof(message_data), 0);
 
     return true;

src/emsdevice.cpp

@@ -1767,6 +1767,11 @@ const char * EMSdevice::telegram_type_name(std::shared_ptr<const Telegram> teleg
 bool EMSdevice::handle_telegram(std::shared_ptr<const Telegram> telegram) {
     for (auto & tf : telegram_functions_) {
         if (tf.telegram_type_id_ == telegram->type_id) {
+            // for telegram desitnation only read telegram
+            if (telegram->dest == device_id_ && telegram->message_length > 0) {
+                tf.process_function_(telegram);
+                return true;
+            }
             // if the data block is empty and we have not received data before, assume that this telegram
             // is not recognized by the bus master. So remove it from the automatic fetch list
             if (telegram->message_length == 0 && telegram->offset == 0 && !tf.received_) {

src/emsesp.cpp

@@ -939,6 +939,8 @@ bool EMSESP::process_telegram(std::shared_ptr<const Telegram> telegram) {
                 emsdevice->add_handlers_ignored(telegram->type_id);
             }
             break;
+        } else if (emsdevice->is_device_id(telegram->dest)) {
+            emsdevice->handle_telegram(telegram);
         }
     }
 

src/locale_common.h

@@ -80,6 +80,7 @@ MAKE_WORD(info)
 MAKE_WORD(settings)
 MAKE_WORD(value)
 MAKE_WORD(entities)
+MAKE_WORD(coldshot)
 
 // device types - lowercase, used in MQTT
 MAKE_WORD(boiler)

src/locale_translations.h

@@ -70,6 +70,7 @@ MAKE_WORD_TRANSLATION(system_info_cmd, "show system status", "Zeige System-Statu
 MAKE_WORD_TRANSLATION(schedule_cmd, "enable schedule item", "Aktiviere Zeitplan", "activeer tijdschema item", "", "aktywuj wybrany harmonogram", "", "", "program öğesini etkinleştir", "abilitare l'elemento programmato") // TODO translate
 MAKE_WORD_TRANSLATION(entity_cmd, "set custom value on ems", "Sende eigene Entitäten zu EMS", "verstuur custom waarde naar EMS", "", "wyślij własną wartość na EMS", "", "", "", "imposta valori personalizzati su EMS") // TODO translate
 MAKE_WORD_TRANSLATION(commands_response, "get response", "Hole Antwort", "Verzoek om antwoord", "", "", "", "", "") // TODO translate
+MAKE_WORD_TRANSLATION(coldshot_cmd, "send a cold shot of water", "", "", "", "", "", "", "", "") // TODO translate
 
 // tags
 MAKE_WORD_TRANSLATION(tag_boiler_data_ww, "dhw", "WW", "dhw", "VV", "CWU", "dhw", "ecs", "SKS", "dhw")

src/mqtt.cpp

@@ -124,6 +124,8 @@ void Mqtt::resubscribe() {
 
 // Main MQTT loop - sends out top item on publish queue
 void Mqtt::loop() {
+    queuecount_ = mqttClient_->queueSize();
+
     // exit if MQTT is not enabled or if there is no network connection
     if (!connected()) {
         return;
@@ -142,7 +144,7 @@ void Mqtt::loop() {
         EMSESP::publish_sensor_values(false);
     }
 
-    queuecount_ = mqttClient_->getQueue();
+    // wait for empty queue before sending scheduled device messages
     if (queuecount_ > 0) {
         return;
     }
@@ -278,9 +280,13 @@ void Mqtt::on_message(const char * topic, const uint8_t * payload, size_t len) c
     if (return_code != CommandRet::OK) {
         char error[100];
         if (output.size()) {
-            snprintf(error, sizeof(error), "Call failed with error: %s (%s)", (const char *)output["message"], Command::return_code_string(return_code).c_str());
+            snprintf(error,
+                     sizeof(error),
+                     "MQTT command failed with error: %s (%s)",
+                     (const char *)output["message"],
+                     Command::return_code_string(return_code).c_str());
         } else {
-            snprintf(error, sizeof(error), "Call failed with error code (%s)", Command::return_code_string(return_code).c_str());
+            snprintf(error, sizeof(error), "MQTT command failed with error code (%s)", Command::return_code_string(return_code).c_str());
         }
         LOG_ERROR(error);
         Mqtt::queue_publish("response", error);
@@ -478,7 +484,7 @@ void Mqtt::on_connect() {
 
     connecting_ = true;
     connectcount_++; // count # reconnects. not currently used.
-    queuecount_ = 0;
+    queuecount_ = mqttClient_->queueSize();
 
     load_settings(); // reload MQTT settings - in case they have changes
 
@@ -506,7 +512,7 @@ void Mqtt::on_connect() {
     // publish to the last will topic (see Mqtt::start() function) to say we're alive
     queue_publish_retain("status", "online", true); // with retain on
 
-    mqtt_publish_fails_ = 0;                        // reset fail count to 0
+    // mqtt_publish_fails_ = 0; // reset fail count to 0
 }
 
 // Home Assistant Discovery - the main master Device called EMS-ESP
@@ -586,14 +592,25 @@ bool Mqtt::queue_message(const uint8_t operation, const std::string & topic, con
     if (!mqtt_enabled_ || topic.empty()) {
         return false; // quit, not using MQTT
     }
+// check free mem
+#ifndef EMSESP_STANDALONE
+    if (ESP.getFreeHeap() < 60 * 1204) {
+        if (operation == Operation::PUBLISH) {
+            mqtt_message_id_++;
+            mqtt_publish_fails_++;
+        }
+        LOG_DEBUG("%s failed: low memory", operation == Operation::PUBLISH ? "Publish" : operation == Operation::SUBSCRIBE ? "Subscribe" : "Unsubscribe");
+        return false; // quit
+    }
+#endif
 
     uint16_t packet_id = 0;
     char     fulltopic[MQTT_TOPIC_MAX_SIZE];
 
     if (topic.find(discovery_prefix_) == 0) {
-        strlcpy(fulltopic, topic.c_str(), sizeof(fulltopic)); // leave topic as it is
+        strlcpy(fulltopic, topic.c_str(), sizeof(fulltopic)); // leave discovery topic as it is
     } else {
-        // it's a discovery topic, added the mqtt base to the topic path
+        // it's not a discovery topic, added the mqtt base to the topic path
         snprintf(fulltopic, sizeof(fulltopic), "%s/%s", mqtt_base_.c_str(), topic.c_str()); // uses base
     }
 
@@ -898,10 +915,20 @@ bool Mqtt::publish_ha_sensor_config(uint8_t               type,        // EMSdev
         // for enums, add options
         if (type == DeviceValueType::ENUM) {
             JsonArray option_list = doc.createNestedArray("ops"); // options
+            if (EMSESP::system_.enum_format() == ENUM_FORMAT_INDEX) {
+                // use index numbers
+                for (uint8_t i = 0; i < options_size; i++) {
+                    option_list.add(Helpers::itoa(i)); // as a string
+                }
+                snprintf(sample_val, sizeof(sample_val), "'0'");
+            } else {
+                // use strings
                 for (uint8_t i = 0; i < options_size; i++) {
                     option_list.add(Helpers::translated_word(options[i]));
                 }
                 snprintf(sample_val, sizeof(sample_val), "'%s'", Helpers::translated_word(options[0]));
+            }
+
         } else if (type != DeviceValueType::STRING && type != DeviceValueType::BOOL) {
             // Must be Numeric....
             doc["mode"] = "box"; // auto, slider or box

src/shower.cpp

@@ -22,6 +22,8 @@ namespace emsesp {
 
 uuid::log::Logger Shower::logger_{F_(shower), uuid::log::Facility::CONSOLE};
 
+static bool force_coldshot = false;
+
 void Shower::start() {
     EMSESP::webSettingsService.read([&](WebSettings & settings) {
         shower_timer_          = settings.shower_timer;
@@ -30,8 +32,28 @@ void Shower::start() {
         shower_alert_coldshot_ = settings.shower_alert_coldshot * 1000; // convert from seconds
     });
 
+    Command::add(
+        EMSdevice::DeviceType::BOILER,
+        F_(coldshot),
+        [&](const char * value, const int8_t id, JsonObject & output) {
+            LOG_INFO("Forcing coldshot...");
+            if (shower_state_) {
+                output["message"] = "OK";
+                force_coldshot    = true;
+            } else {
+                output["message"] = "Coldshot failed. Shower not active";
+                LOG_WARNING("Coldshot failed. Shower not active");
+                force_coldshot = false;
+            }
+            return true;
+        },
+        FL_(coldshot_cmd),
+        CommandFlag::ADMIN_ONLY);
+
+    if (shower_timer_) {
         set_shower_state(false, true); // turns shower to off and creates HA topic if not already done
     }
+}
 
 void Shower::loop() {
     if (!shower_timer_) {
@@ -57,10 +79,10 @@ void Shower::loop() {
                 // first check to see if hot water has been on long enough to be recognized as a Shower/Bath
                 if (!shower_state_ && (time_now - timer_start_) > SHOWER_MIN_DURATION) {
                     set_shower_state(true);
-                    LOG_DEBUG("[Shower] hot water still running, starting shower timer");
+                    LOG_DEBUG("hot water still running, starting shower timer");
                 }
                 // check if the shower has been on too long
-                else if ((time_now - timer_start_) > shower_alert_trigger_) {
+                else if ((shower_alert_ && ((time_now - timer_start_) > shower_alert_trigger_)) || force_coldshot) {
                     shower_alert_start();
                 }
             }
@@ -79,11 +101,11 @@ void Shower::loop() {
                     if (duration_ > SHOWER_MIN_DURATION) {
                         StaticJsonDocument<EMSESP_JSON_SIZE_SMALL> doc;
 
-                        char s[50];
+                        // char s[50];
-                        snprintf(s, 50, "%d minutes and %d seconds", (uint8_t)(duration_ / 60000), (uint8_t)((duration_ / 1000) % 60));
+                        // snprintf(s, 50, "%02u:%02u:%02u", (uint8_t)(duration_ / 3600000UL), (uint8_t)(duration_ / 60000UL), (uint8_t)((duration_ / 1000UL) % 60));
-                        doc["duration"] = s;
+                        doc["duration"] = (uint8_t)(duration_ / 1000UL); // seconds
                         Mqtt::queue_publish("shower_data", doc.as<JsonObject>());
-                        LOG_DEBUG("[Shower] finished with duration %d", duration_);
+                        LOG_INFO("finished with duration %d", duration_);
                     }
                 }
 
@@ -106,21 +128,22 @@ void Shower::loop() {
     }
 }
 
+// turn off hot water to send a shot of cold
+void Shower::shower_alert_start() {
+    LOG_DEBUG("Shower Alert started");
+    (void)Command::call(EMSdevice::DeviceType::BOILER, "wwtapactivated", "false");
+    doing_cold_shot_   = true;
+    force_coldshot     = false;
+    alert_timer_start_ = uuid::get_uptime(); // timer starts now
+}
+
 // turn back on the hot water for the shower
 void Shower::shower_alert_stop() {
     if (doing_cold_shot_) {
         LOG_DEBUG("Shower Alert stopped");
         (void)Command::call(EMSdevice::DeviceType::BOILER, "wwtapactivated", "true");
         doing_cold_shot_ = false;
-    }
+        force_coldshot   = false;
-}
-// turn off hot water to send a shot of cold
-void Shower::shower_alert_start() {
-    if (shower_alert_) {
-        LOG_DEBUG("Shower Alert started");
-        (void)Command::call(EMSdevice::DeviceType::BOILER, "wwtapactivated", "false");
-        doing_cold_shot_   = true;
-        alert_timer_start_ = uuid::get_uptime(); // timer starts now
     }
 }
 
@@ -128,15 +151,11 @@ void Shower::shower_alert_start() {
 // and creates the HA config topic if HA enabled
 // force is used by EMSESP::publish_all_loop()
 void Shower::set_shower_state(bool state, bool force) {
-    if (!shower_timer_ && !shower_alert_) {
-        return;
-    }
-
     // sets the state
     shower_state_ = state;
 
     // only publish if that state has changed
-    static bool old_shower_state_;
+    static bool old_shower_state_ = false;
     if ((shower_state_ == old_shower_state_) && !force) {
         return;
     }
@@ -149,10 +168,15 @@ void Shower::set_shower_state(bool state, bool force) {
     // send out HA MQTT Discovery config topic
     if ((Mqtt::ha_enabled()) && (!ha_configdone_ || force)) {
         StaticJsonDocument<EMSESP_JSON_SIZE_LARGE> doc;
+        char                                       topic[Mqtt::MQTT_TOPIC_MAX_SIZE];
+        char                                       str[70];
+        char                                       stat_t[50];
 
+        //
+        // shower_active topic
+        //
         doc["name"] = "Shower Active";
 
-        char str[70];
         if (Mqtt::entity_format() == Mqtt::entityFormat::MULTI_SHORT) {
             snprintf(str, sizeof(str), "%s_shower_active", Mqtt::basename().c_str());
         } else {
@@ -161,7 +185,6 @@ void Shower::set_shower_state(bool state, bool force) {
         doc["uniq_id"]   = str;
         doc["object_id"] = str;
 
-        char stat_t[50];
         snprintf(stat_t, sizeof(stat_t), "%s/shower_active", Mqtt::basename().c_str());
         doc["stat_t"] = stat_t;
 
@@ -181,13 +204,39 @@ void Shower::set_shower_state(bool state, bool force) {
         JsonArray  ids = dev.createNestedArray("ids");
         ids.add(Mqtt::basename());
 
-        // add "availability" section
+        Mqtt::add_avty_to_doc(stat_t, doc.as<JsonObject>()); // add "availability" section
-        Mqtt::add_avty_to_doc(stat_t, doc.as<JsonObject>());
 
-        char topic[Mqtt::MQTT_TOPIC_MAX_SIZE];
         snprintf(topic, sizeof(topic), "binary_sensor/%s/shower_active/config", Mqtt::basename().c_str());
-
         ha_configdone_ = Mqtt::queue_ha(topic, doc.as<JsonObject>()); // publish the config payload with retain flag
+
+        //
+        // shower_duaration topic
+        //
+        doc.clear();
+
+        snprintf(str, sizeof(str), "%s_shower_duration", Mqtt::basename().c_str());
+
+        doc["uniq_id"]   = str;
+        doc["object_id"] = str;
+
+        snprintf(stat_t, sizeof(stat_t), "%s/shower_data", Mqtt::basename().c_str());
+        doc["stat_t"] = stat_t;
+
+        doc["name"]         = "Shower Duration";
+        doc["val_tpl"]      = "{{value_json.duration if value_json.duration is defined else 0}}";
+        doc["unit_of_meas"] = "s";
+        doc["stat_cla"]     = "measurement";
+        doc["dev_cla"]      = "duration";
+        doc["ent_cat"]      = "diagnostic";
+
+        JsonObject dev2 = doc.createNestedObject("dev");
+        JsonArray  ids2 = dev2.createNestedArray("ids");
+        ids2.add(Mqtt::basename());
+
+        Mqtt::add_avty_to_doc(stat_t, doc.as<JsonObject>(), "value_json.duration is defined"); // add "availability" section
+
+        snprintf(topic, sizeof(topic), "sensor/%s/shower_duration/config", Mqtt::basename().c_str());
+        Mqtt::queue_ha(topic, doc.as<JsonObject>()); // publish the config payload with retain flag
     }
 }
 

src/shower.h

@@ -30,6 +30,9 @@ class Shower {
 
     void set_shower_state(bool state, bool force = false);
 
+    // commands
+    static bool command_coldshot(const char * value, const int8_t id);
+
   private:
     static uuid::log::Logger logger_;
 

src/test/test.cpp

@@ -266,7 +266,7 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
         shell.printfln("Testing adding a general boiler & thermostat...");
         run_test("general");
         // shell.invoke_command("show devices");
-        // shell.invoke_command("show values");
+        shell.invoke_command("show values");
         shell.invoke_command("call system publish");
         // shell.invoke_command("show mqtt");
         ok = true;
@@ -280,6 +280,21 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
         ok = true;
     }
 
+    if (command == "coldshot") {
+        shell.printfln("Testing coldshot...");
+        run_test("general");
+
+#ifdef EMSESP_STANDALONE
+        AsyncWebServerRequest request;
+        request.method(HTTP_GET);
+        request.url("/api/boiler/coldshot");
+        EMSESP::webAPIService.webAPIService_get(&request);
+#else
+        shell.invoke_command("call boiler coldshot");
+#endif
+        ok = true;
+    }
+
     if (command == "string2minutes") {
         shell.printfln("Testing string2minutes()...");
         std::string time_s = "12:00";

src/test/test.h

@@ -26,11 +26,12 @@
 
 namespace emsesp {
 
+// #define EMSESP_DEBUG_DEFAULT "general"
+
 // #define EMSESP_DEBUG_DEFAULT "thermostat"
 // #define EMSESP_DEBUG_DEFAULT "solar"
 // #define EMSESP_DEBUG_DEFAULT "web"
 // #define EMSESP_DEBUG_DEFAULT "mqtt"
-#define EMSESP_DEBUG_DEFAULT "general"
 // #define EMSESP_DEBUG_DEFAULT "boiler"
 // #define EMSESP_DEBUG_DEFAULT "mqtt2"
 // #define EMSESP_DEBUG_DEFAULT "mqtt_nested"
@@ -53,6 +54,7 @@ namespace emsesp {
 // #define EMSESP_DEBUG_DEFAULT "custom"
 // #define EMSESP_DEBUG_DEFAULT "entity_dump"
 // #define EMSESP_DEBUG_DEFAULT "memory"
+#define EMSESP_DEBUG_DEFAULT "coldshot"
 
 class Test {
   public:

src/version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.6.0"
+#define EMSESP_APP_VERSION "3.6.1"

src/web/WebAPIService.cpp

@@ -121,9 +121,9 @@ void WebAPIService::parse(AsyncWebServerRequest * request, JsonObject & input) {
     if (return_code != CommandRet::OK) {
         char error[100];
         if (output.size()) {
-            snprintf(error, sizeof(error), "Call failed with error: %s (%s)", (const char *)output["message"], Command::return_code_string(return_code).c_str());
+            snprintf(error, sizeof(error), "API failed with error: %s (%s)", (const char *)output["message"], Command::return_code_string(return_code).c_str());
         } else {
-            snprintf(error, sizeof(error), "Call failed with error code (%s)", Command::return_code_string(return_code).c_str());
+            snprintf(error, sizeof(error), "API failed with error code (%s)", Command::return_code_string(return_code).c_str());
         }
         emsesp::EMSESP::logger().err(error);
         api_fails_++;

src/web/WebEntityService.cpp

@@ -31,6 +31,7 @@ WebEntityService::WebEntityService(AsyncWebServer * server, FS * fs, SecurityMan
 void WebEntityService::begin() {
     _fsPersistence.readFromFS();
     EMSESP::logger().info("Starting Custom entity service");
+    Mqtt::subscribe(EMSdevice::DeviceType::CUSTOM, "custom/#", nullptr); // use empty function callback
 }
 
 // this creates the entity file, saving it to the FS
@@ -60,6 +61,7 @@ StateUpdateResult WebEntity::update(JsonObject & root, WebEntity & webEntity) {
         Command::erase_command(EMSdevice::DeviceType::CUSTOM, entityItem.name.c_str());
     }
     webEntity.entityItems.clear();
+    EMSESP::webEntityService.ha_reset();
 
     if (root["entities"].is<JsonArray>()) {
         for (const JsonObject ei : root["entities"].as<JsonArray>()) {
@@ -219,8 +221,8 @@ bool WebEntityService::get_value_info(JsonObject & output, const char * cmd) {
         return false;
     }
     if (Helpers::toLower(cmd) == "commands") {
-        output["info"]     = "lists all values";
+        output["info"]     = "list all values";
-        output["commands"] = "lists all commands";
+        output["commands"] = "list all commands";
         for (const auto & entity : *entityItems) {
             output[entity.name] = "custom entitiy";
         }
@@ -319,6 +321,7 @@ void WebEntityService::publish(const bool force) {
 
     DynamicJsonDocument doc(EMSESP_JSON_SIZE_XLARGE);
     JsonObject          output     = doc.to<JsonObject>();
+    bool                ha_created = ha_registered_;
     for (const EntityItem & entityItem : *entityItems) {
         render_value(output, entityItem);
         // create HA config
@@ -382,11 +385,10 @@ void WebEntityService::publish(const bool force) {
 
             // add "availability" section
             Mqtt::add_avty_to_doc(stat_t, config.as<JsonObject>(), val_cond);
-            if (Mqtt::queue_ha(topic, config.as<JsonObject>())) {
+            ha_created |= Mqtt::queue_ha(topic, config.as<JsonObject>());
-                ha_registered_ = true;
-            }
         }
     }
+    ha_registered_ = ha_created;
     if (output.size() > 0) {
         Mqtt::queue_publish("custom_data", output);
     }

src/web/WebEntityService.h

@@ -63,6 +63,9 @@ class WebEntityService : public StatefulService<WebEntity> {
     uint8_t count_entities();
     uint8_t has_commands();
     void    generate_value_web(JsonObject & output);
+    void    ha_reset() {
+        ha_registered_ = false;
+    }
 
 
   private:

src/web/WebSchedulerService.cpp

@@ -31,6 +31,7 @@ WebSchedulerService::WebSchedulerService(AsyncWebServer * server, FS * fs, Secur
 void WebSchedulerService::begin() {
     _fsPersistence.readFromFS();
     EMSESP::logger().info("Starting Scheduler service");
+    Mqtt::subscribe(EMSdevice::DeviceType::SCHEDULER, "scheduler/#", nullptr); // use empty function callback
 }
 
 // this creates the scheduler file, saving it to the FS
@@ -70,6 +71,7 @@ StateUpdateResult WebScheduler::update(JsonObject & root, WebScheduler & webSche
         Command::erase_command(EMSdevice::DeviceType::SCHEDULER, scheduleItem.name.c_str());
     }
     webScheduler.scheduleItems.clear();
+    EMSESP::webSchedulerService.ha_reset();
 
     if (root["schedule"].is<JsonArray>()) {
         for (const JsonObject schedule : root["schedule"].as<JsonArray>()) {
@@ -218,6 +220,12 @@ void WebSchedulerService::publish_single(const char * name, const bool state) {
 
 // publish to Mqtt
 void WebSchedulerService::publish(const bool force) {
+    if (force) {
+        ha_registered_ = false;
+    }
+    if (!Mqtt::enabled()) {
+        return;
+    }
     EMSESP::webSchedulerService.read([&](WebScheduler & webScheduler) { scheduleItems = &webScheduler.scheduleItems; });
     if (scheduleItems->size() == 0) {
         return;
@@ -229,6 +237,7 @@ void WebSchedulerService::publish(const bool force) {
     }
 
     DynamicJsonDocument doc(EMSESP_JSON_SIZE_XLARGE);
+    bool                ha_created = ha_registered_;
     for (const ScheduleItem & scheduleItem : *scheduleItems) {
         if (!scheduleItem.name.empty() && !doc.containsKey(scheduleItem.name)) {
             if (EMSESP::system_.bool_format() == BOOL_FORMAT_TRUEFALSE) {
@@ -241,7 +250,7 @@ void WebSchedulerService::publish(const bool force) {
             }
 
             // create HA config
-            if (Mqtt::ha_enabled() && force) {
+            if (Mqtt::ha_enabled() && !ha_registered_) {
                 StaticJsonDocument<EMSESP_JSON_SIZE_MEDIUM> config;
                 char                                        stat_t[50];
                 snprintf(stat_t, sizeof(stat_t), "%s/scheduler_data", Mqtt::basename().c_str());
@@ -283,10 +292,11 @@ void WebSchedulerService::publish(const bool force) {
 
                 // add "availability" section
                 Mqtt::add_avty_to_doc(stat_t, config.as<JsonObject>(), val_cond);
-                Mqtt::queue_ha(topic, config.as<JsonObject>());
+                ha_created |= Mqtt::queue_ha(topic, config.as<JsonObject>());
             }
         }
     }
+    ha_registered_ = ha_created;
     if (doc.size() > 0) {
         Mqtt::queue_publish("scheduler_data", doc.as<JsonObject>());
     }

src/web/WebSchedulerService.h

@@ -58,6 +58,9 @@ class WebSchedulerService : public StatefulService<WebScheduler> {
     bool has_commands();
     bool command_setvalue(const char * value, const std::string name);
     bool get_value_info(JsonObject & output, const char * cmd);
+    void ha_reset() {
+        ha_registered_ = false;
+    }
 
 // make all functions public so we can test in the debug and standalone mode
 #ifndef EMSESP_STANDALONE
@@ -69,6 +72,7 @@ class WebSchedulerService : public StatefulService<WebScheduler> {
     FSPersistence<WebScheduler> _fsPersistence;
 
     std::list<ScheduleItem> * scheduleItems; // pointer to the list of schedule events
+    bool                      ha_registered_ = false;
 };
 
 } // namespace emsesp

src/web/WebStatusService.cpp

@@ -166,9 +166,9 @@ void WebStatusService::webStatusService(AsyncWebServerRequest * request) {
     if (Mqtt::enabled()) {
         statJson       = statsJson.createNestedObject();
         statJson["id"] = 5;
-        statJson["s"]  = Mqtt::publish_count();
+        statJson["s"]  = Mqtt::publish_count() - Mqtt::publish_fails();
         statJson["f"]  = Mqtt::publish_fails();
-        statJson["q"]  = Mqtt::publish_count() == 0 ? 100 : 100 - (uint8_t)((100 * Mqtt::publish_fails()) / (Mqtt::publish_count() + Mqtt::publish_fails()));
+        statJson["q"]  = Mqtt::publish_count() == 0 ? 100 : 100 - (uint8_t)((100 * Mqtt::publish_fails()) / Mqtt::publish_count());
     }
 
     statJson       = statsJson.createNestedObject();