Merge pull request #409 from MichaelDvP:dev

Customizations to names with more options

CHANGELOG_LATEST.md

@@ -26,7 +26,10 @@
 - remove MQTT retained configs if discovery is disabled
 - timeout 10 min for MQTT-QoS wait
 - Moduline 300 auto-temperatures T1-T4, RC300 romminfluencefactor
-- RC35 parameters [#392](https://github.com/emsesp/EMS-ESP32/issues/392)[#398](https://github.com/emsesp/EMS-ESP32/issues/398)
+- RC35 parameters [#392](https://github.com/emsesp/EMS-ESP32/issues/392), [#398](https://github.com/emsesp/EMS-ESP32/issues/398)
+- sync time with thermostat [#386](https://github.com/emsesp/EMS-ESP32/issues/386), [#408](https://github.com/emsesp/EMS-ESP32/issues/408)
+- set mode has immediate effect [#395](https://github.com/emsesp/EMS-ESP32/issues/395)
+- min/max in web value setting
 
 ### Fixed
 
@@ -44,6 +47,7 @@
 - Non-nested MQTT would corrupt the json [#354](https://github.com/emsesp/EMS-ESP32/issues/354)
 - Burner selected max power can have a value higher than 100% [#314](https://github.com/emsesp/EMS-ESP32/issues/314)
 - some missing fahrenheit calculations
+- limited number of exclusions [#339](https://github.com/emsesp/EMS-ESP32/issues/339)
 
 ### Changed
 
@@ -59,6 +63,7 @@
 - Wired renamed to Ethernet
 - removed system/pin command, new commands in analogsensors
 - system/info device-info split to name/version/brand
+- exclude list uses short-names, possible flags for web/api/mqtt excludes, readonly and favorite (selection not yet implemented)
 
 ## **BREAKING CHANGES:**
 
@@ -70,3 +75,4 @@
   - Boiler `ww` tag renamed to `dhw`. Any custom Home Assistant lovelace dashboards will need updating.
   - Renamed description of `wwtapactivated` to "turn on/off DHW". Otherwise would have looked like "boiler_dhw_turn_on_off_dhw" in HA.
   - `/api/system/info` endpoint has updated keys. Now lowercase, no underscores and not capitalized. Replace "handlers" with "handlers received", "handlers fetched" and "handlers pending".
+  - exclude list changed from 3.4.0b8 to 3.4.0b9 and has to be updated by user.

interface/package.json

@@ -8,10 +8,10 @@
     "@emotion/styled": "^11.8.1",
     "@msgpack/msgpack": "^2.7.2",
     "@mui/icons-material": "^5.5.1",
-    "@mui/material": "^5.5.1",
+    "@mui/material": "^5.5.2",
     "@types/lodash": "^4.14.180",
-    "@types/node": "^17.0.21",
+    "@types/node": "^17.0.23",
-    "@types/react": "^17.0.41",
+    "@types/react": "^17.0.43",
     "@types/react-dom": "^17.0.14",
     "@types/react-router-dom": "^5.3.3",
     "async-validator": "^4.0.7",
@@ -29,7 +29,7 @@
     "react-router-dom": "^6.2.2",
     "react-scripts": "5.0.0",
     "sockette": "^2.0.6",
-    "typescript": "^4.6.2"
+    "typescript": "^4.6.3"
   },
   "scripts": {
     "start": "react-app-rewired start",

interface/src/project/DashboardData.tsx

@@ -239,7 +239,7 @@ const DashboardData: FC = () => {
                 sx={{ width: '30ch' }}
                 type={deviceValue.u ? 'number' : 'text'}
                 onChange={updateValue(setDeviceValue)}
-                inputProps={{ step: deviceValue.s }}
+                inputProps={deviceValue.u ? { min: deviceValue.m, max: deviceValue.x, step: deviceValue.s } : {}}
                 InputProps={{
                   startAdornment: <InputAdornment position="start">{DeviceValueUOM_s[deviceValue.u]}</InputAdornment>
                 }}

interface/src/project/SettingsCustomization.tsx

@@ -138,7 +138,7 @@ const SettingsCustomization: FC = () => {
 
   const saveCustomization = async () => {
     if (deviceEntities && selectedDevice) {
-      const exclude_entities = deviceEntities.filter((de) => de.x).map((new_de) => new_de.i);
+      const exclude_entities = deviceEntities.filter((de) => de.x).map((new_de) => "07" + new_de.s);
       try {
         const response = await EMSESP.writeExcludeEntities({
           id: selectedDevice,

interface/src/project/types.ts

@@ -133,6 +133,8 @@ export interface DeviceValue {
   l: string[]; // list
   h?: string; // help text
   s?: string; // steps for up/down
+  m?: string; // min
+  x?: string; // max
 }
 
 export interface DeviceData {
@@ -150,7 +152,7 @@ export interface DeviceEntity {
 
 export interface ExcludeEntities {
   id: number;
-  entity_ids: number[];
+  entity_ids: string[];
 }
 
 export interface UniqueID {

lib/ESPAsyncWebServer/AsyncJson.h

@@ -173,7 +173,7 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
     ArJsonRequestHandlerFunction _onRequest;
     size_t                       _contentLength;
 #ifndef ARDUINOJSON_5_COMPATIBILITY
-    const size_t maxJsonBufferSize;
+    size_t _maxJsonBufferSize;
 #endif
     size_t _maxContentLength;
 
@@ -182,7 +182,7 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
         : _uri(uri)
         , _method(HTTP_POST | HTTP_PUT | HTTP_PATCH)
         , _onRequest(onRequest)
-        , maxJsonBufferSize(maxJsonBufferSize)
+        , _maxJsonBufferSize(maxJsonBufferSize)
         , _maxContentLength(16384) {
     }
 
@@ -192,6 +192,9 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
     void setMaxContentLength(int maxContentLength) {
         _maxContentLength = maxContentLength;
     }
+    void setMaxJsonBufferSize(size_t maxJsonBufferSize) {
+        _maxJsonBufferSize = maxJsonBufferSize;
+    }
     void onRequest(ArJsonRequestHandlerFunction fn) {
         _onRequest = fn;
     }
@@ -216,7 +219,7 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
     virtual void handleRequest(AsyncWebServerRequest * request) override final {
         if (_onRequest) {
             if (request->_tempObject != NULL) {
-                DynamicJsonDocument  jsonBuffer(this->maxJsonBufferSize);
+                DynamicJsonDocument  jsonBuffer(this->_maxJsonBufferSize);
                 DeserializationError error = deserializeJson(jsonBuffer, (uint8_t *)(request->_tempObject));
                 if (!error) {
                     JsonVariant json = jsonBuffer.as<JsonVariant>();

lib_standalone/AsyncJson.h

@@ -90,8 +90,8 @@ class PrettyAsyncJsonResponse {
 class MsgpackAsyncJsonResponse {
   protected:
     DynamicJsonDocument _jsonBuffer;
-    JsonVariant _root;
+    JsonVariant         _root;
-    bool        _isValid;
+    bool                _isValid;
 
   public:
     MsgpackAsyncJsonResponse(bool isArray = false, size_t maxJsonBufferSize = DYNAMIC_JSON_DOCUMENT_SIZE)
@@ -187,6 +187,7 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
     ArJsonRequestHandlerFunction _onRequest;
     size_t                       _contentLength;
     size_t                       _maxContentLength;
+    size_t                       _maxJsonBufferSize;
 
   public:
     AsyncCallbackJsonWebHandler(const String & uri, ArJsonRequestHandlerFunction onRequest, size_t maxJsonBufferSize = DYNAMIC_JSON_DOCUMENT_SIZE)
@@ -202,6 +203,9 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
     void setMaxContentLength(int maxContentLength) {
         _maxContentLength = maxContentLength;
     }
+    void setMaxJsonBufferSize(int maxJsonBufferSize) {
+        _maxJsonBufferSize = maxJsonBufferSize;
+    }
     void onRequest(ArJsonRequestHandlerFunction fn) {
         _onRequest = fn;
     }

mock-api/server.js

@@ -926,7 +926,7 @@ rest_server.post(EMSESP_DEVICEENTITIES_ENDPOINT, (req, res) => {
 })
 
 rest_server.post(EMSESP_EXCLUDE_ENTITIES_ENDPOINT, (req, res) => {
-  console.log('exclude list for productid ' + req.body.product_id + ' device_id ' + req.body.device_id + ' entities:')
+  console.log('exclude list for unique id ' + req.body.id + ' and entities:')
   console.log(req.body.entity_ids)
   res.sendStatus(200)
 })

src/command.cpp

@@ -281,7 +281,7 @@ uint8_t Command::call(const uint8_t device_type, const char * cmd, const char *
             return_code = ((cf->cmdfunction_json_)(value, id, output)) ? CommandRet::OK : CommandRet::ERROR;
         }
 
-        if (cf->cmdfunction_) {
+        if (cf->cmdfunction_ && !EMSESP::cmd_is_readonly(device_type, cmd, id)) {
             return_code = ((cf->cmdfunction_)(value, id)) ? CommandRet::OK : CommandRet::ERROR;
         }
 

src/devices/boiler.cpp

@@ -1478,12 +1478,12 @@ bool Boiler::set_reset(const char * value, const int8_t id) {
     if (num == 1) {
         // LOG_INFO(F("Reset boiler maintenance message"));
         write_command(0x05, 0x08, 0xFF, 0x1C);
-        has_update(reset_);
+        has_update(&reset_);
         return true;
     } else if (num == 2) {
         // LOG_INFO(F("Reset boiler error message"));
         write_command(0x05, 0x00, 0x5A); // error reset
-        has_update(reset_);
+        has_update(&reset_);
         return true;
     }
     return false;

src/devices/thermostat.cpp

@@ -379,8 +379,8 @@ void Thermostat::add_ha_climate(std::shared_ptr<HeatingCircuit> hc) const {
         return;
     }
 
-    if (Helpers::hasValue(hc->selTemp) && is_visible(&hc->selTemp)) {
+    if (Helpers::hasValue(hc->selTemp) && is_readable(&hc->selTemp)) {
-        if (Helpers::hasValue(hc->roomTemp) && is_visible(&hc->roomTemp)) {
+        if (Helpers::hasValue(hc->roomTemp) && is_readable(&hc->roomTemp)) {
             hc->climate = 1;
         } else {
             hc->climate = 0;
@@ -1205,14 +1205,14 @@ void Thermostat::process_RCTime(std::shared_ptr<const Telegram> telegram) {
         return;
     }
 
-    if (telegram->message_data[7] & 0x0C) { // date and time not valid
+    if ((telegram->message_data[7] & 0x0C) && has_command(&dateTime_)) { // date and time not valid
-        set_datetime("ntp", -1);            // set from NTP
+        set_datetime("ntp", -1);                                         // set from NTP
         return;
     }
 
     // render date to HH:MM:SS DD/MM/YYYY
     // had to create separate buffers because of how printf works
-    char date[25];
+    char date[sizeof(dateTime_)];
     char buf1[6];
     char buf2[6];
     char buf3[6];
@@ -1231,6 +1231,33 @@ void Thermostat::process_RCTime(std::shared_ptr<const Telegram> telegram) {
              Helpers::itoa((telegram->message_data[0] & 0x7F) + 2000, buf6) // year
     );
     has_update(dateTime_, date, sizeof(dateTime_));
+
+    // check clock
+    time_t now    = time(nullptr);
+    tm *   tm_    = localtime(&now);
+    bool   ntp_   = tm_->tm_year > 110; // year 2010 and up, time is valid
+    tm_->tm_year  = telegram->message_data[0] + 100;
+    tm_->tm_mon   = telegram->message_data[1] - 1;
+    tm_->tm_mday  = telegram->message_data[3];
+    tm_->tm_hour  = telegram->message_data[2];
+    tm_->tm_min   = telegram->message_data[4];
+    tm_->tm_sec   = telegram->message_data[5];
+    tm_->tm_isdst = telegram->message_data[7] & 0x01;
+    time_t ttime  = mktime(tm_);           // thermostat time
+    if (ntp_ && has_command(&dateTime_)) { // have NTP time and command
+        double difference = difftime(now, ttime);
+        if (difference > 15 || difference < -15) {
+            set_datetime("ntp", -1); // set from NTP
+            LOG_INFO(F("thermostat time correction from ntp"));
+        }
+    }
+    if (!ntp_ && tm_->tm_year > 110) { // emsesp clock not set, but thermostat clock
+        struct timeval newnow = {.tv_sec = ttime};
+#ifndef EMSESP_STANDALONE
+        settimeofday(&newnow, nullptr);
+#endif
+        LOG_INFO(F("ems-esp time set from thermostat"));
+    }
 }
 
 // process_RCError - type 0xA2 - error message - 14 bytes long
@@ -2001,6 +2028,14 @@ bool Thermostat::set_mode_n(const uint8_t mode, const uint8_t hc_num) {
     // post validate is the corresponding monitor or set type IDs as they can differ per model
     write_command(set_typeid, offset, set_mode_value, validate_typeid);
 
+    // set hc->mode temporary until validate is received
+    if (model() == EMSdevice::EMS_DEVICE_FLAG_RC10) {
+        hc->mode = set_mode_value >> 1;
+    } else {
+        hc->mode = set_mode_value;
+    }
+    has_update(&hc->mode);
+
     return true;
 }
 

src/emsdevice.cpp

@@ -26,7 +26,7 @@ namespace emsesp {
 uint8_t EMSdevice::count_entities() {
     uint8_t count = 0;
     for (auto & dv : devicevalues_) {
-        if (dv.has_state(DeviceValueState::DV_VISIBLE) && dv.hasValue()) {
+        if (!dv.has_state(DeviceValueState::DV_WEB_EXCLUDE) && dv.hasValue()) {
             count++;
         }
     }
@@ -248,7 +248,7 @@ bool EMSdevice::is_fetch(uint16_t telegram_id) const {
 // called from the command 'entities'
 void EMSdevice::list_device_entries(JsonObject & output) const {
     for (const auto & dv : devicevalues_) {
-        if (dv.has_state(DeviceValueState::DV_VISIBLE) && dv.type != DeviceValueType::CMD && dv.full_name) {
+        if (!dv.has_state(DeviceValueState::DV_WEB_EXCLUDE) && dv.type != DeviceValueType::CMD && dv.full_name) {
             // if we have a tag prefix it
             char key[50];
             if (!EMSdevice::tag_to_mqtt(dv.tag).empty()) {
@@ -399,15 +399,17 @@ void EMSdevice::register_device_value(uint8_t                             tag,
     uint8_t dv_id = get_next_dv_id();
 
     // determine state
-    uint8_t state = DeviceValueState::DV_VISIBLE; // default to visible
+    uint8_t state = DeviceValueState::DV_DEFAULT;
 
     // scan through customizations to see if it's on the exclusion list by matching the productID and deviceID
     EMSESP::webCustomizationService.read([&](WebCustomization & settings) {
         for (EntityCustomization entityCustomization : settings.entityCustomizations) {
             if ((entityCustomization.product_id == product_id()) && (entityCustomization.device_id == device_id())) {
-                for (uint8_t entity_id : entityCustomization.entity_ids) {
+                std::string entity = tag < DeviceValueTAG::TAG_HC1 ? read_flash_string(short_name) : tag_to_string(tag) + "/" + read_flash_string(short_name);
-                    if (entity_id == dv_id) {
+                for (std::string entity_id : entityCustomization.entity_ids) {
-                        state = DeviceValueState::DV_DEFAULT; // it's on the exclude list, turn off active and visible flags
+                    uint8_t flag = Helpers::hextoint(entity_id.substr(0, 2).c_str());
+                    if (entity_id.substr(2) == entity) {
+                        state = flag << 4; // set state high bits to flag, turn off active and ha flags
                         break;
                     }
                 }
@@ -476,11 +478,33 @@ void EMSdevice::register_device_value(uint8_t                             tag,
     register_device_value(tag, value_p, type, options, name, uom, nullptr, 0, 0);
 }
 
-// check if value is visible
+// check if value is readable via mqtt/api
-bool EMSdevice::is_visible(const void * value_p) const {
+bool EMSdevice::is_readable(const void * value_p) const {
     for (const auto & dv : devicevalues_) {
         if (dv.value_p == value_p) {
-            return dv.has_state(DeviceValueState::DV_VISIBLE);
+            return !dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE);
+        }
+    }
+    return false;
+}
+
+// check if value/command is readonly
+bool EMSdevice::is_readonly(const std::string & cmd, const int8_t id) const {
+    uint8_t tag = id > 0 ? DeviceValueTAG::TAG_HC1 + id - 1 : DeviceValueTAG::TAG_NONE;
+    for (const auto & dv : devicevalues_) {
+        // check command name and tag, id -1 is default hc and only checks name
+        if (dv.has_cmd && read_flash_string(dv.short_name) == cmd && (dv.tag < DeviceValueTAG::TAG_HC1 || dv.tag == tag || id == -1)) {
+            return dv.has_state(DeviceValueState::DV_READONLY);
+        }
+    }
+    return true; // not found, no write
+}
+
+// check if value has a registered command
+bool EMSdevice::has_command(const void * value_p) const {
+    for (const auto & dv : devicevalues_) {
+        if (dv.value_p == value_p) {
+            return dv.has_cmd && !dv.has_state(DeviceValueState::DV_READONLY);
         }
     }
     return false;
@@ -493,7 +517,7 @@ void EMSdevice::publish_value(void * value_p) const {
     }
 
     for (const auto & dv : devicevalues_) {
-        if (dv.value_p == value_p && dv.has_state(DeviceValueState::DV_VISIBLE)) {
+        if (dv.value_p == value_p && !dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE)) {
             char topic[Mqtt::MQTT_TOPIC_MAX_SIZE];
             if (Mqtt::publish_single2cmd()) {
                 if (dv.tag >= DeviceValueTAG::TAG_HC1 && dv.tag <= DeviceValueTAG::TAG_WWC4) {
@@ -593,7 +617,7 @@ std::string EMSdevice::get_value_uom(const char * key) const {
 
     // look up key in our device value list
     for (const auto & dv : devicevalues_) {
-        if ((dv.has_state(DeviceValueState::DV_VISIBLE) && dv.full_name) && (read_flash_string(dv.full_name) == key_p)) {
+        if ((!dv.has_state(DeviceValueState::DV_WEB_EXCLUDE) && dv.full_name) && (read_flash_string(dv.full_name) == key_p)) {
             // ignore TIME since "minutes" is already added to the string value
             if ((dv.uom == DeviceValueUOM::NONE) || (dv.uom == DeviceValueUOM::MINUTES)) {
                 break;
@@ -618,7 +642,7 @@ void EMSdevice::generate_values_web(JsonObject & output) {
         //  1. full_name cannot be empty
         //  2. it must have a valid value, if it is not a command like 'reset'
 
-        if (dv.has_state(DeviceValueState::DV_VISIBLE) && dv.full_name && (dv.hasValue() || (dv.type == DeviceValueType::CMD))) {
+        if (!dv.has_state(DeviceValueState::DV_WEB_EXCLUDE) && dv.full_name && (dv.hasValue() || (dv.type == DeviceValueType::CMD))) {
             JsonObject obj        = data.createNestedObject(); // create the object, we know there is a value
             uint8_t    fahrenheit = 0;
 
@@ -682,7 +706,7 @@ void EMSdevice::generate_values_web(JsonObject & output) {
             }
 
             // add commands and options
-            if (dv.has_cmd) {
+            if (dv.has_cmd && !dv.has_state(DeviceValueState::DV_READONLY)) {
                 // add the name of the Command function
                 if (dv.tag >= DeviceValueTAG::TAG_HC1) {
                     obj["c"] = tag_to_mqtt(dv.tag) + "/" + read_flash_string(dv.short_name);
@@ -717,6 +741,11 @@ void EMSdevice::generate_values_web(JsonObject & output) {
                     } else if (divider < 0) {
                         obj["s"] = Helpers::render_value(s, (-1) * divider, 0);
                     }
+                    int16_t dv_set_min, dv_set_max;
+                    if (dv.get_min_max(dv_set_min, dv_set_max)) {
+                        obj["m"] = Helpers::render_value(s, dv_set_min, 0);
+                        obj["x"] = Helpers::render_value(s, dv_set_max, 0);
+                    }
                 }
             }
         }
@@ -725,21 +754,24 @@ void EMSdevice::generate_values_web(JsonObject & output) {
 
 // reset all entities to being visible
 // this is called before loading in the exclude entities list from the customization service
-void EMSdevice::reset_exclude_entities() {
+void EMSdevice::reset_entity_masks() {
     for (auto & dv : devicevalues_) {
-        dv.add_state(DeviceValueState::DV_VISIBLE);
+        dv.state &= 0x0F;
     }
 }
 
-// disable/exclude a device entity based on its unique id
+// disable/exclude/mask_out a device entity based on the id
-void EMSdevice::exclude_entity(uint8_t id) {
+void EMSdevice::mask_entity(std::string entity_id) {
+    // first character contains mask flags
+    uint8_t flag = Helpers::hextoint(entity_id.substr(0, 2).c_str());
     for (auto & dv : devicevalues_) {
-        if (dv.id == id) {
+        std::string entity = dv.tag < DeviceValueTAG::TAG_HC1 ? read_flash_string(dv.short_name) : tag_to_string(dv.tag) + "/" + read_flash_string(dv.short_name);
+        if (entity == entity_id.substr(2)) {
 #if defined(EMSESP_USE_SERIAL)
-            Serial.print("exclude_entity() Removing Visible for device value: ");
+            Serial.print("mask_entity() Removing Visible for device value: ");
             Serial.println(read_flash_string(dv.full_name).c_str());
 #endif
-            dv.remove_state(DeviceValueState::DV_VISIBLE); // this will remove from MQTT payloads and showing in web & console
+            dv.state = (dv.state & 0x0F) | (flag << 4); // set state high bits to flag, turn off active and ha flags
             return;
         }
     }
@@ -830,7 +862,7 @@ void EMSdevice::generate_values_web_all(JsonArray & output) {
         }
 
         // is it marked as excluded?
-        obj["x"] = !dv.has_state(DeviceValueState::DV_VISIBLE);
+        obj["x"] = dv.has_state(DeviceValueState::DV_WEB_EXCLUDE);
 
         // add the unique ID
         obj["i"] = dv.id;
@@ -985,8 +1017,9 @@ bool EMSdevice::get_value_info(JsonObject & output, const char * cmd, const int8
                 json["uom"] = uom_to_string(dv.uom);
             }
 
-            json["writeable"] = dv.has_cmd;
+            json["readable"]  = !dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE);
-            json["visible"]   = dv.has_state(DeviceValueState::DV_VISIBLE);
+            json["writeable"] = dv.has_cmd && !dv.has_state(DeviceValueState::DV_READONLY);
+            json["visible"]   = !dv.has_state(DeviceValueState::DV_WEB_EXCLUDE);
 
             // if there is no value, mention it
             if (!json.containsKey(value)) {
@@ -1030,14 +1063,11 @@ bool EMSdevice::generate_values(JsonObject & output, const uint8_t tag_filter, c
 
         // check conditions:
         //  1. it must have a valid value (state is active)
-        //  2. it must have a visible, unless the output_target is MQTT
+        //  2. it must have a visible flag
         //  3. it must match the given tag filter or have an empty tag
-        bool conditions = ((tag_filter == DeviceValueTAG::TAG_NONE) || (tag_filter == dv.tag)) && dv.has_state(DeviceValueState::DV_ACTIVE);
+        //  4. it must not have the exclude flag set or outputs to console
-        //  4. for MQTT we want to always show the special HA entities (they have an empty fullname)
+        if (dv.has_state(DeviceValueState::DV_ACTIVE) && dv.full_name && (tag_filter == DeviceValueTAG::TAG_NONE || tag_filter == dv.tag)
-        bool visible = ((dv.has_state(DeviceValueState::DV_VISIBLE)) || ((output_target == OUTPUT_TARGET::MQTT) && (!dv.full_name)));
+            && (output_target == OUTPUT_TARGET::CONSOLE || !dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE))) {
-        conditions &= visible;
-
-        if (conditions) {
             has_values = true; // flagged if we actually have data
 
             // we have a tag if it matches the filter given, and that the tag name is not empty/""
@@ -1045,7 +1075,7 @@ bool EMSdevice::generate_values(JsonObject & output, const uint8_t tag_filter, c
 
             // create the name for the JSON key
             char name[80];
-            if (output_target == OUTPUT_TARGET::API_VERBOSE) {
+            if (output_target == OUTPUT_TARGET::API_VERBOSE || output_target == OUTPUT_TARGET::CONSOLE) {
                 if (have_tag) {
                     snprintf(name, 80, "%s %s", tag_to_string(dv.tag).c_str(), read_flash_string(dv.full_name).c_str()); // prefix the tag
                 } else {
@@ -1158,7 +1188,7 @@ bool EMSdevice::generate_values(JsonObject & output, const uint8_t tag_filter, c
                 } else if ((dv.type == DeviceValueType::TIME) && Helpers::hasValue(*(uint32_t *)(dv.value_p))) {
                     uint32_t time_value = *(uint32_t *)(dv.value_p);
                     time_value          = Helpers::round2(time_value, divider); // sometimes we need to divide by 60
-                    if (output_target == EMSdevice::OUTPUT_TARGET::API_VERBOSE) {
+                    if (output_target == OUTPUT_TARGET::API_VERBOSE || output_target == OUTPUT_TARGET::CONSOLE) {
                         char time_s[40];
                         snprintf(time_s,
                                  sizeof(time_s),
@@ -1190,7 +1220,7 @@ bool EMSdevice::generate_values(JsonObject & output, const uint8_t tag_filter, c
 void EMSdevice::mqtt_ha_entity_config_remove() {
     for (auto & dv : devicevalues_) {
         if (dv.has_state(DeviceValueState::DV_HA_CONFIG_CREATED)
-            && ((!dv.has_state(DeviceValueState::DV_VISIBLE)) || (!dv.has_state(DeviceValueState::DV_ACTIVE)))) {
+            && ((dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE)) || (!dv.has_state(DeviceValueState::DV_ACTIVE)))) {
             dv.remove_state(DeviceValueState::DV_HA_CONFIG_CREATED);
             if (dv.short_name == FL_(climate)[0]) {
                 Mqtt::publish_ha_climate_config(dv.tag, false, true); // delete topic (remove = true)
@@ -1210,7 +1240,7 @@ void EMSdevice::mqtt_ha_entity_config_create() {
     // create climate if roomtemp is visible
     // create the discovery topic if if hasn't already been created, not a command (like reset) and is active and visible
     for (auto & dv : devicevalues_) {
-        if ((dv.short_name == FL_(climate)[0]) && dv.has_state(DeviceValueState::DV_VISIBLE) && dv.has_state(DeviceValueState::DV_ACTIVE)) {
+        if ((dv.short_name == FL_(climate)[0]) && !dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE) && dv.has_state(DeviceValueState::DV_ACTIVE)) {
             if (*(int8_t *)(dv.value_p) == 1 && (!dv.has_state(DeviceValueState::DV_HA_CONFIG_CREATED) || dv.has_state(DeviceValueState::DV_HA_CLIMATE_NO_RT))) {
                 dv.remove_state(DeviceValueState::DV_HA_CLIMATE_NO_RT);
                 dv.add_state(DeviceValueState::DV_HA_CONFIG_CREATED);
@@ -1223,7 +1253,7 @@ void EMSdevice::mqtt_ha_entity_config_create() {
             }
         }
         if (!dv.has_state(DeviceValueState::DV_HA_CONFIG_CREATED) && (dv.type != DeviceValueType::CMD) && dv.has_state(DeviceValueState::DV_ACTIVE)
-            && dv.has_state(DeviceValueState::DV_VISIBLE)) {
+            && !dv.has_state(DeviceValueState::DV_API_MQTT_EXCLUDE)) {
             // create_device_config is only done once for the EMS device. It can added to any entity, so we take the first
             Mqtt::publish_ha_sensor_config(dv, name(), brand_to_string(), false, create_device_config);
             dv.add_state(DeviceValueState::DV_HA_CONFIG_CREATED);

src/emsdevice.h

@@ -183,8 +183,8 @@ class EMSdevice {
     char * show_telegram_handlers(char * result, const size_t len, const uint8_t handlers);
     void   show_mqtt_handlers(uuid::console::Shell & shell) const;
     void   list_device_entries(JsonObject & output) const;
-    void   exclude_entity(uint8_t entity_id);
+    void   mask_entity(std::string entity_id);
-    void   reset_exclude_entities();
+    void   reset_entity_masks();
 
     using process_function_p = std::function<void(std::shared_ptr<const Telegram>)>;
 
@@ -195,7 +195,7 @@ class EMSdevice {
     bool        get_value_info(JsonObject & root, const char * cmd, const int8_t id);
     void        get_dv_info(JsonObject & json);
 
-    enum OUTPUT_TARGET : uint8_t { API_VERBOSE, API_SHORTNAMES, MQTT };
+    enum OUTPUT_TARGET : uint8_t { API_VERBOSE, API_SHORTNAMES, MQTT, CONSOLE };
     bool generate_values(JsonObject & output, const uint8_t tag_filter, const bool nested, const uint8_t output_target);
     void generate_values_web(JsonObject & output);
     void generate_values_web_all(JsonArray & output);
@@ -242,7 +242,9 @@ class EMSdevice {
 
     void read_command(const uint16_t type_id, uint8_t offset = 0, uint8_t length = 0) const;
 
-    bool is_visible(const void * value_p) const;
+    bool is_readable(const void * value_p) const;
+    bool is_readonly(const std::string & cmd, const int8_t id) const;
+    bool has_command(const void * value_p) const;
     void publish_value(void * value_p) const;
     void publish_all_values();
 

src/emsdevicevalue.h

@@ -112,9 +112,13 @@ class DeviceValue {
     enum DeviceValueState : uint8_t {
         DV_DEFAULT           = 0,        // 0 - does not yet have a value
         DV_ACTIVE            = (1 << 0), // 1 - has a validated real value
-        DV_VISIBLE           = (1 << 1), // 2 - shown on web, console and on MQTT payload. Otherwise hidden
+        DV_HA_CONFIG_CREATED = (1 << 1), // 2 - set if the HA config topic has been created
-        DV_HA_CONFIG_CREATED = (1 << 2), // 4 - set if the HA config topic has been created
+        DV_HA_CLIMATE_NO_RT  = (1 << 2), // 3 - climate created without roomTemp
-        DV_HA_CLIMATE_NO_RT  = (1 << 3)  // 8 - climate created without roomTemp
+                                         // high nibble as mask for exclusions
+        DV_WEB_EXCLUDE      = (1 << 4),  // 16 - not shown on web
+        DV_API_MQTT_EXCLUDE = (1 << 5),  // 32 - not shown on mqtt, API
+        DV_READONLY         = (1 << 6),  // 64 - read only
+        DV_FAVORITE         = (1 << 7)   // 128 - sort to front
     };
 
     uint8_t                             device_type;  // EMSdevice::DeviceType

src/emsesp.cpp

@@ -106,6 +106,15 @@ void EMSESP::fetch_device_values_type(const uint8_t device_type) {
     }
 }
 
+bool EMSESP::cmd_is_readonly(const uint8_t device_type, const char * cmd, const int8_t id) {
+    for (const auto & emsdevice : emsdevices) {
+        if (emsdevice && (emsdevice->device_type() == device_type)) {
+            return emsdevice->is_readonly(cmd, id);
+        }
+    }
+    return false;
+}
+
 // clears list of recognized devices
 void EMSESP::clear_all_devices() {
     // temporarily removed: clearing the list causes a crash, the associated commands and mqtt should also be removed.
@@ -366,7 +375,7 @@ void EMSESP::show_device_values(uuid::console::Shell & shell) {
                 DynamicJsonDocument doc(EMSESP_JSON_SIZE_XXLARGE_DYN); // use max size
                 JsonObject          json = doc.to<JsonObject>();
 
-                emsdevice->generate_values(json, DeviceValueTAG::TAG_NONE, true, EMSdevice::OUTPUT_TARGET::API_VERBOSE); // verbose mode and nested
+                emsdevice->generate_values(json, DeviceValueTAG::TAG_NONE, true, EMSdevice::OUTPUT_TARGET::CONSOLE); // verbose mode and nested
 
                 // print line
                 uint8_t id = 0;

src/emsesp.h

@@ -128,6 +128,7 @@ class EMSESP {
 
     static void send_raw_telegram(const char * data);
     static bool device_exists(const uint8_t device_id);
+    static bool cmd_is_readonly(const uint8_t device_type, const char * cmd, const int8_t id);
 
     static uint8_t count_devices(const uint8_t device_type);
     static uint8_t count_devices();

src/mqtt.cpp

@@ -925,7 +925,7 @@ void Mqtt::publish_ha_sensor_config(DeviceValue & dv, const std::string & model,
                              dv.short_name,
                              dv.uom,
                              remove,
-                             dv.has_cmd,
+                             dv.has_cmd && !dv.has_state(DeviceValueState::DV_READONLY),
                              dv.options,
                              dv.options_size,
                              dv_set_min,

src/system.cpp

@@ -1054,15 +1054,15 @@ bool System::command_customizations(const char * value, const int8_t id, JsonObj
         }
 
         // exclude entities
-        JsonArray exclude_entitiesJson = node.createNestedArray("exclude_entities");
+        JsonArray mask_entitiesJson = node.createNestedArray("masked_entities");
         for (const auto & entityCustomization : settings.entityCustomizations) {
-            JsonObject entityJson    = exclude_entitiesJson.createNestedObject();
+            JsonObject entityJson    = mask_entitiesJson.createNestedObject();
             entityJson["product_id"] = entityCustomization.product_id;
             entityJson["device_id"]  = entityCustomization.device_id;
 
-            JsonArray exclude_entityJson = entityJson.createNestedArray("entity_ids");
+            JsonArray mask_entityJson = entityJson.createNestedArray("entities");
-            for (uint8_t entity_id : entityCustomization.entity_ids) {
+            for (std::string entity_id : entityCustomization.entity_ids) {
-                exclude_entityJson.add(entity_id);
+                mask_entityJson.add(entity_id);
             }
         }
     });

src/test/test.cpp

@@ -609,8 +609,7 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
         // toggle mode
         for (const auto & emsdevice : EMSESP::emsdevices) {
             if (emsdevice->unique_id() == 1) { // boiler
-                uint8_t entity_id = 47;        // wwseltemp
+                emsdevice->mask_entity("07wwseltemp");
-                emsdevice->exclude_entity(entity_id);
                 break;
             }
         }

src/version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.4.0b8"
+#define EMSESP_APP_VERSION "3.4.0b9"

src/web/WebCustomizationService.cpp

@@ -46,7 +46,8 @@ WebCustomizationService::WebCustomizationService(AsyncWebServer * server, FS * f
                securityManager->wrapRequest(std::bind(&WebCustomizationService::reset_customization, this, _1), AuthenticationPredicates::IS_ADMIN));
 
     _exclude_entities_handler.setMethod(HTTP_POST);
-    _exclude_entities_handler.setMaxContentLength(1024);
+    _exclude_entities_handler.setMaxContentLength(2048);
+    _exclude_entities_handler.setMaxJsonBufferSize(2048);
     server->addHandler(&_exclude_entities_handler);
 
     _device_entities_handler.setMethod(HTTP_POST);
@@ -85,7 +86,7 @@ void WebCustomization::read(WebCustomization & settings, JsonObject & root) {
         entityJson["device_id"]  = entityCustomization.device_id;
 
         JsonArray exclude_entityJson = entityJson.createNestedArray("entity_ids");
-        for (uint8_t entity_id : entityCustomization.entity_ids) {
+        for (std::string entity_id : entityCustomization.entity_ids) {
             exclude_entityJson.add(entity_id);
         }
     }
@@ -132,7 +133,7 @@ StateUpdateResult WebCustomization::update(JsonObject & root, WebCustomization &
             new_entry.device_id  = exclude_entities["device_id"];
 
             for (const JsonVariant exclude_entity_id : exclude_entities["entity_ids"].as<JsonArray>()) {
-                new_entry.entity_ids.push_back(exclude_entity_id.as<uint8_t>()); // add entity list
+                new_entry.entity_ids.push_back(exclude_entity_id.as<std::string>()); // add entity list
             }
             settings.entityCustomizations.push_back(new_entry); // save the new object
         }
@@ -215,14 +216,14 @@ void WebCustomizationService::exclude_entities(AsyncWebServerRequest * request,
                 uint8_t unique_device_id = json["id"];
                 if (emsdevice->unique_id() == unique_device_id) {
                     // first reset all the entity ids
-                    emsdevice->reset_exclude_entities();
+                    emsdevice->reset_entity_masks();
 
                     // build a list of entities to exclude and then set the flag to non-visible
-                    JsonArray            entity_ids_json = json["entity_ids"];
+                    JsonArray                entity_ids_json = json["entity_ids"];
-                    std::vector<uint8_t> entity_ids;
+                    std::vector<std::string> entity_ids;
                     for (JsonVariant id : entity_ids_json) {
-                        uint8_t entity_id = id.as<int>();
+                        std::string entity_id = id.as<std::string>();
-                        emsdevice->exclude_entity(entity_id); // this will have immediate affect
+                        emsdevice->mask_entity(entity_id); // this will have immediate affect
                         entity_ids.push_back(entity_id);
                     }
 

src/web/WebCustomizationService.h

@@ -61,9 +61,9 @@ class AnalogCustomization {
 // we use product_id and device_id to make the device unique
 class EntityCustomization {
   public:
-    uint8_t              product_id; // device's product id
+    uint8_t                  product_id; // device's product id
-    uint8_t              device_id;  // device's device id
+    uint8_t                  device_id;  // device's device id
-    std::vector<uint8_t> entity_ids; // array of entity ids to exclude
+    std::vector<std::string> entity_ids; // array of entity ids to exclude
 };
 
 class WebCustomization {