From 24fec191b07e2b13a6d518f3d64e67fb9cac69af Mon Sep 17 00:00:00 2001 From: proddy Date: Thu, 13 Aug 2020 17:49:26 +0200 Subject: [PATCH 1/3] updated commands --- README.md | 94 +++++++++++++++++++++++++------------------------------ 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 46b0482c5..0bf23d351 100644 --- a/README.md +++ b/README.md @@ -2,39 +2,37 @@ ## **Breaking changes** -- MQTT base has been removed. The hostname is only used -- refresh command renamed to fetch -- have to 'wifi reconnect' after changing wifi in console +- MQTT base has been removed. The hostname is only used as prefixed to the topic, e.g. `ems-esp/status` +- `refresh` command in the console has been renamed to `fetch` ## **New Features in v2** -- A new web interface using React and TypeScript that's now secure as each URL endpoint is protected by issuing a JWT which is then sent using Bearer Authentication. Using a Captive Portal in AP mode or connecting to a local wifi network. +- Supports both ESP8266 and ESP32 +- New MQTT option to support Home Assistant MQTT Discovery (https://www.home-assistant.io/docs/mqtt/discovery/) +- A new web interface using React and TypeScript that's now secure as each URL endpoint is protected by issuing a JWT which is then sent using Bearer Authentication. +- The Access Point mode presents the user with a Captive Portal -- A new console. Like in version 1.9 it works with both Serial and Telnet but with a more intuitive Linux shell like behavior. It supports multiple connections and has basic security to prevent any changes to EMS-ESP. A full list of commands is below, here are the key ones: - * `help` lists the commands and keywords - * some commands take you into a new context, a bit like a sub-menu. e.g. `system`, `thermostat`. Use `help` to show which commands this context has and `exit` or CTRL-D to get back to the root. - * To change a setting use the `set` command. Typing `set` shows the current settings. - * `show` shows the data specific to the which context you're in. - * `su` to switch to Admin which enables more commands such as most of the `set` commands. The default password is "ems-esp-neo" which can be changed with `passwd` from the system menu. When in Admin mode the command prompt switches from `$` to `#`. - * `log` sets the logging level. `log off` disables logging. Use `log debug` for debugging commands and actions. - * `watch` will output the incoming Rx telegrams to the console. You can also show these in its 'raw' data format and also watch a particular ID. - * CTRL-D to exit, CTRL-U to remove line, TAB to auto-complete +- A new console. As in version 1.9 it works with both Serial and Telnet but now with a more intuitive Linux shell like behavior. It supports multiple connections and has basic security to prevent any changes to EMS-ESP. A full list of commands is below, here are the key ones: + * `help` lists the commands and keywords. This works in each context. + * `exit` will exit the console or exit the current context. CTRL-D does the same. + * `CTRL-U` for Undo + * `TAB` for auto-complete + * Some specific commands are behind contexts. Think of this as a sub-menu. e.g. `system`, `thermostat`. The path will always show you which context you are in. `$` is the root. + * `su` will switch to super-user or Admin. The default password is "ems-esp-neo" and can be changed with `passwd` from the system menu or via the Web UI (called secret password). When in Admin mode the command prompt switches from `$` to `#`. + * Some settings can be changed in the console. The `set` command will list them. + * `show` shows the data specific to the which context you're in. From the root it will show you all the EMS device information and any external temperature sensors. From a context it will be more specific to that context, e.g. `show mqtt` from `system` will list MQTT subscriptions and show the status and queue. + * `log` sets the logging level. `log off` disables logging. Use `log debug` for debugging commands and actions. This will be reset next time the console is opened. + * `watch` will output the incoming Rx telegrams directly to the console. You can also put on a watch on a specific EMS device ID or telegram ID. Also choose to output as verbose text or raw data bytes. these in its 'raw' data format and also watch a particular ID. -- There is no "serial mode" anymore like with version 1.9. When the Wifi cannot connect to the SSID it will automatically enter a "safe" mode where the Serial console is activated automatically (baud 115200). Note Serial is always available on the ESP32 because it has multiple UARTs. +- There is no "serial mode" anymore like with version 1.9. When the Wifi cannot connect to the SSID it will automatically enter a "safe" mode where the Serial console is automatically activated (note Serial is always available on the ESP32 because it has multiple UARTs). The EMS-ESP will blink fast when in Serial mode. Connect via a USB with baud 115200 to see the serial console. Note in this mode the EMS will be disconnect so there will be no incoming traffic. Use only for debugging or changing settings. - The onboard LED behaves like in 1.9. A solid LED means good connection and EMS data is coming in. A slow pulse means either the WiFi or the EMS bus is not connected yet. A very fast pulse is when the system is booting up and configuring itself, which typically takes 5 seconds. -- Built to work with both EMS8266 and ESP32 - -- Extended MQTT to use MQTT discovery on Home Assistant. - -- For debugging there is an offline mode where the code can be compiled and executed standalone without uploading to an ESP controller. Use `make` (based off GNUMakefile). - ## **Uploading the firmware** - If you're not using PlatformIO, use the command-line and Python. You can download Python from https://www.python.org/downloads/. Make sure you also get: @@ -58,6 +56,10 @@ ## **List of console commands** + (* = available in su/Admin mode) + + The `call` command is to execute a command. The command names (`[cmd]`) are the same as the MQTT command listed in the next section. + ``` common commands available in all contexts: exit @@ -72,10 +74,10 @@ common commands available in all contexts: system (enters a context) boiler (enters a context) thermostat (enters a context) - scan devices [deep] - send telegram <"XX XX ..."> - set bus_id - set tx_mode + scan devices [deep] * + send telegram <"XX XX ..."> * + set bus_id * + set tx_mode * show show devices show ems @@ -84,32 +86,33 @@ common commands available in all contexts: system set show + format * show mqtt - show users - passwd - restart - set wifi hostname - set wifi password - set wifi ssid - wifi reconnect - pin [data] + show users * + passwd * + restart * + set wifi hostname * + set wifi password * + set wifi ssid * + wifi reconnect * + pin [data] * boiler - read - call [cmd] [n] (cmd's: comfort wwactivated wwtapactivated wwonetime wwcirculation flowtemp wwtemp burnmaxpower burnminpower boilhyston boilhystoff burnperiod pumpdelay) + read * + call [cmd] [data] * thermostat set - set master [device ID] - read - call [cmd] [n] (cmd's: wwmode control mode holiday pause party datetime minexttemp clockoffset calinttemp display building language remotetemp temp nighttemp daytemp nofrosttemp ecotemp heattemp summertemp designtemp offsettemp holidaytemp) + set master [device ID] * + read * + call [cmd] [data] [heating circuit] * ``` ---------- -### **mqtt commands** +### **MQTT commands** -commands must be written as `{"cmd": ,"data":, "id": }`. The `id` can be replaced with `hc` for some devices. +commands must be written as `{"cmd": ,"data":, "id":}`. The `id` can be replaced with `hc` for some devices. `cmd` and `data` must be enclose in quotes as a string. ``` *boiler_cmd* @@ -161,13 +164,7 @@ commands must be written as `{"cmd": ,"data":, "id": }`. The `id` - The core services like telnet, logging and shell are based off the libraries from @nomis. I also adopted his general design pattens such as making everything as asynchronous as possible so that no one operation should starve another operation of it's time to execute (https://isocpp.org/wiki/faq/ctors#static-init-order). - All EMS devices (e.g. boiler, thermostat, solar modules, mixing units etc) are derived from a factory base class and each class handles its own registering of telegram and mqtt handlers. This makes the EMS device code easier to manage and we can extend with new telegrams types and features. - -### **Things to tidy up in code later** - -- Replace vectors of class objects with shared pointers and use emplace_back since it instantiates during construction. It may have a performance gain. -- Add real unit tests using platformio's [pio-remote](https://docs.platformio.org/en/latest/plus/pio-remote.html) test bed. -- See if it's easier to use timers instead of millis() based timers, using [polledTimeout](https://github.com/esp8266/Arduino/blob/master/libraries/esp8266/examples/BlinkPolledTimeout/BlinkPolledTimeout.ino). -- Port over to ESP-IDF. The Arduino SDK is showing its limitations +- For debugging there is an offline mode where the code can be compiled and executed standalone without uploading to an ESP controller. Use `make` (based off GNUMakefile). ### **Customizing the Web UI** @@ -229,10 +226,5 @@ The Web is based off Rick's awesome [esp8266-react](https://github.com/rjwats/es * `factory_settings.ini` modified with `ems-esp-neo` as password and `ems-esp` everywhere else - UI customization links: - - * icons: https://material-ui.com/components/material-icons/ - * colors: https://material-ui.com/customization/color/ - * tables: https://material-ui.com/components/tables/#dense-table From e71e17bed1480e13ee4fffe23c2ad2210ccc7d48 Mon Sep 17 00:00:00 2001 From: proddy Date: Thu, 13 Aug 2020 17:50:27 +0200 Subject: [PATCH 2/3] telegram functions can't be static - #445 --- src/devices/boiler.cpp | 2 ++ src/devices/thermostat.cpp | 2 ++ src/emsdevice.cpp | 3 --- src/emsdevice.h | 11 +++++------ src/emsesp.cpp | 12 +++++++----- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/devices/boiler.cpp b/src/devices/boiler.cpp index d5540c261..83cf7e096 100644 --- a/src/devices/boiler.cpp +++ b/src/devices/boiler.cpp @@ -26,6 +26,8 @@ uuid::log::Logger Boiler::logger_{F_(boiler), uuid::log::Facility::CONSOLE}; Boiler::Boiler(uint8_t device_type, int8_t device_id, uint8_t product_id, const std::string & version, const std::string & name, uint8_t flags, uint8_t brand) : EMSdevice(device_type, device_id, product_id, version, name, flags, brand) { + this->reserve_mem(20); // reserve some space for the telegram registries, to avoid memory fragmentation + LOG_DEBUG(F("Adding new Boiler with device ID 0x%02X"), device_id); // the telegram handlers... diff --git a/src/devices/thermostat.cpp b/src/devices/thermostat.cpp index edd7a2ffd..f2211bbac 100644 --- a/src/devices/thermostat.cpp +++ b/src/devices/thermostat.cpp @@ -26,6 +26,8 @@ uuid::log::Logger Thermostat::logger_{F_(thermostat), uuid::log::Facility::CONSO Thermostat::Thermostat(uint8_t device_type, uint8_t device_id, uint8_t product_id, const std::string & version, const std::string & name, uint8_t flags, uint8_t brand) : EMSdevice(device_type, device_id, product_id, version, name, flags, brand) { + this->reserve_mem(25); // reserve some space for the telegram registries, to avoid memory fragmentation + // common telegram handlers register_telegram_type(EMS_TYPE_RCOutdoorTemp, F("RCOutdoorTemp"), false, [&](std::shared_ptr t) { process_RCOutdoorTemp(t); }); register_telegram_type(EMS_TYPE_RCTime, F("RCTime"), false, [&](std::shared_ptr t) { process_RCTime(t); }); diff --git a/src/emsdevice.cpp b/src/emsdevice.cpp index d8e9f7c22..b223e33ba 100644 --- a/src/emsdevice.cpp +++ b/src/emsdevice.cpp @@ -24,8 +24,6 @@ namespace emsesp { uuid::log::Logger EMSdevice::logger_{F_(emsesp), uuid::log::Facility::CONSOLE}; -std::vector EMSdevice::telegram_functions_; - std::string EMSdevice::brand_to_string() const { switch (brand_) { case EMSdevice::Brand::BOSCH: @@ -248,7 +246,6 @@ void EMSdevice::register_mqtt_topic(const std::string & topic, mqtt_subfunction_ } void EMSdevice::register_mqtt_cmd(const __FlashStringHelper * cmd, mqtt_cmdfunction_p f) { - LOG_DEBUG(F("Registering MQTT cmd %s for device type %s"), uuid::read_flash_string(cmd).c_str(), this->device_type_name().c_str()); Mqtt::add_command(this->device_type_, this->device_id_, cmd, f); } diff --git a/src/emsdevice.h b/src/emsdevice.h index 4113b7c1d..6e5025d51 100644 --- a/src/emsdevice.h +++ b/src/emsdevice.h @@ -147,7 +147,7 @@ class EMSdevice { void fetch_values(); void toggle_fetch(uint16_t telegram_id, bool toggle); - static void reserve_mem(size_t n) { + void reserve_mem(size_t n) { telegram_functions_.reserve(n); } @@ -229,8 +229,7 @@ class EMSdevice { }; enum DeviceType : uint8_t { - UNKNOWN = 0, - SERVICEKEY, // us + SERVICEKEY = 0, // this is us BOILER, THERMOSTAT, MIXING, @@ -280,11 +279,11 @@ class EMSdevice { private: uint8_t unique_id_; - uint8_t device_type_ = DeviceType::UNKNOWN; + uint8_t device_type_ = DeviceType::SERVICEKEY; uint8_t device_id_ = 0; uint8_t product_id_ = 0; std::string version_; - std::string name_; // the long name of the EMS model + std::string name_; // the long name for the EMS model uint8_t flags_ = 0; uint8_t brand_ = Brand::NO_BRAND; @@ -304,7 +303,7 @@ class EMSdevice { , process_function_(process_function) { } }; - static std::vector telegram_functions_; // each EMS device has its own set of registered telegram types + std::vector telegram_functions_; // each EMS device has its own set of registered telegram types }; } // namespace emsesp diff --git a/src/emsesp.cpp b/src/emsesp.cpp index 4d8396970..a96a1dab7 100644 --- a/src/emsesp.cpp +++ b/src/emsesp.cpp @@ -573,7 +573,7 @@ void EMSESP::show_devices(uuid::console::Shell & shell) { } shell.println(); emsdevice->show_telegram_handlers(shell); - emsdevice->show_mqtt_handlers(shell); + // emsdevice->show_mqtt_handlers(shell); shell.println(); } } @@ -793,19 +793,21 @@ void EMSESP::start() { esp8266React.begin(); // loads system settings (wifi, mqtt, etc) emsespSettingsService.begin(); // load EMS-ESP specific settings // system_.check_upgrade(); // see if we need to migrate from previous versions - mqtt_.start(); // mqtt init console_.start(); // telnet and serial console + mqtt_.start(); // mqtt init system_.start(); // starts syslog, uart, sets version, initializes LED. Requires pre-loaded settings. shower_.start(); // initialize shower timer and shower alert txservice_.start(); // sets bus ID, sends out request for EMS devices sensors_.start(); // dallas external sensors webServer.begin(); // start web server - // reserve some space for the telegram registries, to avoid memory fragmentation - EMSdevice::reserve_mem(EMSdevice::EMS_DEVICES_MAX_TELEGRAMS); // space for 20 telegram handlers - emsdevices.reserve(5); // reserve space for initially 5 devices + emsdevices.reserve(5); // reserve space for initially 5 devices to avoid mem LOG_INFO("EMS Device library loaded with %d records", device_library_.size()); + +#if defined(EMSESP_STANDALONE) + mqtt_.on_connect(); // simulate an MQTT connection +#endif } // main loop calling all services From 0d18e841412d8768beeccb7e20799e17ab32d8dc Mon Sep 17 00:00:00 2001 From: proddy Date: Thu, 13 Aug 2020 17:50:48 +0200 Subject: [PATCH 3/3] minor changes --- src/mqtt.cpp | 29 ++++++++++++++--------------- src/mqtt.h | 4 ++-- src/system.h | 2 -- src/test/test.cpp | 3 --- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/mqtt.cpp b/src/mqtt.cpp index acef4ff51..11a703603 100644 --- a/src/mqtt.cpp +++ b/src/mqtt.cpp @@ -78,6 +78,8 @@ void Mqtt::add_command(const uint8_t device_type, const uint8_t device_id, const Mqtt::subscribe(device_type, cmd_topic, nullptr); // use an empty function handler to signal this is a command function } + LOG_DEBUG(F("Registering MQTT cmd %s with topic %s"), uuid::read_flash_string(cmd).c_str(), EMSdevice::device_type_topic_name(device_type).c_str()); + mqtt_cmdfunctions_.emplace_back(device_type, device_id, cmd, cb); } @@ -135,11 +137,16 @@ void Mqtt::show_mqtt(uuid::console::Shell & shell) { // show subscriptions shell.printfln(F("MQTT subscriptions:")); for (const auto & mqtt_subfunction : mqtt_subfunctions_) { - shell.printf(F(" topic: %s, [cmd]:"), mqtt_subfunction.full_topic_.c_str()); - // show the commands associated with this subscription - for (const auto & mqtt_cmdfunction : mqtt_cmdfunctions_) { - if (EMSdevice::device_type_topic_name(mqtt_cmdfunction.device_type_) == mqtt_subfunction.topic_) { - shell.printf(F(" %s"), uuid::read_flash_string(mqtt_cmdfunction.cmd_).c_str()); + // don't show commands if its homeassistant + if ((strncmp(mqtt_subfunction.full_topic_.c_str(), "homeassistant/", 13) == 0)) { + shell.printf(F(" topic: %s"), mqtt_subfunction.full_topic_.c_str()); + } else { + // show the commands associated with this subscription + shell.printf(F(" topic: %s, [cmd]:"), mqtt_subfunction.full_topic_.c_str()); + for (const auto & mqtt_cmdfunction : mqtt_cmdfunctions_) { + if (EMSdevice::device_type_topic_name(mqtt_cmdfunction.device_type_) == mqtt_subfunction.topic_) { + shell.printf(F(" %s"), uuid::read_flash_string(mqtt_cmdfunction.cmd_).c_str()); + } } } shell.println(); @@ -365,11 +372,6 @@ void Mqtt::start() { on_publish(packetId); }); -#if defined(EMSESP_STANDALONE) - // simulate an MQTT connection - on_connect(); -#endif - // create space for command buffer, to avoid heap memory fragmentation mqtt_cmdfunctions_.reserve(40); // current count with boiler+thermostat is 37 mqtt_subfunctions_.reserve(10); @@ -400,12 +402,9 @@ void Mqtt::on_connect() { resubscribe(); // in case this is a reconnect, re-subscribe again to all MQTT topics - // add the system MQTT subscriptions, only if its a fresh start with no previous subscriptions // these commands respond to the topic "system_cmd" and take a payload like {cmd:"", data:"", id:""} - if (mqtt_subfunctions_.empty()) { - add_command(EMSdevice::DeviceType::SERVICEKEY, bus_id_, F("pin"), System::mqtt_command_pin); - add_command(EMSdevice::DeviceType::SERVICEKEY, bus_id_, F("send"), System::mqtt_command_send); - } + add_command(EMSdevice::DeviceType::SERVICEKEY, bus_id_, F("pin"), System::mqtt_command_pin); + add_command(EMSdevice::DeviceType::SERVICEKEY, bus_id_, F("send"), System::mqtt_command_send); LOG_INFO(F("MQTT connected")); } diff --git a/src/mqtt.h b/src/mqtt.h index 6c700137c..8c9e36faf 100644 --- a/src/mqtt.h +++ b/src/mqtt.h @@ -171,8 +171,8 @@ class Mqtt { // function handlers for MQTT subscriptions struct MQTTSubFunction { - uint8_t device_type_; // which device type, from DeviceType:: - const std::string topic_; + uint8_t device_type_; // which device type, from DeviceType:: + const std::string topic_; // short topic name const std::string full_topic_; // the fully qualified topic name, usually with the hostname prefixed mqtt_subfunction_p mqtt_subfunction_; // can be empty diff --git a/src/system.h b/src/system.h index 65ed7fd98..9cb62f9ab 100644 --- a/src/system.h +++ b/src/system.h @@ -35,8 +35,6 @@ using uuid::console::Shell; -#define MQTT_SYSTEM_CMD "system_cmd" - namespace emsesp { class System { diff --git a/src/test/test.cpp b/src/test/test.cpp index e6a92e59d..6b951ba93 100644 --- a/src/test/test.cpp +++ b/src/test/test.cpp @@ -537,9 +537,6 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & command) { if (command == "mqtt") { shell.printfln(F("Testing MQTT...")); - // simulate an on connect - Mqtt::on_connect(); - // add a boiler // question: do we need to set the mask? std::string version("1.2.3");