added mqtt log

src/MyESP.cpp

@@ -22,7 +22,6 @@ union system_rtcmem_t {
     uint32_t value;
 };
 
-
 // nasty global variables that are called from internal ws functions
 static char * _general_password = nullptr;
 static bool   _shouldRestart    = false;
@@ -103,6 +102,13 @@ MyESP::MyESP() {
 
     // get the build time
     _buildTime = _getBuildTime();
+
+    // MQTT log
+    for (uint8_t i = 0; i < MYESP_MQTTLOG_MAX; i++) {
+        MQTT_log[i].timestamp = 0;
+        MQTT_log[i].topic     = nullptr;
+        MQTT_log[i].payload   = nullptr;
+    }
 }
 
 MyESP::~MyESP() {
@@ -190,12 +196,6 @@ uint32_t MyESP::_getInitialFreeHeap() {
     return _heap;
 }
 
-// used heap mem
-// note calls to getFreeHeap sometimes causes some ESPs to crash
-uint32_t MyESP::_getUsedHeap() {
-    return _getInitialFreeHeap() - ESP.getFreeHeap();
-}
-
 // called when WiFi is connected, and used to start OTA, MQTT
 void MyESP::_wifiCallback(justwifi_messages_t code, char * parameter) {
     if ((code == MESSAGE_CONNECTED)) {
@@ -370,6 +370,8 @@ void MyESP::mqttUnsubscribe(const char * topic) {
 void MyESP::mqttPublish(const char * topic, const char * payload) {
     // myDebug_P(PSTR("[MQTT] Sending pubish to %s with payload %s"), _mqttTopic(topic), payload); // for debugging
     mqttClient.publish(_mqttTopic(topic), _mqtt_qos, _mqtt_retain, payload);
+
+    _addMQTTLog(topic, payload); // add to the log
 }
 
 // MQTT onConnect - when a connect is established
@@ -380,7 +382,7 @@ void MyESP::_mqttOnConnect() {
     _mqtt_last_connection = millis();
 
     // say we're alive to the Last Will topic
-    mqttClient.publish(_mqttTopic(_mqtt_will_topic), 1, true, _mqtt_will_online_payload);
+    mqttClient.publish(_mqttTopic(_mqtt_will_topic), 1, true, _mqtt_will_online_payload); // qos=1, retain=true
 
     // subscribe to general subs
     mqttSubscribe(MQTT_TOPIC_RESTART);
@@ -389,6 +391,9 @@ void MyESP::_mqttOnConnect() {
     mqttSubscribe(MQTT_TOPIC_START);
     mqttPublish(MQTT_TOPIC_START, MQTT_TOPIC_START_PAYLOAD);
 
+    // send heartbeat if enabled
+    _heartbeatCheck(true);
+
     // call custom function to handle mqtt receives
     (_mqtt_callback_f)(MQTT_CONNECT_EVENT, nullptr, nullptr);
 }
@@ -604,7 +609,7 @@ void MyESP::_consoleShowHelp() {
     myDebug_P(PSTR("*"));
     myDebug_P(PSTR("* Commands:"));
     myDebug_P(PSTR("*  ?=help, CTRL-D/quit=exit telnet session"));
-    myDebug_P(PSTR("*  set, system, restart"));
+    myDebug_P(PSTR("*  set, system, restart, mqttlog"));
 #ifdef CRASH
     myDebug_P(PSTR("*  crash <dump | clear | test [n]>"));
 #endif
@@ -944,6 +949,13 @@ void MyESP::_telnetCommand(char * commandLine) {
     // restart command
     if ((strcmp(ptrToCommandName, "restart") == 0) && (wc == 1)) {
         resetESP();
+        return;
+    }
+
+    // print mqtt log command
+    if ((strcmp(ptrToCommandName, "mqttlog") == 0) && (wc == 1)) {
+        _printMQTTLog();
+        return;
     }
 
     // show system stats
@@ -1108,7 +1120,7 @@ bool MyESP::_rtcmemStatus() {
     if (reason == REASON_EXT_SYS_RST) { // external system reset
         if (getSystemBootStatus() == MYESP_BOOTSTATUS_BOOTING) {
             _setSystemBootStatus(MYESP_BOOTSTATUS_RESETNEEDED);
-            // _formatreq = true; // do a wipe next in the loop() TODO commented out for now
+            // _formatreq = true; // do a wipe next in the loop() - commented out for now because we use the web
         } else {
             _setSystemBootStatus(MYESP_BOOTSTATUS_POWERON);
         }
@@ -1199,7 +1211,6 @@ void MyESP::_systemCheckLoop() {
     }
 }
 
-
 // print out ESP system stats
 // for battery power is ESP.getVcc()
 void MyESP::showSystemStats() {
@@ -1340,6 +1351,8 @@ void MyESP::_heartbeatCheck(bool force = false) {
     if ((millis() - last_heartbeat > MYESP_HEARTBEAT_INTERVAL) || force) {
         last_heartbeat = millis();
 
+        // _printHeap("Heartbeat"); // for heartbeat debugging
+
         if (!isMQTTConnected() || !(_mqtt_heartbeat)) {
             return;
         }
@@ -1819,7 +1832,7 @@ bool MyESP::_fs_createCustomConfig() {
 // if it doesn't exist try and create it
 void MyESP::_fs_setup() {
     if (!SPIFFS.begin()) {
-        Serial.print(F("[WARN] Formatting filesystem..."));
+        myDebug_P(PSTR("[FS] Formatting filesystem..."));
         if (SPIFFS.format()) {
             _writeEvent("WARN", "sys", "File system formatted", "");
         } else {
@@ -2108,7 +2121,7 @@ void MyESP::_writeEvent(const char * type, const char * src, const char * desc,
         return;
     }
 
-    StaticJsonDocument<300> root;
+    StaticJsonDocument<MYESP_JSON_LOG_MAXSIZE> root;
     root["type"] = type;
     root["src"]  = src;
     root["desc"] = desc;
@@ -2132,9 +2145,16 @@ void MyESP::_sendEventLog(uint8_t page) {
     if (!eventlog) {
         eventlog.close();
         myDebug_P(PSTR("[WEB] Event log is missing"));
+        if (_ota_post_callback_f) {
+            (_ota_post_callback_f)(); // call custom function
+        }
         return; // file can't be opened
     }
 
+    if (_ota_pre_callback_f) {
+        (_ota_pre_callback_f)(); // call custom function
+    }
+
     // the size of the json will be quite big so best not to use stack (StaticJsonDocument)
     DynamicJsonDocument doc(MYESP_JSON_MAXSIZE);
     JsonObject          root = doc.to<JsonObject>();
@@ -2143,31 +2163,67 @@ void MyESP::_sendEventLog(uint8_t page) {
 
     JsonArray list = doc.createNestedArray("list");
 
-    uint8_t first = (page - 1) * 10;
+    uint8_t  first      = ((page - 1) * 10) + 1;
-    uint8_t last  = page * 10;
+    uint8_t  last       = page * 10;
-    uint8_t i     = 0;
+    uint8_t  char_count = 0;
+    uint8_t  line_count = 0;
+    uint16_t read_count = 0;
+    bool     abort      = false;
+    char     char_buffer[MYESP_JSON_LOG_MAXSIZE];
 
-    while (eventlog.available()) {
+    // if at start, start immediately recording
-        String item = String();
+    bool record = (first == 1) ? true : false;
-        item        = eventlog.readStringUntil('\n');
+
-        if (i >= first && i < last) {
+    // start at top and read until we find the page we want (sets of 10)
-            list.add(item);
+    while (eventlog.available() && !abort) {
+        char c = eventlog.read();
+
+        // see if we've overrun, which means corrupt so ignore rest
+        if (read_count++ > MYESP_JSON_LOG_MAXSIZE - 1) {
+            abort = true;
         }
-        i++;
-    }
-    eventlog.close();
 
-    float pages      = i / 10.0;
+        // see if we have reached the end of the string
+        if (c == '\0' || c == '\n') {
+            line_count++;
+
+            // save line
+            if (record) {
+                char_buffer[char_count] = '\0';
+                list.add(char_buffer);
+            }
+
+            char_count = 0;
+            read_count = 0;
+            if (line_count == first - 1) { // have we come to the start position, start recording
+                record = true;
+            } else if (line_count == last) { // finish recording and exit loop
+                record = false;
+            }
+        } else {
+            // add the char to the buffer if recording
+            if (record && (char_count < MYESP_JSON_LOG_MAXSIZE)) {
+                char_buffer[char_count++] = c;
+            }
+        }
+    }
+    eventlog.close(); // close SPIFFS
+
+    float pages      = line_count / 10.0;
     root["haspages"] = ceil(pages);
 
     char   buffer[MYESP_JSON_MAXSIZE];
     size_t len = serializeJson(root, buffer);
 
-    //Serial.printf("\nEVENTLOG: page %d\n", page); // turn on for debugging
+    //Serial.printf("\nEVENTLOG: page %d\n", page); // turn on for debugging XXX
-    //serializeJson(root, Serial);                   // turn on for debugging
+    //serializeJson(root, Serial);                  // turn on for debugging
 
     _ws->textAll(buffer, len);
     _ws->textAll("{\"command\":\"result\",\"resultof\":\"eventlist\",\"result\": true}");
+
+    if (_ota_post_callback_f) {
+        (_ota_post_callback_f)(); // call custom function
+    }
 }
 
 // Handles WebSocket Events
@@ -2334,22 +2390,27 @@ void MyESP::_sendCustomStatus() {
 
 // send system status via ws
 void MyESP::_sendStatus() {
-    StaticJsonDocument<400> doc;
+    // capture memory before we stick in a huge json buffer on the heap!
-    JsonObject              root = doc.to<JsonObject>();
+    uint32_t total_memory = _getInitialFreeHeap();
+    uint32_t free_memory  = ESP.getFreeHeap();
+
+    DynamicJsonDocument doc(MQTT_MAX_PAYLOAD_SIZE_LARGE);
+    JsonObject          root = doc.to<JsonObject>();
+    root["command"]          = "status";
 
     FSInfo fsinfo;
     if (!SPIFFS.info(fsinfo)) {
         myDebug("[SYSTEM] Error getting info on SPIFFS");
+    } else {
+        root["availspiffs"] = (fsinfo.totalBytes - fsinfo.usedBytes) / 1000;
+        root["spiffssize"]  = (fsinfo.totalBytes / 1000);
     }
 
-    root["command"] = "status";
-
     // all sizes in bytes converted to KB
-    root["heap"]        = ESP.getFreeHeap() / 1000;
+    root["initheap"]   = total_memory;
-    root["sketchsize"]  = ESP.getSketchSize() / 1000;
+    root["heap"]       = free_memory;
-    root["availsize"]   = ESP.getFreeSketchSpace() / 1000;
+    root["sketchsize"] = ESP.getSketchSize() / 1000;
-    root["availspiffs"] = (fsinfo.totalBytes - fsinfo.usedBytes) / 1000;
+    root["availsize"]  = ESP.getFreeSketchSpace() / 1000;
-    root["spiffssize"]  = (fsinfo.totalBytes / 1000);
 
     if (isAPmode()) {
         root["ip"]   = WiFi.softAPIP().toString();
@@ -2377,8 +2438,32 @@ void MyESP::_sendStatus() {
     sprintf(uptime, "%d day%s %d hour%s %d minute%s %d second%s", d, (d == 1) ? "" : "s", h, (h == 1) ? "" : "s", m, (m == 1) ? "" : "s", sec, (sec == 1) ? "" : "s");
     root["uptime"] = uptime;
 
-    char   buffer[400];
+    char topic_s[MQTT_MAX_TOPIC_SIZE] = {0};
+    if (_hasValue(_mqtt_base)) {
+        strlcpy(topic_s, _mqtt_base, sizeof(topic_s));
+        strlcat(topic_s, "/", sizeof(topic_s));
+        strlcat(topic_s, _general_hostname, sizeof(topic_s));
+    } else {
+        strlcpy(topic_s, _general_hostname, sizeof(topic_s));
+    }
+    strlcat(topic_s, "/", sizeof(topic_s));
+    root["mqttloghdr"] = topic_s;
+
+    // create MQTT log
+    JsonArray list = root.createNestedArray("mqttlog");
+
+    for (uint8_t i = 0; i < MYESP_MQTTLOG_MAX; i++) {
+        if (MQTT_log[i].topic != nullptr) {
+            JsonObject item = list.createNestedObject();
+            item["topic"]   = MQTT_log[i].topic;
+            item["payload"] = MQTT_log[i].payload;
+            item["time"]    = MQTT_log[i].timestamp;
+        }
+    }
+
+    char   buffer[MQTT_MAX_PAYLOAD_SIZE_LARGE];
     size_t len = serializeJson(root, buffer);
+
     _ws->textAll(buffer, len);
 }
 
@@ -2525,6 +2610,72 @@ void MyESP::_webserver_setup() {
     myDebug_P(PSTR("[WEB] Web server started"));
 }
 
+// print memory
+void MyESP::_printHeap(const char * s) {
+    uint32_t total_memory = _getInitialFreeHeap();
+    uint32_t free_memory  = ESP.getFreeHeap();
+
+    myDebug(" [%s] Free Heap: %d bytes initially | %d bytes used (%2u%%) | %d bytes free (%2u%%)",
+            s,
+            total_memory,
+            total_memory - free_memory,
+            100 * (total_memory - free_memory) / total_memory,
+            free_memory,
+            100 * free_memory / total_memory);
+}
+
+// print MQTT log - everything that was published last per topic
+void MyESP::_printMQTTLog() {
+    myDebug_P(PSTR("MQTT publish log:"));
+
+    for (uint8_t i = 0; i < MYESP_MQTTLOG_MAX; i++) {
+        if (MQTT_log[i].topic != nullptr) {
+            myDebug_P(PSTR("(%d) [%lu] Topic:%s Payload:%s"), i, MQTT_log[i].timestamp, MQTT_log[i].topic, MQTT_log[i].payload);
+        }
+    }
+
+    myDebug_P(PSTR("")); // newline
+}
+
+// add an MQTT log entry
+void MyESP::_addMQTTLog(const char * topic, const char * payload) {
+    static uint8_t logCount   = 0;
+    uint8_t        logPointer = 0;
+    bool           found      = false;
+
+    // myDebug("Publish [#%d] %s (%d) %s (%d)", logCount, topic, strlen(topic), payload, strlen(payload)); // for debugging
+
+    // find the topic
+    while ((_hasValue(MQTT_log[logPointer].topic) && logPointer < MYESP_MQTTLOG_MAX)) {
+        if (strcmp(MQTT_log[logPointer].topic, topic) == 0) {
+            found = true;
+            break;
+        }
+        logPointer++;
+    }
+
+    // if not found add it and increment next free space pointer
+    if (!found) {
+        logPointer = logCount;
+        if (++logCount == MYESP_MQTTLOG_MAX) {
+            logCount = 0; // rotate
+        }
+    }
+
+    // delete old record
+    if (MQTT_log[logPointer].topic) {
+        free(MQTT_log[logPointer].topic);
+    }
+
+    if (MQTT_log[logPointer].payload) {
+        free(MQTT_log[logPointer].payload);
+    }
+
+    // add new record
+    MQTT_log[logPointer].topic     = strdup(topic);
+    MQTT_log[logPointer].payload   = strdup(payload);
+    MQTT_log[logPointer].timestamp = now();
+}
 
 // send UTC time via ws
 void MyESP::_sendTime() {
@@ -2594,6 +2745,10 @@ void MyESP::begin(const char * app_hostname, const char * app_name, const char *
 
     _telnet_setup(); // Telnet setup, called first to set Serial
 
+    // _fs_printFile(MYESP_CONFIG_FILE); // for debugging
+    //_fs_printFile(MYESP_CUSTOMCONFIG_FILE); // for debugging
+    //_fs_printFile(MYESP_EVENTLOG_FILE);     // for debugging
+
     // print a welcome message
     myDebug_P(PSTR("\n\n* %s version %s"), _app_name, _app_version);
 
@@ -2613,10 +2768,6 @@ void MyESP::begin(const char * app_hostname, const char * app_name, const char *
     _heartbeatCheck(true);  // force heartbeat
 
     SerialAndTelnet.flush();
-
-    //_fs_printFile(MYESP_CONFIG_FILE);       // for debugging
-    //_fs_printFile(MYESP_CUSTOMCONFIG_FILE); // for debugging
-    //_fs_printFile(MYESP_EVENTLOG_FILE); // for debugging
 }
 
 /*
@@ -2643,7 +2794,7 @@ void MyESP::loop() {
     }
 
     if (_formatreq) {
-        myDebug("[SYSTEM] Factory reset initiated");
+        myDebug("[SYSTEM] Factory reset initiated. Please wait. System will automatically restart when complete...");
         SPIFFS.end();
         _ws->enable(false);
         SPIFFS.format();

src/MyESP.h

@@ -72,7 +72,6 @@ extern struct rst_info resetInfo;
 #define MQTT_RECONNECT_DELAY_MIN 2000   // Try to reconnect in 3 seconds upon disconnection
 #define MQTT_RECONNECT_DELAY_STEP 3000  // Increase the reconnect delay in 3 seconds after each failed attempt
 #define MQTT_RECONNECT_DELAY_MAX 120000 // Set reconnect time to 2 minutes at most
-#define MQTT_MAX_TOPIC_SIZE 50          // max length of MQTT topic
 #define MQTT_TOPIC_START "start"
 #define MQTT_TOPIC_HEARTBEAT "heartbeat"
 #define MQTT_TOPIC_START_PAYLOAD "start"
@@ -82,7 +81,13 @@ extern struct rst_info resetInfo;
 #define MQTT_RETAIN false
 #define MQTT_KEEPALIVE 60 // 1 minute
 #define MQTT_QOS 1
-#define MQTT_WILL_TOPIC "status" // for last will & testament topic name
+#define MQTT_WILL_TOPIC "status"         // for last will & testament topic name
+#define MQTT_MAX_TOPIC_SIZE 50           // max length of MQTT topic
+#define MQTT_MAX_PAYLOAD_SIZE 500        // max size of a JSON object. See https://arduinojson.org/v6/assistant/
+#define MQTT_MAX_PAYLOAD_SIZE_LARGE 2000 // max size of a large JSON object, like for sending MQTT log
+#define MYESP_JSON_MAXSIZE 2000          // for large Dynamic json files
+#define MYESP_MQTTLOG_MAX 20             // max number of log entries for MQTT publishes
+#define MYESP_JSON_LOG_MAXSIZE 300       // max size of an JSON log entry
 
 // Internal MQTT events
 #define MQTT_CONNECT_EVENT 0
@@ -134,7 +139,6 @@ PROGMEM const char * const custom_reset_string[]   = {custom_reset_hardware, cus
 
 // SPIFFS
 #define MYESP_SPIFFS_MAXSIZE 800 // https://arduinojson.org/v6/assistant/
-#define MYESP_JSON_MAXSIZE 2000  // for large Dynamic json files
 
 // CRASH
 /**
@@ -186,8 +190,8 @@ struct RtcmemData {
 
 static_assert(sizeof(RtcmemData) <= (RTCMEM_BLOCKS * 4u), "RTCMEM struct is too big");
 
-#define MYESP_SYSTEM_CHECK_TIME 60000   // The system is considered stable after these many millis (1 minute)
+#define MYESP_SYSTEM_CHECK_TIME 60000 // The system is considered stable after these many millis (1 minute)
-#define MYESP_SYSTEM_CHECK_MAX 10       // After this many crashes on boot
+#define MYESP_SYSTEM_CHECK_MAX 10     // After this many crashes on boot
 #define MYESP_HEARTBEAT_INTERVAL 120000 // in milliseconds, how often the MQTT heartbeat is sent (2 mins)
 
 typedef struct {
@@ -205,6 +209,13 @@ typedef enum {
     MYESP_BOOTSTATUS_RESETNEEDED = 3
 } MYESP_BOOTSTATUS; // boot messages
 
+// for storing all MQTT publish messages
+typedef struct {
+    char * topic;
+    char * payload;
+    time_t timestamp;
+} _MQTT_Log;
+
 typedef std::function<void(unsigned int, const char *, const char *)>            mqtt_callback_f;
 typedef std::function<void()>                                                    wifi_callback_f;
 typedef std::function<void()>                                                    ota_callback_f;
@@ -297,28 +308,34 @@ class MyESP {
 
   private:
     // mqtt
-    AsyncMqttClient mqttClient;
+    void   _mqttOnMessage(char * topic, char * payload, size_t len);
-    unsigned long   _mqtt_reconnect_delay;
+    void   _mqttConnect();
-    void            _mqttOnMessage(char * topic, char * payload, size_t len);
+    void   _mqtt_setup();
-    void            _mqttConnect();
+    void   _mqttOnConnect();
-    void            _mqtt_setup();
+    void   _sendStart();
+    char * _mqttTopic(const char * topic);
+
+    // mqtt log
+    _MQTT_Log MQTT_log[MYESP_MQTTLOG_MAX]; // log for publish messages
+    void      _printMQTTLog();
+    void      _addMQTTLog(const char * topic, const char * payload);
+
+    AsyncMqttClient mqttClient; // the MQTT class
+    uint32_t        _mqtt_reconnect_delay;
     mqtt_callback_f _mqtt_callback_f;
-    void            _mqttOnConnect();
-    void            _sendStart();
-    char *          _mqttTopic(const char * topic);
     char *          _mqtt_ip;
     char *          _mqtt_user;
     char *          _mqtt_password;
     int             _mqtt_port;
     char *          _mqtt_base;
     bool            _mqtt_enabled;
-    unsigned long   _mqtt_keepalive;
+    uint32_t        _mqtt_keepalive;
-    unsigned char   _mqtt_qos;
+    uint8_t         _mqtt_qos;
     bool            _mqtt_retain;
     char *          _mqtt_will_topic;
     char *          _mqtt_will_online_payload;
     char *          _mqtt_will_offline_payload;
-    unsigned long   _mqtt_last_connection;
+    uint32_t        _mqtt_last_connection;
     bool            _mqtt_connecting;
     bool            _mqtt_heartbeat;
 
@@ -383,6 +400,7 @@ class MyESP {
     bool          _timerequest;
     bool          _formatreq;
     bool          _hasValue(char * s);
+    void          _printHeap(const char * s);
 
     // reset reason and rtcmem
     bool _rtcmem_status;

src/custom.htm

@@ -9,7 +9,7 @@
         <label class="col-xs-3">LED<i style="margin-left: 10px;"
                 class="glyphicon glyphicon-exclamation-sign text-danger" aria-hidden="true" data-toggle="popover"
                 data-trigger="hover" data-placement="right"
-                data-content="Please choose if you want to enable the LED"></i></label>
+                data-content="Please choose if you want to enable an LED to show status"></i></label>
         <div class="col-xs-9">
             <form>
                 <label class="radio-inline">
@@ -148,19 +148,15 @@
 
 <div id="custom_statuscontent">
     <br>
-    <div class="row text-center">
+    <div class="row text-left">
         <div class="col-md-8 col-md-offset-2">
-            <h1>Dashboard</h1>
+            <h2>EMS Dashboard</h2>
-            <p>Real-time values from the EMS-ESP device are shown here.
+            <p>Real-time values from the EMS-ESP device are shown here.</p>
-                <div class="row form-group">
-                    <button onclick="refreshEMS()" class="btn btn-primary">Refresh Data</button>
-                </div>
-            </p>
         </div>
         <div class="col-md-8 col-md-offset-2">
             <div class="panel panel-default table-responsive">
                 <table class="table table-hover table-striped table-condensed">
-                    <caption>EMS Bus stats</caption>
+                    <caption>EMS Bus Status</caption>
                     <tr>
                         <td colspan="2">
                             <b>
@@ -169,7 +165,7 @@
                         </td>
                     </tr>
                     <tr>
-                        <th>Detected Devices:</th>
+                        <th id="devicesshow">Discovered Devices:</th>
                         <td>
                             <ul class="list-group">
                                 <div id="devices"></div>
@@ -228,4 +224,7 @@
 
         </div>
     </div>
+    <div class="row form-group" style="text-align: center;">
+        <button onclick="refreshEMS()" class="btn btn-primary">Refresh Data</button>
+    </div>
 </div>

src/custom.js

@@ -82,6 +82,7 @@ function listCustomStats() {
         document.getElementById("msg").className = "alert alert-success";
     } else {
         document.getElementById("msg").className = "alert alert-danger";
+        document.getElementById("devicesshow").style.display = "none";
         document.getElementById("thermostat_show").style.display = "none";
         document.getElementById("boiler_show").style.display = "none";
         return;
@@ -89,6 +90,9 @@ function listCustomStats() {
 
     var list = document.getElementById("devices");
     var obj = ajaxobj.emsbus.devices;
+
+    document.getElementById("devicesshow").style.display = "block";
+
     for (var i = 0; i < obj.length; i++) {
         var l = document.createElement("li");
         var type = obj[i].type;

src/ems-esp.cpp

@@ -67,8 +67,6 @@ Ticker showerColdShotStopTimer;
 #define SHOWER_COLDSHOT_DURATION 10 // in seconds. 10 seconds for cold water before turning back hot water
 #define SHOWER_MAX_DURATION 420000  // in ms. 7 minutes, before trigger a shot of cold water
 
-#define MQTT_MAX_SIZE 700 // max size of a JSON object. See https://arduinojson.org/v6/assistant/
-
 typedef struct {
     uint32_t timestamp;      // for internal timings, via millis()
     uint8_t  dallas_sensors; // count of dallas sensors
@@ -508,7 +506,7 @@ void showInfo() {
     _renderIntValue("Burner current power", "%", EMS_Boiler.curBurnPow);
     _renderShortValue("Flame current", "uA", EMS_Boiler.flameCurr);
     _renderIntValue("System pressure", "bar", EMS_Boiler.sysPress, 10);
-    if (EMS_Boiler.serviceCode == EMS_VALUE_SHORT_NOTSET) {
+    if (EMS_Boiler.serviceCode == EMS_VALUE_USHORT_NOTSET) {
         myDebug_P(PSTR("  System service code: %s"), EMS_Boiler.serviceCodeChar);
     } else {
         myDebug_P(PSTR("  System service code: %s (%d)"), EMS_Boiler.serviceCodeChar, EMS_Boiler.serviceCode);
@@ -688,16 +686,17 @@ void publishSensorValues() {
 // CRC check is done to see if there are changes in the values since the last send to avoid too much wifi traffic
 // a check is done against the previous values and if there are changes only then they are published. Unless force=true
 void publishValues(bool force) {
-    // don't send if MQTT is connected
+    // don't send if MQTT is not connected
     if (!myESP.isMQTTConnected()) {
         return;
     }
 
-    char                              s[20] = {0}; // for formatting strings
+    char                                      s[20] = {0}; // for formatting strings
-    StaticJsonDocument<MQTT_MAX_SIZE> doc;
+    StaticJsonDocument<MQTT_MAX_PAYLOAD_SIZE> doc;
-    char                              data[MQTT_MAX_SIZE] = {0};
+    char                                      data[MQTT_MAX_PAYLOAD_SIZE] = {0};
-    CRC32                             crc;
+    CRC32                                     crc;
-    uint32_t                          fchecksum;
+    uint32_t                                  fchecksum;
+    uint8_t                                   jsonSize;
 
     static uint8_t  last_boilerActive            = 0xFF; // for remembering last setting of the tap water or heating on/off
     static uint32_t previousBoilerPublishCRC     = 0;    // CRC check for boiler values
@@ -769,22 +768,28 @@ void publishValues(bool force) {
     if (abs(EMS_Boiler.heatWorkMin) != EMS_VALUE_LONG_NOTSET)
         rootBoiler["heatWorkMin"] = (double)EMS_Boiler.heatWorkMin;
 
-    rootBoiler["ServiceCode"]       = EMS_Boiler.serviceCodeChar;
+    if (EMS_Boiler.serviceCode != EMS_VALUE_USHORT_NOTSET) {
-    rootBoiler["ServiceCodeNumber"] = EMS_Boiler.serviceCode;
+        rootBoiler["ServiceCode"]       = EMS_Boiler.serviceCodeChar;
+        rootBoiler["ServiceCodeNumber"] = EMS_Boiler.serviceCode;
+    }
 
     serializeJson(doc, data, sizeof(data));
 
-    // calculate hash and send values if something has changed, to save unnecessary wifi traffic
+    // check for empty json
-    for (size_t i = 0; i < measureJson(doc) - 1; i++) {
+    jsonSize = measureJson(doc);
-        crc.update(data[i]);
+    if (jsonSize > 2) {
-    }
+        // calculate hash and send values if something has changed, to save unnecessary wifi traffic
-    fchecksum = crc.finalize();
+        for (uint8_t i = 0; i < (jsonSize - 1); i++) {
-    if ((previousBoilerPublishCRC != fchecksum) || force) {
+            crc.update(data[i]);
-        previousBoilerPublishCRC = fchecksum;
+        }
-        myDebugLog("Publishing boiler data via MQTT");
+        fchecksum = crc.finalize();
+        if ((previousBoilerPublishCRC != fchecksum) || force) {
+            previousBoilerPublishCRC = fchecksum;
+            myDebugLog("Publishing boiler data via MQTT");
 
-        // send values via MQTT
+            // send values via MQTT
-        myESP.mqttPublish(TOPIC_BOILER_DATA, data);
+            myESP.mqttPublish(TOPIC_BOILER_DATA, data);
+        }
     }
 
     // see if the heating or hot tap water has changed, if so send
@@ -856,18 +861,22 @@ void publishValues(bool force) {
         data[0] = '\0'; // reset data for next package
         serializeJson(doc, data, sizeof(data));
 
-        // calculate new CRC
+        // check for empty json
-        crc.reset();
+        jsonSize = measureJson(doc);
-        for (size_t i = 0; i < measureJson(doc) - 1; i++) {
+        if (jsonSize > 2) {
-            crc.update(data[i]);
+            // calculate new CRC
-        }
+            crc.reset();
-        fchecksum = crc.finalize();
+            for (uint8_t i = 0; i < (jsonSize - 1); i++) {
-        if ((previousThermostatPublishCRC != fchecksum) || force) {
+                crc.update(data[i]);
-            previousThermostatPublishCRC = fchecksum;
+            }
-            myDebugLog("Publishing thermostat data via MQTT");
+            fchecksum = crc.finalize();
+            if ((previousThermostatPublishCRC != fchecksum) || force) {
+                previousThermostatPublishCRC = fchecksum;
+                myDebugLog("Publishing thermostat data via MQTT");
 
-            // send values via MQTT
+                // send values via MQTT
-            myESP.mqttPublish(TOPIC_THERMOSTAT_DATA, data);
+                myESP.mqttPublish(TOPIC_THERMOSTAT_DATA, data);
+            }
         }
     }
 
@@ -906,18 +915,22 @@ void publishValues(bool force) {
         data[0] = '\0'; // reset data for next package
         serializeJson(doc, data, sizeof(data));
 
-        // calculate new CRC
+        // check for empty json
-        crc.reset();
+        jsonSize = measureJson(doc);
-        for (size_t i = 0; i < measureJson(doc) - 1; i++) {
+        if (jsonSize > 2) {
-            crc.update(data[i]);
+            // calculate new CRC
-        }
+            crc.reset();
-        fchecksum = crc.finalize();
+            for (uint8_t i = 0; i < (jsonSize - 1); i++) {
-        if ((previousSMPublishCRC != fchecksum) || force) {
+                crc.update(data[i]);
-            previousSMPublishCRC = fchecksum;
+            }
-            myDebugLog("Publishing SM data via MQTT");
+            fchecksum = crc.finalize();
+            if ((previousSMPublishCRC != fchecksum) || force) {
+                previousSMPublishCRC = fchecksum;
+                myDebugLog("Publishing SM data via MQTT");
 
-            // send values via MQTT
+                // send values via MQTT
-            myESP.mqttPublish(TOPIC_SM_DATA, data);
+                myESP.mqttPublish(TOPIC_SM_DATA, data);
+            }
         }
     }
 
@@ -1675,7 +1688,7 @@ void WebCallback(JsonObject root) {
 
     if (myESP.getUseSerial()) {
         emsbus["ok"]  = false;
-        emsbus["msg"] = "EMS Bus is disabled when in Serial mode. Check Settings->General Settings";
+        emsbus["msg"] = "EMS Bus is disabled when in Serial mode. Check Settings->General Settings->Serial Port";
     } else {
         if (ems_getBusConnected()) {
             if (ems_getTxDisabled()) {

src/ems.cpp

@@ -273,10 +273,10 @@ void ems_init() {
     EMS_Boiler.wWCirc      = EMS_VALUE_INT_NOTSET;    // Circulation on/off
     EMS_Boiler.selBurnPow  = EMS_VALUE_INT_NOTSET;    // Burner max power
     EMS_Boiler.curBurnPow  = EMS_VALUE_INT_NOTSET;    // Burner current power
-    EMS_Boiler.flameCurr   = EMS_VALUE_SHORT_NOTSET;  // Flame current in micro amps
+    EMS_Boiler.flameCurr   = EMS_VALUE_USHORT_NOTSET; // Flame current in micro amps
     EMS_Boiler.sysPress    = EMS_VALUE_INT_NOTSET;    // System pressure
     strlcpy(EMS_Boiler.serviceCodeChar, "??", sizeof(EMS_Boiler.serviceCodeChar));
-    EMS_Boiler.serviceCode = EMS_VALUE_SHORT_NOTSET;
+    EMS_Boiler.serviceCode = EMS_VALUE_USHORT_NOTSET;
 
     // UBAMonitorSlow
     EMS_Boiler.extTemp     = EMS_VALUE_SHORT_NOTSET;  // Outside temperature
@@ -335,7 +335,6 @@ void ems_init() {
     EMS_Thermostat.product_id = EMS_ID_NONE;
     strlcpy(EMS_Thermostat.version, "?", sizeof(EMS_Thermostat.version));
 
-
     // default logging is none
     ems_setLogging(EMS_SYS_LOGGING_DEFAULT);
 }
@@ -605,7 +604,9 @@ void _ems_sendTelegram() {
         _EMS_TX_STATUS _txStatus                       = emsuart_tx_buffer(EMS_TxTelegram.data, EMS_TxTelegram.length); // send the telegram to the UART Tx
         if (EMS_TX_BRK_DETECT == _txStatus || EMS_TX_WTD_TIMEOUT == _txStatus) {
             // Tx Error!
-            myDebug_P(PSTR("** error sending buffer: %s"), _txStatus == EMS_TX_BRK_DETECT ? "BRK" : "WDTO");
+            if (EMS_Sys_Status.emsLogging == EMS_SYS_LOGGING_VERBOSE) {
+                myDebug_P(PSTR("** error sending buffer: %s"), _txStatus == EMS_TX_BRK_DETECT ? "BRK" : "WDTO");
+            }
             // EMS_Sys_Status.emsTxStatus = EMS_TX_STATUS_IDLE;
         }
         EMS_TxQueue.shift(); // and remove from queue
@@ -668,8 +669,9 @@ void _ems_sendTelegram() {
         EMS_Sys_Status.emsTxStatus = EMS_TX_STATUS_WAIT;
     else {
         // Tx Error!
-        // Tx Error!
+        if (EMS_Sys_Status.emsLogging == EMS_SYS_LOGGING_VERBOSE) {
-        myDebug_P(PSTR("** error sending buffer: %s"), _txStatus == EMS_TX_BRK_DETECT ? "BRK" : "WDTO");
+            myDebug_P(PSTR("** error sending buffer: %s"), _txStatus == EMS_TX_BRK_DETECT ? "BRK" : "WDTO");
+        }
         EMS_Sys_Status.emsTxStatus = EMS_TX_STATUS_IDLE;
     }
 }
@@ -2509,10 +2511,6 @@ void ems_setThermostatTemp(float temperature, uint8_t temptype) {
         }
     }
 
-    // TODO XXX hack, please remove
-    EMS_TxTelegram.type   = EMS_TYPE_RCPLUSSet; // for 3000 and 1010, e.g. 0B 10 FF (0A | 08) 01 89 2B
-    EMS_TxTelegram.offset = 0x08;               // auto offset
-
     EMS_TxTelegram.length           = EMS_MIN_TELEGRAM_LENGTH;
     EMS_TxTelegram.dataValue        = (uint8_t)((float)temperature * (float)2); // value * 2
     EMS_TxTelegram.type_validate    = EMS_TxTelegram.type;

src/version.h

@@ -1,6 +1,6 @@
 
 #define APP_NAME "EMS-ESP"
-#define APP_VERSION "1.9.0b2_web"
+#define APP_VERSION "1.9.0b3_web"
 #define APP_HOSTNAME "ems-esp"
 #define APP_URL "https://github.com/proddy/EMS-ESP"
 #define APP_UPDATEURL "https://api.github.com/repos/proddy/EMS-ESP/releases/latest"

src/websrc/index.html

@@ -158,7 +158,8 @@
                         <div class="alert alert-warning">
                             <h5><b>Warning!</b> This action <strong>cannot</strong> be undone. This will permanently
                                 delete <strong>all
-                                    the settings and logs.</strong> Please make sure you've made a backup before resetting!</h5>
+                                    the settings and logs.</strong> Please make sure you've made a backup before
+                                resetting!</h5>
                         </div>
                         <div class="modal-body">
                             <h5>Please type in the hostname of the device to confirm.</h5>

src/websrc/myesp.htm

@@ -107,7 +107,7 @@
     <div class="row form-group">
         <label class="col-xs-3">Serial Port<i style="margin-left: 10px;" class="glyphicon glyphicon-info-sign"
                 aria-hidden="true" data-toggle="popover" data-trigger="hover" data-placement="right"
-                data-content="Please choose if you want to enable Serial port for debugging"></i></label>
+                data-content="Serial port for console output directly via USB as well as Telnet"></i></label>
         <div class="col-xs-9">
             <form>
                 <label class="radio-inline">
@@ -224,7 +224,7 @@
     <div class="row form-group">
         <label class="col-xs-3">Heartbeat<i style="margin-left: 10px;" class="glyphicon glyphicon-info-sign"
                 aria-hidden="true" data-toggle="popover" data-trigger="hover" data-placement="right"
-                data-content="Please choose if you want to enable the MQTT heartbeat"></i></label>
+                data-content="enable/disable an automatic MQTT publish with system stats"></i></label>
         <div class="col-xs-9">
             <form>
                 <label class="radio-inline">
@@ -391,7 +391,7 @@
     <br><br>
     <legend>System Status</legend>
     <br>
-    <div class="row text-center">
+    <div class="row text-left">
         <div class="col-md-8 col-md-offset-2">
             <div class="panel panel-default table-responsive">
                 <table class="table table-hover table-striped table-condensed">
@@ -476,9 +476,15 @@
                         </td>
                     </tr>
                 </table>
+
+                <div class="panel-heading">
+                    <div class="panel-title" id="mqttloghdr"></div>
+                </div>
+                <div>
+                    <table id="mqttlogtable" class="table" data-paging="false" data-filtering="false"
+                        data-sorting="false" data-editing="false" data-state="true"></table>
+                </div>
             </div>
         </div>
-        <br>
-        <br>
     </div>
 </div>

src/websrc/myesp.js

@@ -346,8 +346,8 @@ function listStats() {
 
     document.getElementById("uptime").innerHTML = ajaxobj.uptime;
 
-    document.getElementById("heap").innerHTML = ajaxobj.heap + " KB";
+    document.getElementById("heap").innerHTML = ajaxobj.heap + " bytes";
-    document.getElementById("heap").style.width = (ajaxobj.heap * 100) / 41 + "%";
+    document.getElementById("heap").style.width = (ajaxobj.heap * 100) / ajaxobj.initheap + "%";
     colorStatusbar(document.getElementById("heap"));
 
     document.getElementById("flash").innerHTML = ajaxobj.availsize + " KB";
@@ -380,6 +380,8 @@ function listStats() {
         document.getElementById("mqttheartbeat").className = "label label-primary";
     }
 
+    document.getElementById("mqttloghdr").innerHTML = "MQTT Publish Log: (topics are prefixed with <b>" + ajaxobj.mqttloghdr + "</b>)";
+
 }
 
 function getContent(contentname) {
@@ -605,6 +607,55 @@ function initEventTable() {
     });
 }
 
+function initMQTTLogTable() {
+    var newlist = [];
+    for (var i = 0; i < ajaxobj.mqttlog.length; i++) {
+        var data = JSON.stringify(ajaxobj.mqttlog[i]);
+        newlist[i] = {};
+        newlist[i].options = {};
+        newlist[i].value = {};
+        newlist[i].value = JSON.parse(data);
+        newlist[i].options.classes = "warning";
+        newlist[i].options.style = "color: blue";
+    }
+    jQuery(function ($) {
+        window.FooTable.init("#mqttlogtable", {
+            columns: [{
+                "name": "time",
+                "title": "Last Published",
+                "style": { "min-width": "160px" },
+                "parser": function (value) {
+                    if (value < 1563300000) {
+                        return "(" + value + ")";
+                    } else {
+                        var comp = new Date();
+                        value = Math.floor(value + ((comp.getTimezoneOffset() * 60) * -1));
+                        var vuepoch = new Date(value * 1000);
+                        var formatted = vuepoch.getUTCFullYear() +
+                            "-" + twoDigits(vuepoch.getUTCMonth() + 1) +
+                            "-" + twoDigits(vuepoch.getUTCDate()) +
+                            " " + twoDigits(vuepoch.getUTCHours()) +
+                            ":" + twoDigits(vuepoch.getUTCMinutes()) +
+                            ":" + twoDigits(vuepoch.getUTCSeconds());
+                        return formatted;
+                    }
+                },
+                "breakpoints": "xs sm"
+            },
+            {
+                "name": "topic",
+                "title": "Topic",
+            },
+            {
+                "name": "payload",
+                "title": "Payload",
+            },
+            ],
+            rows: newlist
+        });
+    });
+}
+
 function restartESP() {
     inProgress("restart");
 }
@@ -617,6 +668,7 @@ function socketMessageListener(evt) {
         switch (obj.command) {
             case "status":
                 ajaxobj = obj;
+                initMQTTLogTable();
                 getContent("#statuscontent");
                 break;
             case "custom_settings":

tools/wsemulator/wserver.js

@@ -123,20 +123,32 @@ function sendEventLog() {
 function sendStatus() {
     var stats = {
         "command": "status",
-        "heap": 30,
+        "availspiffs": 948,
-        "availsize": 555,
+        "spiffssize": 957,
-        "availspiffs": 445,
+        "initheap": 25392,
-        "spiffssize": 888,
+        "heap": 13944,
-        "sketchsize": 222,
+        "sketchsize": 673,
-        "uptime": "1 Day 6 Hours",
+        "availsize": 2469,
-        "ssid": "SSID",
+        "ip": "10.10.10.198",
-        "mac": "EM:44:11:33:22",
+        "ssid": "derbyshire",
-        "ip": "192.168.2.2",
+        "mac": "DC:4F:11:22:93:06",
-        "signalstr": 66,
+        "signalstr": 62,
-        "systemload": 10,
+        "systemload": 0,
         "mqttconnected": true,
-        "mqttheartbeat": false
+        "mqttheartbeat": false,
+        "uptime": "0 days 0 hours 1 minute 45 seconds",
+        "mqttloghdr": "home/ems-esp/",
+        "mqttlog": [
+            { "topic": "start", "payload": "start", "time": 1565956388 },
+            { "topic": "shower_timer", "payload": "1", "time": 1565956388 },
+            { "topic": "shower_alert", "payload": "0", "time": 1565956388 },
+            { "topic": "boiler_data", "payload": "{\"wWComfort\":\"Hot\",\"wWSelTemp\":60,\"selFlowTemp\":5,\"selBurnPow\":0,\"curBurnPow\":0,\"pumpMod\":0,\"wWCurTmp\":48.4,\"wWCurFlow\":0,\"curFlowTemp\":49.3,\"retTemp\":49.3,\"sysPress\":1.8,\"boilTemp\":50.5,\"wWActivated\":\"on\",\"burnGas\":\"off\",\"heatPmp\":\"off\",\"fanWork\":\"off\",\"ignWork\":\"off\",\"wWCirc\":\"off\",\"wWHeat\":\"on\",\"burnStarts\":223397,\"burnWorkMin\":366019,\"heatWorkMin\":294036,\"ServiceCode\":\"0H\",\"ServiceCodeNumber\":203}", "time": 1565956463 },
+            { "topic": "tapwater_active", "payload": "0", "time": 1565956408 },
+            { "topic": "heating_active", "payload": "0", "time": 1565956408 },
+            { "topic": "thermostat_data", "payload": "{\"thermostat_hc\":\"1\",\"thermostat_seltemp\":15,\"thermostat_currtemp\":23,\"thermostat_mode\":\"auto\"}", "time": 1565956444 }
+        ]
     };
+
     wss.broadcast(stats);
 }
 
@@ -157,7 +169,6 @@ function sendCustomStatus() {
                 { "type": 3, "model": "model 3", "deviceid": "device id3", "version": "version id3", "productid": "product id3" },
                 { "type": 4, "model": "model 4", "deviceid": "device id3", "version": "version id3", "productid": "product id3" },
                 { "type": 5, "model": "model 5", "deviceid": "device id3", "version": "version id3", "productid": "product id3" }
-
             ]
         },
 
@@ -181,6 +192,7 @@ function sendCustomStatus() {
         }
 
     };
+
     wss.broadcast(stats);
 }