store weblog settings, no log filtering, prevent double messages

interface/src/system/LogEventConsole.tsx

@@ -65,9 +65,7 @@ const useStyles = makeStyles((theme: Theme) => ({
 const LogEventConsole: FC<LogEventConsoleProps> = (props) => {
   useWindowSize();
   const classes = useStyles({ topOffset, leftOffset });
-  const { events, compact, level } = props;
+  const { events, compact } = props;
-
-  const filter_events = events.filter((e) => e.l <= level);
 
   const styleLevel = (level: LogLevel) => {
     switch (level) {
@@ -109,7 +107,7 @@ const LogEventConsole: FC<LogEventConsoleProps> = (props) => {
 
   const paddedLevelLabel = (level: LogLevel, compact: boolean) => {
     const label = levelLabel(level);
-    return compact ? ' ' + label[0] : label.padStart(8, '\xa0');
+    return compact ? ' ' + label[0] : label.padStart(7, '\xa0');
   };
 
   const paddedNameLabel = (name: string, compact: boolean) => {
@@ -124,7 +122,7 @@ const LogEventConsole: FC<LogEventConsoleProps> = (props) => {
 
   return (
     <Box id="log-window" className={classes.console}>
-      {filter_events.map((e) => (
+      {events.map((e) => (
         <div className={classes.entry} key={e.i}>
           <span>{e.t}</span>
           {compact && <span>{paddedLevelLabel(e.l, compact)} </span>}

interface/src/system/LogEventController.tsx

@@ -88,6 +88,11 @@ class LogEventController extends Component<
     this.setState({
       compact: checked
     });
+    this.send_data(
+      this.state.level,
+      this.state.max_messages,
+      checked as boolean
+    );
   };
 
   fetchLog = () => {
@@ -118,7 +123,11 @@ class LogEventController extends Component<
         throw Error('Unexpected status code: ' + response.status);
       })
       .then((json) => {
-        this.setState({ level: json.level, max_messages: json.max_messages });
+        this.setState({
+          level: json.level,
+          max_messages: json.max_messages,
+          compact: json.compact
+        });
       })
       .catch((error) => {
         const errorMessage = error.message || 'Unknown error';
@@ -159,22 +168,27 @@ class LogEventController extends Component<
     this.setState({
       max_messages: value as number
     });
-    this.send_data(this.state.level, value as number);
+    this.send_data(this.state.level, value as number, this.state.compact);
   };
 
   changeLevel = (event: React.ChangeEvent<HTMLSelectElement>) => {
     this.setState({
       level: parseInt(event.target.value)
     });
-    this.send_data(parseInt(event.target.value), this.state.max_messages);
+    this.send_data(
+      parseInt(event.target.value),
+      this.state.max_messages,
+      this.state.compact
+    );
   };
 
-  send_data = (level: number, max_messages: number) => {
+  send_data = (level: number, max_messages: number, compact: boolean) => {
     redirectingAuthorizedFetch(LOG_SETTINGS_ENDPOINT, {
       method: 'POST',
       body: JSON.stringify({
         level: level,
-        max_messages: max_messages
+        max_messages: max_messages,
+        compact: compact
       }),
       headers: {
         'Content-Type': 'application/json'
@@ -280,11 +294,12 @@ class LogEventController extends Component<
                 marks={[
                   { value: 25, label: '25' },
                   { value: 50, label: '50' },
-                  { value: 75, label: '75' }
+                  { value: 75, label: '75' },
+                  { value: 100, label: '100' }
                 ]}
                 step={25}
                 min={25}
-                max={75}
+                max={100}
                 onChange={this.changeMaxMessages}
               />
             </Grid>

interface/src/system/types.ts

@@ -58,4 +58,5 @@ export interface LogEvent {
 export interface LogSettings {
   level: LogLevel;
   max_messages: number;
+  compact: boolean;
 }

src/default_settings.h

@@ -164,4 +164,16 @@
 #define EMSESP_DEFAULT_SENSOR_NAME ""
 #endif
 
+#ifndef EMSESP_DEFAULT_WEBLOG_LEVEL
+#define EMSESP_DEFAULT_WEBLOG_LEVEL 6 // INFO
+#endif
+
+#ifndef EMSESP_DEFAULT_WEBLOG_BUFFER
+#define EMSESP_DEFAULT_WEBLOG_BUFFER 50
+#endif
+
+#ifndef EMSESP_DEFAULT_WEBLOG_COMPACT
+#define EMSESP_DEFAULT_WEBLOG_COMPACT true
+#endif
+
 #endif

src/devices/thermostat.cpp

@@ -2265,6 +2265,7 @@ void Thermostat::register_device_values() {
         register_device_value(TAG_THERMOSTAT_DATA, &dateTime_, DeviceValueType::TEXT, nullptr, FL_(dateTime), DeviceValueUOM::NONE, MAKE_CF_CB(set_datetime));
         break;
     case EMS_DEVICE_FLAG_EASY:
+        // Easy TC100 have no date/time, see issue #100, not sure about CT200, so leave it.
         register_device_value(TAG_THERMOSTAT_DATA, &dateTime_, DeviceValueType::TEXT, nullptr, FL_(dateTime), DeviceValueUOM::NONE); // can't set datetime
         break;
     default:

src/web/WebLogService.cpp

@@ -46,7 +46,11 @@ void WebLogService::forbidden(AsyncWebServerRequest * request) {
 
 // start event source service
 void WebLogService::start() {
-    uuid::log::Logger::register_handler(this, uuid::log::Level::INFO); // default is INFO
+    EMSESP::webSettingsService.read([&](WebSettings & settings) {
+        maximum_log_messages_ = settings.weblog_buffer;
+        compact_              = settings.weblog_compact;
+        uuid::log::Logger::register_handler(this, (uuid::log::Level)settings.weblog_level);
+    });
 }
 
 uuid::log::Level WebLogService::log_level() const {
@@ -54,6 +58,10 @@ uuid::log::Level WebLogService::log_level() const {
 }
 
 void WebLogService::log_level(uuid::log::Level level) {
+    EMSESP::webSettingsService.update([&](WebSettings & settings) {
+        settings.weblog_level = level;
+        return StateUpdateResult::CHANGED;
+     }, "local");
     uuid::log::Logger::register_handler(this, level);
 }
 
@@ -66,6 +74,22 @@ void WebLogService::maximum_log_messages(size_t count) {
     while (log_messages_.size() > maximum_log_messages_) {
         log_messages_.pop_front();
     }
+    EMSESP::webSettingsService.update([&](WebSettings & settings) {
+        settings.weblog_buffer = count;
+        return StateUpdateResult::CHANGED;
+     }, "local");
+}
+
+bool WebLogService::compact() {
+    return compact_;
+}
+
+void WebLogService::compact(bool compact) {
+    compact_ = compact;
+    EMSESP::webSettingsService.update([&](WebSettings & settings) {
+        settings.weblog_compact = compact;
+        return StateUpdateResult::CHANGED;
+     }, "local");
 }
 
 WebLogService::QueuedLogMessage::QueuedLogMessage(unsigned long id, std::shared_ptr<uuid::log::Message> && content)
@@ -83,8 +107,13 @@ void WebLogService::operator<<(std::shared_ptr<uuid::log::Message> message) {
     EMSESP::esp8266React.getNTPSettingsService()->read([&](NTPSettings & settings) {
         if (!settings.enabled || (time(nullptr) < 1500000000L)) {
             time_offset_ = 0;
-        } else if (!time_offset_) {
+        } else {
-            time_offset_ = time(nullptr) - uuid::get_uptime_sec();
+            uint32_t offset = time(nullptr) - uuid::get_uptime_sec();
+            // if NTP is more that 1 sec apart, correct setting
+            if (time_offset_ < offset - 1 || time_offset_ > offset + 1) {
+                time_offset_ = offset;
+            }
+
         }
     });
 }
@@ -94,26 +123,25 @@ void WebLogService::loop() {
         return;
     }
 
-    // put a small delay in
-    const uint64_t now = uuid::get_uptime_ms();
-    if (now < last_transmit_ || now - last_transmit_ < REFRESH_SYNC) {
-        return;
-    }
-
     // see if we've advanced
     if (log_messages_.back().id_ <= log_message_id_tail_) {
         return;
     }
 
+    // put a small delay in
+    if (uuid::get_uptime_ms() - last_transmit_ < REFRESH_SYNC) {
+        return;
+    }
+    last_transmit_ = uuid::get_uptime_ms();
+
     // flush
-    for (auto it = log_messages_.begin(); it != log_messages_.end(); it++) {
+    for (const auto & message : log_messages_) {
-        if (it->id_ > log_message_id_tail_) {
+        if (message.id_ > log_message_id_tail_) {
-            transmit(*it);
+            log_message_id_tail_ = message.id_;
+            transmit(message);
+            return;
         }
     }
-
-    log_message_id_tail_ = log_messages_.back().id_;
-    last_transmit_       = uuid::get_uptime_ms();
 }
 
 // convert time to real offset
@@ -130,7 +158,7 @@ char * WebLogService::messagetime(char * out, const uint64_t t) {
 
 // send to web eventsource
 void WebLogService::transmit(const QueuedLogMessage & message) {
-    DynamicJsonDocument jsonDocument = DynamicJsonDocument(EMSESP_JSON_SIZE_SMALL);
+    DynamicJsonDocument jsonDocument = DynamicJsonDocument(EMSESP_JSON_SIZE_MEDIUM);
     JsonObject          logEvent     = jsonDocument.to<JsonObject>();
     char                time_string[25];
 
@@ -144,33 +172,30 @@ void WebLogService::transmit(const QueuedLogMessage & message) {
     char * buffer = new char[len + 1];
     if (buffer) {
         serializeJson(jsonDocument, buffer, len + 1);
-        events_.send(buffer, "message", millis());
+        events_.send(buffer, "message", message.id_);
     }
     delete[] buffer;
 }
 
-// send the complete log buffer to the API, filtering on log level
+// send the complete log buffer to the API, not filtering on log level
 void WebLogService::fetchLog(AsyncWebServerRequest * request) {
     MsgpackAsyncJsonResponse * response = new MsgpackAsyncJsonResponse(false, EMSESP_JSON_SIZE_XXLARGE_DYN); // 8kb buffer
     JsonObject                 root     = response->getRoot();
-
+    JsonArray                  log      = root.createNestedArray("events");
-    JsonArray log = root.createNestedArray("events");
-
-    for (const auto & message : log_messages_) {
-        if (message.content_->level <= log_level()) {
-            JsonObject logEvent = log.createNestedObject();
-            char       time_string[25];
-
-            logEvent["t"] = messagetime(time_string, message.content_->uptime_ms);
-            logEvent["l"] = message.content_->level;
-            logEvent["i"] = message.id_;
-            logEvent["n"] = message.content_->name;
-            logEvent["m"] = message.content_->text;
-        }
-    }
 
     log_message_id_tail_ = log_messages_.back().id_;
+    last_transmit_       = uuid::get_uptime_ms();
+    for (const auto & message : log_messages_) {
+        JsonObject logEvent = log.createNestedObject();
+        char       time_string[25];
 
+        logEvent["t"] = messagetime(time_string, message.content_->uptime_ms);
+        logEvent["l"] = message.content_->level;
+        logEvent["i"] = message.id_;
+        logEvent["n"] = message.content_->name;
+        logEvent["m"] = message.content_->text;
+    }
+    log_message_id_tail_ = log_messages_.back().id_;
     response->setLength();
     request->send(response);
 }
@@ -189,6 +214,9 @@ void WebLogService::setValues(AsyncWebServerRequest * request, JsonVariant & jso
     uint8_t max_messages = body["max_messages"];
     maximum_log_messages(max_messages);
 
+    bool comp = body["compact"];
+    compact(comp);
+
     request->send(200); // OK
 }
 
@@ -198,6 +226,7 @@ void WebLogService::getValues(AsyncWebServerRequest * request) {
     JsonObject          root     = response->getRoot();
     root["level"]                = log_level();
     root["max_messages"]         = maximum_log_messages();
+    root["compact"]              = compact();
     response->setLength();
     request->send(response);
 }

src/web/WebLogService.h

@@ -35,7 +35,7 @@ namespace emsesp {
 class WebLogService : public uuid::log::Handler {
   public:
     static constexpr size_t MAX_LOG_MESSAGES = 50;
-    static constexpr size_t REFRESH_SYNC     = 200;
+    static constexpr size_t REFRESH_SYNC     = 50;
 
     WebLogService(AsyncWebServer * server, SecurityManager * securityManager);
 
@@ -44,6 +44,8 @@ class WebLogService : public uuid::log::Handler {
     void             log_level(uuid::log::Level level);
     size_t           maximum_log_messages() const;
     void             maximum_log_messages(size_t count);
+    bool             compact();
+    void             compact(bool compact);
     void             loop();
 
     virtual void operator<<(std::shared_ptr<uuid::log::Message> message);
@@ -77,6 +79,7 @@ class WebLogService : public uuid::log::Handler {
     unsigned long               log_message_id_tail_  = 0;                // last event shown on the screen after fetch
     std::list<QueuedLogMessage> log_messages_;                            // Queued log messages, in the order they were received
     time_t                      time_offset_ = 0;
+    bool                        compact_     = true;
 };
 
 } // namespace emsesp

src/web/WebSettingsService.cpp

@@ -64,6 +64,9 @@ void WebSettings::read(WebSettings & settings, JsonObject & root) {
     root["dallas_format"]        = settings.dallas_format;
     root["bool_format"]          = settings.bool_format;
     root["enum_format"]          = settings.enum_format;
+    root["weblog_level"]         = settings.weblog_level;
+    root["weblog_buffer"]        = settings.weblog_buffer;
+    root["weblog_compact"]       = settings.weblog_compact;
 
     for (uint8_t i = 0; i < NUM_SENSOR_NAMES; i++) {
         char buf[20];
@@ -202,6 +205,10 @@ StateUpdateResult WebSettings::update(JsonObject & root, WebSettings & settings)
     settings.enum_format = root["enum_format"] | EMSESP_DEFAULT_ENUM_FORMAT;
     EMSESP::enum_format(settings.enum_format);
 
+    settings.weblog_level   = root["weblog_level"] | EMSESP_DEFAULT_WEBLOG_LEVEL;
+    settings.weblog_buffer  = root["weblog_buffer"] | EMSESP_DEFAULT_WEBLOG_BUFFER;
+    settings.weblog_compact = root["weblog_compact"] | EMSESP_DEFAULT_WEBLOG_COMPACT;
+
     for (uint8_t i = 0; i < NUM_SENSOR_NAMES; i++) {
         char buf[20];
         snprintf_P(buf, sizeof(buf), PSTR("sensor_id%d"), i);

src/web/WebSettingsService.h

@@ -64,6 +64,9 @@ class WebSettings {
     uint8_t  dallas_format;
     uint8_t  bool_format;
     uint8_t  enum_format;
+    int8_t   weblog_level;
+    uint8_t  weblog_buffer;
+    bool     weblog_compact;
 
     struct {
         String  id;