Show realtime debug log in WebUI #71

interface/src/components/SectionContent.tsx

@@ -15,13 +15,14 @@ const useStyles = makeStyles((theme: Theme) =>
 interface SectionContentProps {
   title: string;
   titleGutter?: boolean;
+  id?: string;
 }
 
 const SectionContent: React.FC<SectionContentProps> = (props) => {
-  const { children, title, titleGutter } = props;
+  const { children, title, titleGutter, id } = props;
   const classes = useStyles();
   return (
-    <Paper className={classes.content}>
+    <Paper id={id} className={classes.content}>
       <Typography variant="h6" gutterBottom={titleGutter}>
         {title}
       </Typography>

interface/src/project/EMSESPDevicesController.tsx

@@ -6,6 +6,7 @@ import {
   RestFormLoader,
   SectionContent
 } from '../components';
+
 import { ENDPOINT_ROOT } from '../api';
 import EMSESPDevicesForm from './EMSESPDevicesForm';
 import { EMSESPDevices } from './EMSESPtypes';

interface/src/project/EMSESPDevicesForm.tsx

@@ -36,6 +36,7 @@ import {
   withAuthenticatedContext,
   AuthenticatedContextProps
 } from '../authentication';
+
 import { RestFormProps, FormButton, extractEventValue } from '../components';
 
 import {

interface/src/system/LogEventConsole.tsx

@@ -38,10 +38,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     whiteSpace: 'nowrap'
   },
   debug: {
-    color: '#0000ff'
+    color: '#00FFFF'
   },
   info: {
-    color: '#00ff00'
+    color: '#ffff00'
   },
   notice: {
     color: '#ffff00'
@@ -67,7 +67,8 @@ const LogEventConsole: FC<LogEventConsoleProps> = (props) => {
         return classes.info;
       case LogLevel.NOTICE:
         return classes.notice;
-      case LogLevel.ERR:
+      case LogLevel.WARNING:
+      case LogLevel.ERROR:
         return classes.err;
       default:
         return classes.unknown;
@@ -80,10 +81,14 @@ const LogEventConsole: FC<LogEventConsoleProps> = (props) => {
         return 'DEBUG';
       case LogLevel.INFO:
         return 'INFO';
-      case LogLevel.ERR:
+      case LogLevel.ERROR:
-        return 'ERR';
+        return 'ERROR';
       case LogLevel.NOTICE:
         return 'NOTICE';
+      case LogLevel.WARNING:
+        return 'WARNING';
+      case LogLevel.TRACE:
+        return 'TRACE';
       default:
         return '?';
     }
@@ -91,17 +96,23 @@ const LogEventConsole: FC<LogEventConsoleProps> = (props) => {
 
   const paddedLevelLabel = (level: LogLevel) => {
     const label = levelLabel(level);
-    return label.padStart(7, '\xa0');
+    return label.padStart(8, '\xa0');
+  };
+
+  const paddedNameLabel = (name: string) => {
+    const label = '[' + name + ']';
+    return label.padStart(8, '\xa0');
   };
 
   return (
-    <Box className={classes.console}>
+    <Box id="log-window" className={classes.console}>
       {events.map((e) => (
         <div className={classes.entry}>
-          <span>{e.time} </span>
+          <span>{e.time}</span>
           <span className={styleLevel(e.level)}>
             {paddedLevelLabel(e.level)}{' '}
           </span>
+          <span>{paddedNameLabel(e.name)} </span>
           <span>{e.message}</span>
         </div>
       ))}

interface/src/system/LogEventController.tsx

@@ -1,23 +1,26 @@
 import { Component } from 'react';
-import { FormActions, FormButton } from '../components';
+
+import { createStyles, WithStyles, Theme } from '@material-ui/core';
 
 import {
-  createStyles,
+  restController,
-  WithStyles,
+  RestControllerProps,
-  withStyles,
+  RestFormLoader,
-  Typography,
+  SectionContent
-  Theme,
+} from '../components';
-  Paper
-} from '@material-ui/core';
 
-import { LogEvent } from './types';
-import { EVENT_SOURCE_ROOT } from '../api/Env';
-import LogEventConsole from './LogEventConsole';
 import { addAccessTokenParameter } from '../authentication';
 
-import SaveIcon from '@material-ui/icons/Save';
+import { ENDPOINT_ROOT, EVENT_SOURCE_ROOT } from '../api';
+export const FETCH_LOG_ENDPOINT = ENDPOINT_ROOT + 'fetchLog';
+export const LOG_SETTINGS_ENDPOINT = ENDPOINT_ROOT + 'logSettings';
 
-const LOG_EVENT_EVENT_SOURCE_URL = EVENT_SOURCE_ROOT + 'log';
+export const LOG_EVENT_EVENT_SOURCE_URL = EVENT_SOURCE_ROOT + 'log';
+
+import LogEventForm from './LogEventForm';
+import LogEventConsole from './LogEventConsole';
+
+import { LogEvent, LogSettings } from './types';
 
 interface LogEventControllerState {
   eventSource?: EventSource;
@@ -32,7 +35,8 @@ const styles = (theme: Theme) =>
     }
   });
 
-type LogEventControllerProps = WithStyles<typeof styles>;
+type LogEventControllerProps = RestControllerProps<LogSettings> &
+  WithStyles<typeof styles>;
 
 class LogEventController extends Component<
   LogEventControllerProps,
@@ -49,6 +53,8 @@ class LogEventController extends Component<
   }
 
   componentDidMount() {
+    this.props.loadData();
+    this.fetchLog();
     this.configureEventSource();
   }
 
@@ -61,6 +67,24 @@ class LogEventController extends Component<
     }
   }
 
+  fetchLog = () => {
+    fetch(FETCH_LOG_ENDPOINT)
+      .then((response) => {
+        if (response.status === 200) {
+          return response.json();
+        } else {
+          throw Error('Unexpected status code: ' + response.status);
+        }
+      })
+      .then((json) => {
+        this.setState({ events: json.events });
+      })
+      .catch((error) => {
+        this.setState({ events: [] });
+        throw Error('Unexpected error: ' + error);
+      });
+  };
+
   configureEventSource = () => {
     this.eventSource = new EventSource(
       addAccessTokenParameter(LOG_EVENT_EVENT_SOURCE_URL)
@@ -86,24 +110,16 @@ class LogEventController extends Component<
   };
 
   render() {
-    const { classes } = this.props;
     return (
-      <Paper id="log-window" className={classes.content}>
+      <SectionContent title="System Log" id="log-window">
-        <Typography variant="h6">System Log</Typography>
+        <RestFormLoader
-        <FormActions>
+          {...this.props}
-          <FormButton
+          render={(formProps) => <LogEventForm {...formProps} />}
-            startIcon={<SaveIcon />}
+        />
-            variant="contained"
-            color="secondary"
-            // onClick={this.requestNetworkScan}
-          >
-            Save
-          </FormButton>
-        </FormActions>
         <LogEventConsole events={this.state.events} />
-      </Paper>
+      </SectionContent>
     );
   }
 }
 
-export default withStyles(styles)(LogEventController);
+export default restController(LOG_SETTINGS_ENDPOINT, LogEventController);

interface/src/system/LogEventForm.tsx

@@ -0,0 +1,87 @@
+import { Component } from 'react';
+
+import {
+  ValidatorForm,
+  SelectValidator
+} from 'react-material-ui-form-validator';
+
+import MenuItem from '@material-ui/core/MenuItem';
+
+import {
+  redirectingAuthorizedFetch,
+  withAuthenticatedContext,
+  AuthenticatedContextProps
+} from '../authentication';
+
+import { RestFormProps } from '../components';
+import { LogSettings } from './types';
+
+import { ENDPOINT_ROOT } from '../api';
+export const LOG_SETTINGS_ENDPOINT = ENDPOINT_ROOT + 'logSettings';
+
+type LogEventFormProps = AuthenticatedContextProps & RestFormProps<LogSettings>;
+
+class LogEventForm extends Component<LogEventFormProps> {
+  changeLevel = (event: React.ChangeEvent<HTMLSelectElement>) => {
+    const { data, setData } = this.props;
+    setData({
+      ...data,
+      level: parseInt(event.target.value)
+    });
+
+    redirectingAuthorizedFetch(LOG_SETTINGS_ENDPOINT, {
+      method: 'POST',
+      body: JSON.stringify({ level: event.target.value }),
+      headers: {
+        'Content-Type': 'application/json'
+      }
+    })
+      .then((response) => {
+        if (response.status === 200) {
+          return response.json();
+        }
+        throw Error('Unexpected response code: ' + response.status);
+      })
+      .then((json) => {
+        this.props.enqueueSnackbar('Log settings changed', {
+          variant: 'success'
+        });
+        setData({
+          ...data,
+          level: json.level
+        });
+      })
+      .catch((error) => {
+        this.props.enqueueSnackbar(
+          error.message || 'Problem changing log settings',
+          { variant: 'warning' }
+        );
+      });
+  };
+
+  render() {
+    const { data, saveData } = this.props;
+    return (
+      <ValidatorForm onSubmit={saveData}>
+        <SelectValidator
+          name="level"
+          label="Log Level"
+          value={data.level}
+          variant="outlined"
+          onChange={this.changeLevel}
+          margin="normal"
+        >
+          <MenuItem value={-1}>OFF</MenuItem>
+          <MenuItem value={3}>ERROR</MenuItem>
+          <MenuItem value={4}>WARNING</MenuItem>
+          <MenuItem value={5}>NOTICE</MenuItem>
+          <MenuItem value={6}>INFO</MenuItem>
+          <MenuItem value={7}>DEBUG</MenuItem>
+          <MenuItem value={8}>TRACE</MenuItem>
+        </SelectValidator>
+      </ValidatorForm>
+    );
+  }
+}
+
+export default withAuthenticatedContext(LogEventForm);

interface/src/system/OTASettingsController.tsx

@@ -1,4 +1,4 @@
-import React, { Component } from 'react';
+import { Component } from 'react';
 
 import {
   restController,

interface/src/system/OTASettingsForm.tsx

@@ -11,6 +11,7 @@ import {
   FormButton,
   FormActions
 } from '../components';
+
 import { isIP, isHostname, or } from '../validators';
 
 import { OTASettings } from './types';

interface/src/system/types.ts

@@ -38,14 +38,21 @@ export interface OTASettings {
 }
 
 export enum LogLevel {
-  ERR = 3,
+  ERROR = 3,
+  WARNING = 4,
   NOTICE = 5,
   INFO = 6,
-  DEBUG = 7
+  DEBUG = 7,
+  TRACE = 8
 }
 
 export interface LogEvent {
   time: string;
   level: LogLevel;
+  name: string;
   message: string;
 }
+
+export interface LogSettings {
+  level: LogLevel;
+}

mock-api/server.js

@@ -14,6 +14,54 @@ const server = express()
 const es_port = 3090
 const ES_ENDPOINT_ROOT = '/es/'
 
+// LOG
+const LOG_SETTINGS_ENDPOINT = REST_ENDPOINT_ROOT + 'logSettings'
+const log_settings = {
+  level: 6,
+}
+
+const FETCH_LOG_ENDPOINT = REST_ENDPOINT_ROOT + 'fetchLog'
+const fetch_log = {
+  events: [
+    {
+      time: '000+00:00:00.001',
+      level: 3,
+      name: 'system',
+      message: 'this is message 3',
+    },
+    {
+      time: '000+00:00:00.002',
+      level: 4,
+      name: 'system',
+      message: 'this is message 4',
+    },
+    {
+      time: '000+00:00:00.002',
+      level: 5,
+      name: 'system',
+      message: 'this is message 5',
+    },
+    {
+      time: '000+00:00:00.002',
+      level: 6,
+      name: 'system',
+      message: 'this is message 6',
+    },
+    {
+      time: '000+00:00:00.002',
+      level: 7,
+      name: 'emsesp',
+      message: 'this is message 7',
+    },
+    {
+      time: '000+00:00:00.002',
+      level: 8,
+      name: 'mqtt',
+      message: 'this is message 8',
+    },
+  ],
+}
+
 // NTP
 const NTP_STATUS_ENDPOINT = REST_ENDPOINT_ROOT + 'ntpStatus'
 const NTP_SETTINGS_ENDPOINT = REST_ENDPOINT_ROOT + 'ntpSettings'
@@ -703,6 +751,21 @@ const emsesp_devicedata_3 = {
   data: [],
 }
 
+// LOG
+app.get(FETCH_LOG_ENDPOINT, (req, res) => {
+  res.json(fetch_log)
+})
+app.get(LOG_SETTINGS_ENDPOINT, (req, res) => {
+  res.json(log_settings)
+})
+app.post(LOG_SETTINGS_ENDPOINT, (req, res) => {
+  console.log('New log level is ' + req.body.level)
+  const data = {
+    level: req.body.level,
+  }
+  res.json(data)
+})
+
 // NETWORK
 app.get(NETWORK_STATUS_ENDPOINT, (req, res) => {
   res.json(network_status)
@@ -943,7 +1006,8 @@ const streamLog = (req, res) => {
 
     const data = {
       time: '000+00:00:00.000',
-      level: 4,
+      level: 3,
+      name: 'system',
       message: 'this is message #' + count,
     }
 

src/system.cpp

@@ -413,7 +413,6 @@ void System::loop() {
         send_heartbeat();
     }
 
-    /*
 #ifndef EMSESP_STANDALONE
 #if defined(EMSESP_DEBUG)
     static uint32_t last_memcheck_ = 0;
@@ -423,7 +422,6 @@ void System::loop() {
     }
 #endif
 #endif
-*/
 
 #endif
 }

src/test/test.cpp

@@ -930,6 +930,7 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd) {
     }
 
     if (command == "api") {
+#if defined(EMSESP_STANDALONE)
         shell.printfln(F("Testing RESTful API..."));
         Mqtt::ha_enabled(false);
         run_test("general");
@@ -942,6 +943,7 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd) {
 
         request.url("/api/boiler/syspress");
         EMSESP::webAPIService.webAPIService_get(&request);
+#endif
     }
 }
 

src/web/WebAPIService.cpp

@@ -223,11 +223,8 @@ void WebAPIService::send_message_response(AsyncWebServerRequest * request, uint1
 }
 
 /**
- * Extract only the path component from the passed URI 
+ * Extract only the path component from the passed URI and normalized it.
- * and normalized it.
+ * Ex. //one/two////three/// becomes /one/two/three
- * Ex. //one/two////three///
- * becomes
- *  /one/two/three
  */
 std::string SUrlParser::path() {
     std::string s = "/"; // set up the beginning slash

src/web/WebLogService.cpp

@@ -23,10 +23,22 @@ using namespace std::placeholders;
 namespace emsesp {
 
 WebLogService::WebLogService(AsyncWebServer * server, SecurityManager * securityManager)
-    : _events(EVENT_SOURCE_LOG_PATH) {
+    : _events(EVENT_SOURCE_LOG_PATH)
+    , _setLevel(LOG_SETTINGS_PATH, std::bind(&WebLogService::setLevel, this, _1, _2), 256) { // for POSTS
+
     _events.setFilter(securityManager->filterRequest(AuthenticationPredicates::IS_ADMIN));
     server->addHandler(&_events);
     server->on(EVENT_SOURCE_LOG_PATH, HTTP_GET, std::bind(&WebLogService::forbidden, this, _1));
+
+    // for bring back the whole log
+    server->on(FETCH_LOG_PATH, HTTP_GET, std::bind(&WebLogService::fetchLog, this, _1));
+
+    server->on(LOG_SETTINGS_PATH, HTTP_GET, std::bind(&WebLogService::getLevel, this, _1));
+
+    // for setting a level
+    server->addHandler(&_setLevel);
+
+    // start event source service
     start();
 }
 
@@ -42,22 +54,6 @@ uuid::log::Level WebLogService::log_level() const {
     return uuid::log::Logger::get_log_level(this);
 }
 
-void WebLogService::remove_queued_messages(uuid::log::Level level) {
-    unsigned long offset = 0;
-
-    for (auto it = log_messages_.begin(); it != log_messages_.end();) {
-        if (it->content_->level > level) {
-            offset++;
-            it = log_messages_.erase(it);
-        } else {
-            it->id_ -= offset;
-            it++;
-        }
-    }
-
-    log_message_id_ -= offset;
-}
-
 void WebLogService::log_level(uuid::log::Level level) {
     uuid::log::Logger::register_handler(this, level);
 }
@@ -86,22 +82,22 @@ void WebLogService::operator<<(std::shared_ptr<uuid::log::Message> message) {
 }
 
 void WebLogService::loop() {
-    if (!_events.count()) {
+    if (!_events.count() || log_messages_.empty()) {
         return;
     }
 
-    while (!log_messages_.empty() && can_transmit()) {
+    // put a small delay in
-        transmit(log_messages_.front());
-        log_messages_.pop_front();
-    }
-}
-
-bool WebLogService::can_transmit() {
     const uint64_t now = uuid::get_uptime_ms();
-    if (now < last_transmit_ || now - last_transmit_ < 100) {
+    if (now < last_transmit_ || now - last_transmit_ < 50) {
-        return false;
+        return;
+    }
+
+    // see if we've advanced
+    if (log_messages_.back().id_ > log_message_id_tail_) {
+        transmit(log_messages_.back());
+        log_message_id_tail_ = log_messages_.back().id_;
+        last_transmit_       = uuid::get_uptime_ms();
     }
-    return true;
 }
 
 // send to web eventsource
@@ -110,6 +106,7 @@ void WebLogService::transmit(const QueuedLogMessage & message) {
     JsonObject          logEvent     = jsonDocument.to<JsonObject>();
     logEvent["time"]                 = uuid::log::format_timestamp_ms(message.content_->uptime_ms, 3);
     logEvent["level"]                = message.content_->level;
+    logEvent["name"]                 = message.content_->name;
     logEvent["message"]              = message.content_->text;
 
     size_t len    = measureJson(jsonDocument);
@@ -119,8 +116,57 @@ void WebLogService::transmit(const QueuedLogMessage & message) {
         _events.send(buffer, "message", millis());
     }
     delete[] buffer;
+}
 
-    last_transmit_ = uuid::get_uptime_ms();
+// send the current log buffer to the API
+void WebLogService::fetchLog(AsyncWebServerRequest * request) {
+    AsyncJsonResponse * response = new AsyncJsonResponse(false, EMSESP_JSON_SIZE_XLARGE_DYN);
+    JsonObject          root     = response->getRoot();
+
+    JsonArray log = root.createNestedArray("events");
+
+    for (const auto & msg : log_messages_) {
+        JsonObject logEvent = log.createNestedObject();
+        auto       message  = std::move(msg);
+        logEvent["time"]    = uuid::log::format_timestamp_ms(message.content_->uptime_ms, 3);
+        logEvent["level"]   = message.content_->level;
+        logEvent["name"]    = message.content_->name;
+        logEvent["message"] = message.content_->text;
+    }
+
+    log_message_id_tail_ = log_messages_.back().id_;
+
+    response->setLength();
+    request->send(response);
+}
+
+// sets the level
+void WebLogService::setLevel(AsyncWebServerRequest * request, JsonVariant & json) {
+    if (not json.is<JsonObject>()) {
+        return;
+    }
+    auto &&          body  = json.as<JsonObject>();
+    uuid::log::Level level = body["level"];
+    log_level(level);
+
+    // send the value back
+    AsyncJsonResponse * response = new AsyncJsonResponse(false, EMSESP_JSON_SIZE_SMALL);
+    JsonObject          root     = response->getRoot();
+    root["level"]                = log_level();
+    response->setLength();
+    request->send(response);
+}
+
+// return the current log level
+void WebLogService::getLevel(AsyncWebServerRequest * request) {
+    AsyncJsonResponse * response = new AsyncJsonResponse(false, EMSESP_JSON_SIZE_SMALL);
+    JsonObject          root     = response->getRoot();
+
+    auto level    = log_level();
+    root["level"] = level;
+
+    response->setLength();
+    request->send(response);
 }
 
 } // namespace emsesp

src/web/WebLogService.h

@@ -27,6 +27,8 @@
 #include <uuid/log.h>
 
 #define EVENT_SOURCE_LOG_PATH "/es/log"
+#define FETCH_LOG_PATH "/rest/fetchLog"
+#define LOG_SETTINGS_PATH "/rest/logSettings"
 
 namespace emsesp {
 
@@ -59,13 +61,17 @@ class WebLogService : public uuid::log::Handler {
     };
 
     void forbidden(AsyncWebServerRequest * request);
-    void remove_queued_messages(uuid::log::Level level);
-    bool can_transmit();
     void transmit(const QueuedLogMessage & message);
+    void fetchLog(AsyncWebServerRequest * request);
+    void getLevel(AsyncWebServerRequest * request);
+
+    void                        setLevel(AsyncWebServerRequest * request, JsonVariant & json);
+    AsyncCallbackJsonWebHandler _setLevel; // for POSTs
 
     uint64_t                    last_transmit_        = 0;                // Last transmit time
     size_t                      maximum_log_messages_ = MAX_LOG_MESSAGES; // Maximum number of log messages to buffer before they are output
     unsigned long               log_message_id_       = 0;                // The next identifier to use for queued log messages
+    unsigned long               log_message_id_tail_  = 0;
     std::list<QueuedLogMessage> log_messages_; // Queued log messages, in the order they were received
 };