Merge pull request #2018 from proddy/dev

network

Makefile

@@ -3,8 +3,9 @@
 # This is mainly used to generate the .o files for SonarQube analysis
 #
 
-NUMJOBS=${NUMJOBS:-" -j2 "}
+# NUMJOBS=${NUMJOBS:-" -j10 "}
-MAKEFLAGS+="j "
+# MAKEFLAGS+="j "
+
 #----------------------------------------------------------------------
 # Project Structure
 #----------------------------------------------------------------------

README.md

@@ -29,17 +29,19 @@
 
 ## **Installing**
 
-For a quick install of the latest stable release go to [https://install.emsesp.org](https://install.emsesp.org). For other methods of installing and upgrading, and switching to the development version go to the section in the [documentation](https://emsesp.org).
+For a quick install of the latest stable release go to [https://install.emsesp.org](https://install.emsesp.org). For other methods of installing and upgrading, and switching over to the development version go to [this section](https://emsesp.org/Getting-Started/#first-time-install) in the documentation.
 
-## **Documentation and Wiki**
+If you're upgrading a BBQKees Electronics EMS Gateway and unsure which firmware to use, please refer to the [this overview](https://emsesp.org/Getting-Started/#bbqkees-electronics-ems-gateway).
 
-Visit [emsesp.org](https://emsesp.org) for details on how to install and configure EMS-ESP.
+## **Documentation**
+
+Visit [emsesp.org](https://emsesp.org) for more details on how to install and configure EMS-ESP. There is also a collection of Frequently Asked Questions and Troubleshooting tips with example customizations from the community.
 
 ## **Getting Support**
 
 To chat with the community reach out on our [Discord Server](https://discord.gg/3J3GgnzpyT).
 
-If you like **EMS-ESP**, please give it a ✨ on GitHub, or even better fork it and contribute. You can also offer a small donation. This is an open-source project maintained by volunteers, and your support is greatly appreciated.
+If you find an issue or have a request, see [here](https://emsesp.org/Support/) on how to submit a bug report or feature request.
 
 ## **Live Demo**
 
@@ -49,6 +51,10 @@ For a live demo go to [demo.emsesp.org](https://demo.emsesp.org). Pick a languag
 
 EMS-ESP is a project created by [proddy](https://github.com/proddy) and owned and maintained by both [proddy](https://github.com/proddy) and [MichaelDvP](https://github.com/MichaelDvP) with support from [BBQKees Electronics](https://bbqkees-electronics.nl).
 
+You can contact us using [this form](https://emsesp.org/Contact/).
+
+If you like **EMS-ESP**, please give it a ✨ on GitHub, or even better fork it and contribute. You can also offer a small donation. This is an open-source project maintained by volunteers, and your support is greatly appreciated.
+
 ## **Libraries used**
 
 - [esp8266-react](https://github.com/rjwats/esp8266-react) by @rjwats for the core framework that provides the Web UI, which has been heavily modified

interface/package.json

@@ -48,7 +48,7 @@
     "@trivago/prettier-plugin-sort-imports": "^4.3.0",
     "@types/formidable": "^3",
     "@types/node": "^22.5.5",
-    "@types/react": "^18.3.5",
+    "@types/react": "^18.3.6",
     "@types/react-dom": "^18.3.0",
     "@types/react-router-dom": "^5.3.3",
     "concurrently": "^9.0.1",

interface/yarn.lock

@@ -1487,7 +1487,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/react@npm:*, @types/react@npm:^18.3.5":
+"@types/react@npm:*":
   version: 18.3.5
   resolution: "@types/react@npm:18.3.5"
   dependencies:
@@ -1497,6 +1497,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/react@npm:^18.3.6":
+  version: 18.3.6
+  resolution: "@types/react@npm:18.3.6"
+  dependencies:
+    "@types/prop-types": "npm:*"
+    csstype: "npm:^3.0.2"
+  checksum: 10c0/e347787a496f2006fe61358316e040c22548a1669559277b72dad39e2f1f713080115be9fe6c30f9d1d28a54054b96cf6c06471be6967a3605e31f5b84a6cec7
+  languageName: node
+  linkType: hard
+
 "@types/responselike@npm:^1.0.0":
   version: 1.0.3
   resolution: "@types/responselike@npm:1.0.3"
@@ -1648,7 +1658,7 @@ __metadata:
     "@trivago/prettier-plugin-sort-imports": "npm:^4.3.0"
     "@types/formidable": "npm:^3"
     "@types/node": "npm:^22.5.5"
-    "@types/react": "npm:^18.3.5"
+    "@types/react": "npm:^18.3.6"
     "@types/react-dom": "npm:^18.3.0"
     "@types/react-router-dom": "npm:^5.3.3"
     alova: "npm:3.0.16"

lib/framework/FSPersistence.h

@@ -26,11 +26,11 @@ class FSPersistence {
             if (error == DeserializationError::Ok && jsonDocument.is<JsonObject>()) {
                 JsonObject jsonObject = jsonDocument.as<JsonObject>();
                 _statefulService->updateWithoutPropagation(jsonObject, _stateUpdater);
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
-                Serial.println();
+                // Serial.println();
-                Serial.printf("Reading settings from %s ", _filePath);
+                // Serial.printf("Reading settings from %s ", _filePath);
-                serializeJson(jsonDocument, Serial);
+                // serializeJson(jsonDocument, Serial);
-                Serial.println();
+                // Serial.println();
 #endif
                 settingsFile.close();
                 return;
@@ -40,10 +40,10 @@ class FSPersistence {
 
 // If we reach here we have not been successful in loading the config,
 // hard-coded emergency defaults are now applied.
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
-        Serial.println();
+        // Serial.println();
-        Serial.printf("Applying defaults to %s", _filePath);
+        // Serial.printf("Applying defaults to %s", _filePath);
-        Serial.println();
+        // Serial.println();
 #endif
         applyDefaults();
         writeToFS(); // added to make sure the initial file is created
@@ -74,11 +74,11 @@ class FSPersistence {
         }
 
         // serialize the data to the file
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
-        Serial.println();
+        // Serial.println();
-        Serial.printf("Writing settings to %s ", _filePath);
+        // Serial.printf("Writing settings to %s ", _filePath);
-        serializeJson(jsonDocument, Serial);
+        // serializeJson(jsonDocument, Serial);
-        Serial.println();
+        // Serial.println();
 #endif
         serializeJson(jsonDocument, settingsFile);
         settingsFile.close();

lib/framework/MqttSettingsService.cpp

@@ -133,11 +133,7 @@ MqttClient * MqttSettingsService::getMqttClient() {
 
 void MqttSettingsService::onMqttConnect(bool sessionPresent) {
     (void)sessionPresent;
-    // _disconnectedAt = 0;
     emsesp::EMSESP::mqtt_.on_connect();
-#ifdef EMSESP_DEBUG
-    emsesp::EMSESP::logger().debug("Connected to MQTT, %s", (sessionPresent) ? ("with persistent session") : ("without persistent session"));
-#endif
 }
 
 void MqttSettingsService::onMqttDisconnect(espMqttClientTypes::DisconnectReason reason) {
@@ -190,7 +186,7 @@ bool MqttSettingsService::configureMqtt() {
         _reconfigureMqtt = false;
 #ifndef TASMOTA_SDK
         if (_state.enableTLS) {
-#if EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
             emsesp::EMSESP::logger().debug("Start secure MQTT with rootCA");
 #endif
             static_cast<espMqttClientSecure *>(_mqttClient)->setServer(_state.host.c_str(), _state.port);

lib/framework/NetworkSettingsService.cpp

@@ -8,6 +8,7 @@ NetworkSettingsService::NetworkSettingsService(AsyncWebServer * server, FS * fs,
     , _lastConnectionAttempt(0)
     , _stopping(false) {
     addUpdateHandler([this] { reconfigureWiFiConnection(); }, false);
+    // Eth is also bound to the WifiGeneric event handler
     WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { WiFiEvent(event, info); });
 }
 
@@ -180,9 +181,8 @@ void NetworkSettingsService::setWiFiPowerOnRSSI() {
     else if (min_tx_pwr > 20)
         p = WIFI_POWER_5dBm;
 
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
-    uint8_t set_power = min_tx_pwr / 10; // this is the recommended power setting to use
+        // emsesp::EMSESP::logger().debug("Recommended WiFi Tx Power (set_power %d, new power %d, rssi %d, threshold %d)", min_tx_pwr / 10, p, rssi, threshold);
-    emsesp::EMSESP::logger().debug("Recommended WiFi Tx Power (set_power %d, new power %d, rssi %d, threshold %d)", set_power, p, rssi, threshold);
 #endif
 
     if (!WiFi.setTxPower(p)) {
@@ -312,7 +312,7 @@ void NetworkSettingsService::WiFiEvent(WiFiEvent_t event, WiFiEventInfo_t info)
 
     case ARDUINO_EVENT_WIFI_STA_GOT_IP:
         char result[10];
-        emsesp::EMSESP::logger().info("WiFi connected (IP=%s, hostname=%s, TxPower=%s dBm)",
+        emsesp::EMSESP::logger().info("WiFi connected (Local IP=%s, hostname=%s, TxPower=%s dBm)",
                                       WiFi.localIP().toString().c_str(),
                                       WiFi.getHostname(),
                                       emsesp::Helpers::render_value(result, ((double)(WiFi.getTxPower()) / 4), 1));
@@ -328,16 +328,18 @@ void NetworkSettingsService::WiFiEvent(WiFiEvent_t event, WiFiEventInfo_t info)
         break;
 
     case ARDUINO_EVENT_ETH_GOT_IP:
-        // prevent double calls
+        // prevent double calls to mDNS
         if (!emsesp::EMSESP::system_.ethernet_connected()) {
-            emsesp::EMSESP::logger().info("Ethernet connected (IP=%s, speed %d Mbps)", ETH.localIP().toString().c_str(), ETH.linkSpeed());
+            emsesp::EMSESP::logger().info("Ethernet connected (Local IP=%s, speed %d Mbps)", ETH.localIP().toString().c_str(), ETH.linkSpeed());
             emsesp::EMSESP::system_.ethernet_connected(true);
             mDNS_start();
         }
         break;
 
     case ARDUINO_EVENT_ETH_DISCONNECTED:
-        emsesp::EMSESP::logger().warning("Ethernet disconnected");
+        emsesp::EMSESP::logger().warning("Ethernet disconnected. Reason: %s (%d)",
+                                         disconnectReason(info.wifi_sta_disconnected.reason),
+                                         info.wifi_sta_disconnected.reason);
         emsesp::EMSESP::system_.ethernet_connected(false);
         emsesp::EMSESP::system_.has_ipv6(false);
         break;
@@ -364,27 +366,24 @@ void NetworkSettingsService::WiFiEvent(WiFiEvent_t event, WiFiEventInfo_t info)
 #endif
         break;
 
-        // IPv6 specific
+        // IPv6 specific - WiFi
     case ARDUINO_EVENT_WIFI_STA_GOT_IP6:
 #if !TASMOTA_SDK && ESP_IDF_VERSION_MAJOR < 5
         emsesp::EMSESP::logger().info("Local IPv6 (WiFi)=%s", WiFi.localIPv6().toString().c_str());
 #else
-        emsesp::EMSESP::logger().info("Local IPv6=%s", IPAddress(IPv6, (uint8_t *)info.got_ip6.ip6_info.ip.addr, 0).toString().c_str());
+        emsesp::EMSESP::logger().info("IPv6 (WiFi)=%s", IPAddress(IPv6, (uint8_t *)info.got_ip6.ip6_info.ip.addr, 0).toString().c_str());
 #endif
         emsesp::EMSESP::system_.has_ipv6(true);
         break;
 
-        // IPv6 specific
+        // IPv6 specific - Eth
-        // This a bug in arduino where this is triggered twice, so we prevent it
     case ARDUINO_EVENT_ETH_GOT_IP6:
-        if (!emsesp::EMSESP::system_.has_ipv6()) {
 #if !TASMOTA_SDK && ESP_IDF_VERSION_MAJOR < 5
-            emsesp::EMSESP::logger().info("Local IPv6 (Eth)=%s", ETH.localIPv6().toString().c_str());
+        emsesp::EMSESP::logger().info("Local IPv6 (Eth)=%s", ETH.localIPv6().toString().c_str());
 #else
-            emsesp::EMSESP::logger().info("Local IPv6=%s", IPAddress(IPv6, (uint8_t *)info.got_ip6.ip6_info.ip.addr, 0).toString().c_str());
+        emsesp::EMSESP::logger().info("IPv6 (Eth)=%s", IPAddress(IPv6, (uint8_t *)info.got_ip6.ip6_info.ip.addr, 0).toString().c_str());
 #endif
-            emsesp::EMSESP::system_.has_ipv6(true);
+        emsesp::EMSESP::system_.has_ipv6(true);
-        }
         break;
 
     default:

scripts/rename_fw.py

@@ -20,6 +20,8 @@ def bin_copy(source, target, env):
 
     app_version = bag.get('app_version')
     
+    # print(env.Dump())
+
     # get the chip type, in uppercase
     mcu = env.get('BOARD_MCU').upper()
     # alternatively take platform from the pio target
@@ -40,11 +42,6 @@ def bin_copy(source, target, env):
     if parts[index].endswith("M"):
         flash_mem = parts[index] + "B"
     
-    # print(env.Dump())
-    
-    # my_flags = env.ParseFlags(env['BUILD_FLAGS'])
-    # defines = {k: v for (k, v) in my_flags.get("CPPDEFINES")}
-    
     # find if BOARD_HAS_PSRAM is in the cppdefines
     cppdefines = env.get("CPPDEFINES")
     if 'BOARD_HAS_PSRAM' in cppdefines:
@@ -99,6 +96,28 @@ def bin_copy(source, target, env):
         file1.write(result.hexdigest())
         file1.close()
         
+    # make a copy using the old 3.6.x filename format for backwards compatibility with the WebUI version check, e.g.
+    #  EMS-ESP-3_6_5-ESP32_S3.bin (16MB) with target ci_s3_16M_P
+    #  EMS-ESP-3_6_5-ESP32.bin (4MB) with target ci_s_4M
+    extra_variant = ""
+    if env.get('PIOENV') == "ci_s3_16M_P":
+        extra_variant = "EMS-ESP-" + app_version.replace(".", "_") + "-ESP32_S3"
+    elif env.get('PIOENV') == "ci_s_4M":
+        extra_variant = "EMS-ESP-" + app_version.replace(".", "_") + "-ESP32"
+      
+    if extra_variant:        
+        extra_bin_file = "{}firmware{}{}.bin".format(OUTPUT_DIR, os.path.sep, extra_variant)
+        if os.path.isfile(extra_bin_file):
+            os.remove(extra_bin_file)
+            
+        extra_md5_file = "{}firmware{}{}.md5".format(OUTPUT_DIR, os.path.sep, extra_variant)
+        if os.path.isfile(extra_md5_file):
+            os.remove(extra_md5_file)   
+        
+        shutil.copy(bin_file, extra_bin_file)
+        shutil.copy(md5_file, extra_md5_file)
+        print("Filename copy for 3.6.x: "+extra_bin_file)
+    
     print("*********************************************")
 
 env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", [bin_copy])

src/console.cpp

@@ -686,7 +686,7 @@ void EMSESPShell::end_of_transmission() {
 
 void EMSESPShell::main_help_function(Shell & shell, const std::vector<std::string> & arguments) {
     shell.println();
-#ifndef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
     shell.printfln("%s%sEMS-ESP version %s%s", COLOR_BRIGHT_GREEN, COLOR_BOLD_ON, EMSESP_APP_VERSION, COLOR_RESET);
 #else
     shell.printfln("%s%sEMS-ESP version %s%s (DEBUG)", COLOR_BRIGHT_GREEN, COLOR_BOLD_ON, EMSESP_APP_VERSION, COLOR_RESET);

src/emsdevice.cpp

@@ -1682,8 +1682,8 @@ bool EMSdevice::generate_values(JsonObject output, const int8_t tag_filter, cons
 
             // do not overwrite
             if (json.containsKey(name)) {
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
-                EMSESP::logger().debug("double json key: %s", name);
+                EMSESP::logger().debug("warning: double json key: %s", name);
 #endif
                 continue;
             }

src/system.cpp

@@ -788,6 +788,13 @@ void System::network_init(bool refresh) {
     //  ETH_CLOCK_GPIO17_OUT = 3  RMII clock output from GPIO17, for 50hz inverted clock
     auto clock_mode = (eth_clock_mode_t)eth_clock_mode_;
 
+    // reset power and add a delay as ETH doesn't not always start up correctly after a warm boot
+    // TODO still experimental
+    pinMode(eth_power_, OUTPUT);
+    digitalWrite(eth_power_, LOW);
+    delay(1000);
+    digitalWrite(eth_power_, HIGH);
+
 #if ESP_IDF_VERSION_MAJOR < 5
     eth_present_ = ETH.begin(phy_addr, power, mdc, mdio, type, clock_mode);
 #else
@@ -1791,7 +1798,7 @@ bool System::load_board_profile(std::vector<int8_t> & data, const std::string &
 bool System::command_format(const char * value, const int8_t id) {
     LOG_INFO("Removing all config files");
 #ifndef EMSESP_STANDALONE
-    // TODO To replaced with fs.rmdir(FS_CONFIG_DIRECTORY) now we're using IDF 4.2+
+    // TODO To replaced with LittleFS.rmdir(FS_CONFIG_DIRECTORY) now we're using IDF 4.2+
     File root = LittleFS.open(EMSESP_FS_CONFIG_DIRECTORY);
     File file;
     while ((file = root.openNextFile())) {

src/telegram.h

@@ -132,19 +132,6 @@ class Telegram {
         // check for out of bounds, if so don't modify the value
         auto msg_size = (index - this->offset + num_bytes - 1);
 
-#ifdef EMSESP_DEBUG
-// Serial.print(" index: ");
-// Serial.print(index);
-// Serial.print(" offset: ");
-// Serial.print(offset);
-// Serial.print(" index: ");
-// Serial.print(" message_length: ");
-// Serial.print(this->message_length);
-// Serial.print(" msg_size: ");
-// Serial.print(msg_size);
-// Serial.println();
-#endif
-
         if ((index < this->offset) || (msg_size >= this->message_length) || (msg_size > EMS_MAX_TELEGRAM_MESSAGE_LENGTH)) {
             return false;
         }
@@ -443,13 +430,8 @@ class TxService : public EMSbus {
         return tx_telegrams_.empty();
     }
 
-#if defined(EMSESP_DEBUG)
+    static constexpr uint8_t  MAXIMUM_TX_RETRIES = 3;
-    static constexpr uint8_t MAXIMUM_TX_RETRIES = 0; // when compiled with EMSESP_DEBUG don't retry
+    static constexpr uint32_t POST_SEND_DELAY    = 2000;
-#else
-    static constexpr uint8_t MAXIMUM_TX_RETRIES = 3;
-#endif
-
-    static constexpr uint32_t POST_SEND_DELAY = 2000;
 
   private:
     std::deque<QueuedTxTelegram> tx_telegrams_; // the Tx queue

src/version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.7.0-dev.37"
+#define EMSESP_APP_VERSION "3.7.0-dev.38"

src/web/WebSettingsService.cpp

@@ -93,7 +93,7 @@ StateUpdateResult WebSettings::update(JsonObject root, WebSettings & settings) {
     bool psram = false;
 #endif
 
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
     EMSESP::logger().debug("NVS boot value=[%s], board profile=[%s], EMSESP_DEFAULT_BOARD_PROFILE=[%s]",
                            EMSESP::nvs_.getString("boot").c_str(),
                            root["board_profile"] | "",
@@ -111,7 +111,7 @@ StateUpdateResult WebSettings::update(JsonObject root, WebSettings & settings) {
     if (!EMSESP::system_.getBBQKeesGatewayDetails().isEmpty()) {
         String nvs_boot = EMSESP::nvs_.getString("boot");
         if (!nvs_boot.isEmpty()) {
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
             EMSESP::logger().debug("Overriding board profile with NVS boot value %s");
 #endif
             settings.board_profile = nvs_boot;
@@ -145,7 +145,7 @@ StateUpdateResult WebSettings::update(JsonObject root, WebSettings & settings) {
 
     // still don't have a valid board profile. Let's see if we can determine one
     if (settings.board_profile == "default") {
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
         EMSESP::logger().debug("Trying to detect board and set board profile...");
 #endif
 

src/web/WebStatusService.cpp

@@ -82,7 +82,7 @@ void WebStatusService::systemStatus(AsyncWebServerRequest * request) {
 #endif
     }
 
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
 #ifdef EMSESP_TEST
     root["build_flags"] = "DEBUG,TEST";
 #else
@@ -155,7 +155,7 @@ void WebStatusService::checkUpgrade(AsyncWebServerRequest * request, JsonVariant
     const std::string          latest_version = json["version"] | EMSESP_APP_VERSION;
     version::Semver200_version this_version(latest_version);
 
-#ifdef EMSESP_DEBUG
+#if defined(EMSESP_DEBUG)
     emsesp::EMSESP::logger().debug("Checking for upgrade: %s > %s", EMSESP_APP_VERSION, latest_version.c_str());
 #endif