andry/EMS-ESP32
1
0
Fork 0
mirror of https://github.com/emsesp/EMS-ESP32.git synced 2026-10-28 11:14:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/custom-sdkconfig' into dev

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
proddy
2026-10-10 12:34:51 +02:00
co-authored by Cursor
parent c3894bab5c 04d373b93c
commit e10983cca1
31 changed files with 518 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dev_release.yml
+205 -26
View File
@@ -11,31 +11,210 @@ on:
permissions:
contents: write
# Parallel graph (wall-clock is ~max of the firmware jobs, not the sum):
# webui ──────┐
# ├─► firmware (s_4M / s_16M / s_16M_P / s3_16M_P) ─┐
# modbus ─────┤ ├─► release
# └─► standalone ───────────────────────────────────┘
#
# Extra -j on a single runner does not help: PlatformIO already compiles each
# environment with all CPUs. The win is separate GitHub runners.
jobs:
pre-release:
name: 'Build Dev Release'
webui:
name: Build webUI
if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
python-version: '3.13'
package_json_file: interface/package.json
- name: Install Node.js 26
uses: actions/setup-node@v7
with:
node-version: 26
- name: Cache pnpm store
uses: actions/cache@v4
with:
path: ~/.local/share/pnpm/store
key: ${{ runner.os }}-pnpm-${{ hashFiles('interface/package.json') }}
restore-keys: |
${{ runner.os }}-pnpm-
- name: Build webUI
env:
CI: true
working-directory: interface
run: |
pnpm install
pnpm build-webUI
- name: Upload WWWData.h
uses: actions/upload-artifact@v4
with:
name: wwwdata
path: src/ESP32React/WWWData.h
if-no-files-found: error
modbus:
name: Build modbus
if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "latest"
enable-cache: false
- name: Install pioarduino
run: uv pip install --system pioarduino==6.2.0
- name: Build modbus
run: platformio run -e build_modbus
- name: Upload modbus header
uses: actions/upload-artifact@v4
with:
name: modbus-header
path: src/core/modbus_entity_parameters.hpp
if-no-files-found: error
standalone:
name: Build standalone
needs: modbus
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "latest"
enable-cache: false
- name: Install pioarduino
run: uv pip install --system pioarduino==6.2.0
- name: Restore generated modbus header
uses: actions/download-artifact@v4
with:
name: modbus-header
path: src/core
- name: Build standalone
run: platformio run -e build_standalone
- name: Upload generated docs
uses: actions/upload-artifact@v4
with:
name: generated-docs
path: |
docs/dump_entities.csv
docs/dump_telegrams.csv
docs/Modbus-Entity-Registers.md
if-no-files-found: error
firmware:
name: Firmware ${{ matrix.env }}
needs: [webui, modbus]
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
# Keep in sync with platformio.ini default_envs
env: [s_4M, s_16M, s_16M_P, s3_16M_P]
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "latest"
enable-cache: false
- name: Cache PlatformIO packages
uses: actions/cache@v4
with:
path: |
~/.platformio/.cache
~/.platformio/packages
~/.platformio/platforms
key: ${{ runner.os }}-pio-${{ matrix.env }}-${{ hashFiles('platformio.ini') }}
restore-keys: |
${{ runner.os }}-pio-${{ matrix.env }}-
${{ runner.os }}-pio-
- name: Install pioarduino
run: uv pip install --system pioarduino==6.2.0 intelhex
- name: Restore WWWData.h
uses: actions/download-artifact@v4
with:
name: wwwdata
path: src/ESP32React
- name: Restore generated modbus header
uses: actions/download-artifact@v4
with:
name: modbus-header
path: src/core
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: firmware-${{ matrix.env }}
restore-keys: firmware-${{ matrix.env }}-
max-size: 1G
- name: Build ${{ matrix.env }}
env:
CCACHE_COMPILERCHECK: content
run: platformio run -e ${{ matrix.env }}
- name: Upload firmware
uses: actions/upload-artifact@v4
with:
name: firmware-${{ matrix.env }}
path: build/firmware
if-no-files-found: error
release:
name: Create GitHub Release
needs: [firmware, standalone]
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Install pnpm
run: npm install -g pnpm
- name: Get the EMS-ESP version
id: build_info
run: |
@@ -49,27 +228,27 @@ jobs:
prev=`git rev-parse -q --verify 'refs/tags/latest^{commit}' || git describe --tags --abbrev=0 --match 'v*'`
echo "REF=$prev" >> $GITHUB_OUTPUT
- name: Install PlatformIO
run: |
python -m pip install --upgrade pip
pip install -U platformio
python -m pip install intelhex
- name: Restore generated modbus header
uses: actions/download-artifact@v4
with:
name: modbus-header
path: src/core
- name: Build webUI
run: |
platformio run -e build-webUI
- name: Restore generated docs
uses: actions/download-artifact@v4
with:
name: generated-docs
path: .
- name: Build modbus
run: |
platformio run -e build_modbus
- name: Restore firmware binaries
uses: actions/download-artifact@v4
with:
pattern: firmware-*
merge-multiple: true
path: build/firmware
- name: Build standalone
run: |
platformio run -e build_standalone
- name: Build all PIO target environments, from default_envs
run: |
platformio run
- name: List firmware files
run: ls -la build/firmware
- name: Build the recovery firmware for the 16MB boards
run: |
CHANGELOG_LATEST.md
+8
View File
@@ -25,6 +25,9 @@ This release is based on the latest Espressif/Arduino core version 3. It brings
- on 16MB boards with the Recovery installed, EMS-ESP switches to it after crashing 5 times in a row. The Recovery shows why it was started and which firmware failed to start
- the Version page shows the installed Recovery version on 16MB boards
- crash test builds to try out the rollback and recovery, see `[env:crashtest_s3_16M_P]` in `platformio.ini`
- `s_16M_P_custom` PlatformIO env that HybridCompiles Arduino/IDF with a trimmed sdkconfig (no Bluetooth, PPP/modem, IP forward/NAPT, ULP, SPI Ethernet, GDBSTUB, FreeRTOS trace, mDNS browse/CLI) and places the UART ISR in IRAM
- 16MB builds use EMS-ESP's own Arduino/IDF libs from [esp32-arduino-lib-builder](https://github.com/emsesp/esp32-arduino-lib-builder) (`0910-1604-5.5.5`) instead of Tasmota's prebuilt cores
- optional `ccache` wrapping when it is installed (`scripts/enable_ccache.py`)
## Fixed
@@ -50,6 +53,9 @@ This release is based on the latest Espressif/Arduino core version 3. It brings
- add 0kW to heat limits [#3262](https://github.com/emsesp/EMS-ESP32/issues/3262), [#3265](https://github.com/emsesp/EMS-ESP32/issues/3265)
- show temperature value for sensor 0x40 [#3272](https://github.com/emsesp/EMS-ESP32/issues/3272)
- no EMS bus on custom boards whose interface relies on a pull-up on the Rx line. The newer ESP-IDF no longer enables the internal pull-up on the UART Rx pin, so it is now set explicitly as it was in 3.8 [#3258](https://github.com/emsesp/EMS-ESP32/discussions/3258)
- Guru Meditation (cache error) on HybridCompile builds when the EMS or console UART ISR ran from flash during a SPI flash read (WiFi power-down, mDNS, NVS). The UART ISR and ringbuf ISR helpers are now in IRAM
- compile errors against Arduino 3.3.12 from Tasmota/Arduino headers named `console.h`, `network.h` and `httpClient.h`. Those EMS-ESP headers are now `emsesp_console.h`, `emsesp_network.h` and `emsesp_http_client.h`
- HTTP OTA from WSL/mDNS stalled on IPv6 and used tiny write chunks. Uploads now resolve IPv4 and stream in 64 KB blocks (`scripts/upload.py`, `scripts/upload_cli.py`)
## Changed
@@ -68,3 +74,5 @@ This release is based on the latest Espressif/Arduino core version 3. It brings
- failed versions.json fetches back off from 5 up to 160 minutes instead of retrying every 5 minutes, as each attempt blocks the main loop for as long as the DNS, connect and read timeouts allow
- optimized the LED library to support WS2812B V5 and the newer V6 RGB LEDs
- hold Tx while a K30RF/KM200 gateway is syncing [#3094](https://github.com/emsesp/EMS-ESP32/issues/3094)
- Tasmota platform 2026.05.50 (Arduino 3.3.8 / IDF 5.5.4) to 2026.09.50 (Arduino 3.3.12 / IDF 5.5.5)
- CI dev-release firmware jobs run in parallel (webUI, firmware targets, standalone) instead of one serial job
boards/s3_32M_P.json
+1 -1
View File
@@ -21,7 +21,7 @@
"arduino",
"espidf"
],
"name": "Tasmota ESP32-S3 32M Flash OPI PSRAM, 4608KB Code/OTA, 2MB FS",
"name": "ESP32-S3 32M Flash OPI PSRAM, 4608KB Code/OTA, 2MB FS",
"upload": {
"flash_size": "32MB",
"maximum_ram_size": 327680,
boards/s_16M.json
+1 -1
View File
@@ -18,7 +18,7 @@
"arduino",
"espidf"
],
"name": "Tasmota ESP32 16M Flash, 4608KB Code/OTA, 2MB FS",
"name": "ESP32 16M Flash, 4608KB Code/OTA, 2MB FS",
"upload": {
"flash_size": "16MB",
"maximum_ram_size": 327680,
boards/s_16M_P.json
+1 -1
View File
@@ -19,7 +19,7 @@
"arduino",
"espidf"
],
"name": "Tasmota ESP32 16M Flash DIO PSRAM, 4608KB Code/OTA, 2MB FS",
"name": "ESP32 16M Flash DIO PSRAM, 4608KB Code/OTA, 2MB FS",
"upload": {
"flash_size": "16MB",
"maximum_ram_size": 327680,
boards/s_4M.json
+1 -1
View File
@@ -17,7 +17,7 @@
"arduino",
"espidf"
],
"name": "Tasmota ESP32 4M Flash, 4608KB Code/OTA, 2MB FS",
"name": "ESP32 4M Flash, 4608KB Code/OTA, 2MB FS",
"upload": {
"flash_size": "4MB",
"maximum_ram_size": 327680,
mock-api/restServer.ts
+3 -3
View File
@@ -103,7 +103,7 @@ let system_status = {
cpu_cores: 2,
cpu_freq_mhz: 240,
max_alloc_heap: 191,
arduino_version: 'Tasmota Arduino v3.3.8',
arduino_version: 'Arduino v3.3.8',
sdk_version: '5.5.4.260407',
partition: 'app0',
flash_chip_size: 16384,
@@ -210,7 +210,7 @@ switch (emulate_esp) {
case 'ESP32':
system_status.esp_platform = 'ESP32';
system_status.cpu_type = 'ESP32-D0WD-V3';
system_status.arduino_version = 'Tasmota Arduino v3.3.8';
system_status.arduino_version = 'Arduino v3.3.8';
system_status.sdk_version = '5.5.4.260407';
system_status.psram = true;
system_status.psram_size = 4096;
@@ -226,7 +226,7 @@ switch (emulate_esp) {
default:
system_status.esp_platform = 'ESP32S3';
system_status.cpu_type = 'ESP32-S3';
system_status.arduino_version = 'Tasmota Arduino v3.3.8';
system_status.arduino_version = 'Arduino v3.3.8';
system_status.sdk_version = '5.5.4.260407';
system_status.psram = true;
system_status.psram_size = 8189;
platformio.ini
+41 -8
View File
@@ -21,8 +21,8 @@ extra_configs =
pio_local.ini
[common]
core_build_flags = -std=gnu++20 -O3 -flto=auto -Wno-type-limits -Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -Wno-format -Wno-missing-field-initializers
core_unbuild_flags = -std=gnu++11 -std=gnu++14 -std=gnu++17 -fno-lto
core_build_flags = -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-unused-variable -Wno-format -Wno-missing-field-initializers
core_unbuild_flags = -O2 -O3 -Og -fno-lto
my_build_flags =
@@ -31,6 +31,9 @@ build_flags =
${factory_settings.build_flags}
${common.my_build_flags}
-I./src/core
; core 3.3.12 uses spi_host_device_t in ETH.h without including it. A framework
; refresh restores that header, so force the declaration on every compile.
-include hal/spi_types.h
-D ARDUINO_ARCH_ESP32
-D ONEWIRE_CRC16=0
-D CONFIG_UART_ISR_IN_IRAM
@@ -60,15 +63,18 @@ unbuild_flags =
[espressif32_base_4M]
framework = arduino
board_build.partitions = partitions/esp32_partition_4M.csv
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.05.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.8 based on IDF 5.5.4.260407
; platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5
board_upload.flash_size = 4MB
board_build.app_partition_name = app0
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Arduino Core 3.3.12 based on IDF 5.5.5
; 16MB Flash variants
[espressif32_base_16M]
framework = arduino
board_build.partitions = partitions/esp32_partition_16M.csv
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.05.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.8 based on IDF 5.5.4.260407
; platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5
board_upload.flash_size = 16MB
board_build.app_partition_name = app0
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Arduino Core 3.3.12 based on IDF 5.5.5
platform_packages = framework-arduinoespressif32 @ https://github.com/emsesp/esp32-arduino-lib-builder/releases/download/0910-1604-5.5.5/framework-arduinoespressif32-v5.5.5-a63b839f.tar.xz
build_flags =
${common.build_flags}
-D EMSESP_HAS_RECOVERY ; installs and falls back to the recovery firmware in the factory partition
@@ -78,8 +84,9 @@ build_flags =
[espressif32_base_32M]
framework = arduino
board_build.partitions = partitions/esp32_partition_32M.csv
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.05.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.8 based on IDF 5.5.4.260407
; platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5
board_upload.flash_size = 32MB
board_build.app_partition_name = app0
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Arduino Core 3.3.12 based on IDF 5.5.5
[env]
build_flags =
@@ -87,6 +94,7 @@ build_flags =
build_unflags =
${common.unbuild_flags}
extra_scripts =
pre:scripts/enable_ccache.py ; uses ccache when it is installed
post:scripts/rename_fw.py ; renames the firmware .bin file
monitor_speed = 115200
monitor_filters = direct, esp32_exception_decoder
@@ -129,6 +137,31 @@ board = s_16M
extends = espressif32_base_16M
board = s_16M_P
[env:s_16M_P_custom]
; same as s_16M_P, but the Arduino/IDF libs are rebuilt with HybridCompile using the custom_sdkconfig below
extends = env:s_16M_P
custom_sdkconfig =
'# CONFIG_BT_ENABLED is not set'
'# CONFIG_LWIP_PPP_SUPPORT is not set'
'# CONFIG_LWIP_IP_FORWARD is not set'
'# CONFIG_LWIP_IPV4_NAPT is not set'
'# CONFIG_LWIP_IPV4_NAPT_PORTMAP is not set'
'# CONFIG_ULP_COPROC_ENABLED is not set'
'# CONFIG_ETH_USE_SPI_ETHERNET is not set'
'# CONFIG_ETH_SPI_ETHERNET_W5500 is not set'
'# CONFIG_ETH_SPI_ETHERNET_DM9051 is not set'
'# CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL is not set'
'# CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS is not set'
'# CONFIG_FREERTOS_USE_TRACE_FACILITY is not set'
'# CONFIG_FREERTOS_ENABLE_TASK_SNAPSHOT is not set'
'# CONFIG_ESP_MODEM_USE_PPP_MODE is not set'
'# CONFIG_ESP_GDBSTUB_ENABLED is not set'
'# CONFIG_MDNS_ENABLE_CONSOLE_CLI is not set'
'# CONFIG_MDNS_ENABLE_BROWSE is not set'
'# CONFIG_I2S_ISR_IRAM_SAFE is not set'
'# CONFIG_RINGBUF_PLACE_ISR_FUNCTIONS_INTO_FLASH is not set'
'CONFIG_UART_ISR_IN_IRAM=y'
[env:s3_16M_P]
; 16MB ESP32-S3 - with PSRAM - like a BBQKees S3
extends = espressif32_base_16M
scripts/enable_ccache.py
+27
View File
@@ -0,0 +1,27 @@
# Wrap the C/C++ compilers with ccache when it is on PATH.
# No-op locally unless ccache is installed; CI installs it via ccache-action.
Import("env")
import shutil
ccache = shutil.which("ccache")
if ccache:
def wrap(tool):
cmd = env.get(tool)
if not cmd:
return
resolved = env.subst(cmd) if isinstance(cmd, str) else cmd
if isinstance(resolved, str):
if resolved.startswith("ccache"):
return
env[tool] = f"{ccache} {resolved}"
return
if isinstance(resolved, (list, tuple)):
if resolved and "ccache" in str(resolved[0]):
return
env[tool] = [ccache, *resolved]
wrap("CC")
wrap("CXX")
print("ccache enabled")
scripts/upload.py
+92 -30
View File
@@ -14,12 +14,17 @@
# and
# extra_scripts = scripts/upload.py
#
# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency
# and can stall on IPv6 before falling back to IPv4.
import hashlib
import os
import socket
import time
from urllib.parse import urlparse
import requests
import hashlib
from urllib.parse import urlparse
import time
import os
from requests.adapters import HTTPAdapter
Import("env")
@@ -35,6 +40,10 @@ except ImportError:
from tqdm import tqdm
from termcolor import cprint
# http.client/urllib3 default is 8–16 KiB, which starves a LAN TCP window.
UPLOAD_BLOCK_SIZE = 64 * 1024
UPLOAD_SNDBUF = 256 * 1024
def print_success(x):
cprint(x, 'green')
@@ -44,6 +53,43 @@ def print_fail(x):
cprint(f'Error: {x}', 'red')
def resolve_ipv4(host):
"""Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request."""
if host.startswith('['):
return host
if ':' in host and host.count(':') == 1:
host = host.rsplit(':', 1)[0]
try:
socket.inet_pton(socket.AF_INET, host)
return host
except OSError:
pass
infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM)
if not infos:
raise socket.gaierror(f'No IPv4 address for {host}')
return infos[0][4][0]
class FastHTTPAdapter(HTTPAdapter):
"""Larger write chunks and socket send buffer for streaming POSTs."""
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
socket_options = [
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
(socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF),
]
pool_kwargs.setdefault('socket_options', socket_options)
pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE)
super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
def make_session():
session = requests.Session()
adapter = FastHTTPAdapter(pool_connections=2, pool_maxsize=2)
session.mount('http://', adapter)
return session
def build_headers(host_ip, emsesp_url, content_type='application/json', access_token=None, extra_headers=None):
"""Build common HTTP headers with optional overrides."""
headers = {
@@ -51,7 +97,6 @@ def build_headers(host_ip, emsesp_url, content_type='application/json', access_t
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0',
'Accept': '*/*',
'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip, deflate',
'Referer': emsesp_url,
'Content-Type': content_type,
'Connection': 'keep-alive'
@@ -79,24 +124,33 @@ def on_upload(source, target, env):
username = env.GetProjectOption('custom_username')
password = env.GetProjectOption('custom_password')
emsesp_ip = env.GetProjectOption('custom_emsesp_ip')
except Exception as e:
except Exception:
print_fail(f'Missing settings. Add these to your pio_local.ini file:\n\ncustom_username=username\ncustom_password=password\ncustom_emsesp_ip=ems-esp.local\n')
return
emsesp_url = f"http://{emsesp_ip}"
parsed_url = urlparse(emsesp_url)
host_ip = parsed_url.netloc
parsed_url = urlparse(f"http://{emsesp_ip}")
host_name = parsed_url.hostname or emsesp_ip
try:
host_ip = resolve_ipv4(host_name)
except OSError as e:
print_fail(f"Could not resolve {host_name} to IPv4: {e}")
return
if host_ip != host_name:
print_success(f"Resolved {host_name} -> {host_ip}")
emsesp_url = f"http://{host_ip}"
session = make_session()
try:
signon_url = f"{emsesp_url}/rest/signIn"
signon_headers = build_headers(host_ip, emsesp_url)
username_password = {
"username": username,
"password": password
}
response = requests.post(
signon_url, json=username_password, headers=signon_headers)
response = session.post(
signon_url,
json={"username": username, "password": password},
headers=signon_headers,
timeout=15)
if response.status_code != 200:
print_fail("Authentication with EMS-ESP failed (code " +
@@ -106,17 +160,16 @@ def on_upload(source, target, env):
print_success("Authentication with EMS-ESP successful")
access_token = response.json().get('access_token')
# start the upload
firmware_path = str(source[0])
with open(firmware_path, 'rb') as firmware_file:
firmware = firmware_file.read()
with open(firmware_path, 'rb') as firmware:
md5 = hashlib.md5(firmware.read()).hexdigest()
firmware.seek(0)
md5 = hashlib.md5(firmware).hexdigest()
filename = os.path.basename(firmware_path)
encoder = MultipartEncoder(fields={
'MD5': md5,
'file': (firmware_path, firmware, 'application/octet-stream')}
'file': (filename, firmware, 'application/octet-stream')}
)
bar = tqdm(desc='Upload Progress',
@@ -124,7 +177,8 @@ def on_upload(source, target, env):
dynamic_ncols=True,
unit='B',
unit_scale=True,
unit_divisor=1024
unit_divisor=1024,
mininterval=0.2
)
monitor = MultipartEncoderMonitor(
@@ -142,27 +196,35 @@ def on_upload(source, target, env):
)
upload_url = f"{emsesp_url}/rest/uploadFile"
response = requests.post(
upload_url, data=monitor, headers=post_headers)
started = time.perf_counter()
response = session.post(
upload_url, data=monitor, headers=post_headers, timeout=None)
elapsed = time.perf_counter() - started
bar.close()
time.sleep(0.1)
print()
if response.status_code != 200:
print_fail(f"Upload failed (code {response.status_code}).")
else:
print_success("Upload successful. Rebooting device.")
kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0
print_success(
f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.")
restart_headers = build_headers(
host_ip, emsesp_url, access_token=access_token)
restart_url = f"{emsesp_url}/api/system/restart"
response = requests.get(restart_url, headers=restart_headers)
try:
response = session.get(
restart_url, headers=restart_headers, timeout=10)
if response.status_code != 200:
print_fail(f"Restart failed (code {response.status_code})")
except requests.RequestException:
# Device often drops the connection as it reboots — that's OK.
pass
print()
finally:
session.close()
if env.get('UPLOAD_PROTOCOL') == 'custom':
scripts/upload_cli.py
+70 -35
View File
@@ -9,21 +9,27 @@
#
# Run using for example:
# python3 upload_cli.py -i 10.10.10.175 -f ../build/firmware/EMS-ESP-3_7_0-dev_34-ESP32S3-16MB+.bin
#
# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency
# and can stall on IPv6 before falling back to IPv4.
import argparse
import hashlib
import socket
import sys
import time
from pathlib import Path
from urllib.parse import urlparse
import requests
from requests.adapters import HTTPAdapter
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
from termcolor import cprint
from tqdm import tqdm
# Constants
USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0'
CHUNK_SIZE = 8192 # 8KB chunks for MD5 calculation
UPLOAD_BLOCK_SIZE = 64 * 1024
UPLOAD_SNDBUF = 256 * 1024
def print_success(x):
@@ -34,13 +40,34 @@ def print_fail(x):
return cprint(x, 'red')
def calculate_md5(file_path):
"""Calculate MD5 hash of a file in chunks for memory efficiency."""
md5_hash = hashlib.md5()
with open(file_path, 'rb') as f:
for chunk in iter(lambda: f.read(CHUNK_SIZE), b''):
md5_hash.update(chunk)
return md5_hash.hexdigest()
def resolve_ipv4(host):
"""Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request."""
if host.startswith('['):
return host
if ':' in host and host.count(':') == 1:
host = host.rsplit(':', 1)[0]
try:
socket.inet_pton(socket.AF_INET, host)
return host
except OSError:
pass
infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM)
if not infos:
raise socket.gaierror(f'No IPv4 address for {host}')
return infos[0][4][0]
class FastHTTPAdapter(HTTPAdapter):
"""Larger write chunks and socket send buffer for streaming POSTs."""
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
socket_options = [
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
(socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF),
]
pool_kwargs.setdefault('socket_options', socket_options)
pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE)
super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
def create_base_headers(host_ip, emsesp_url):
@@ -50,7 +77,6 @@ def create_base_headers(host_ip, emsesp_url):
'User-Agent': USER_AGENT,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip, deflate',
'Referer': emsesp_url,
'Connection': 'keep-alive'
}
@@ -58,35 +84,39 @@ def create_base_headers(host_ip, emsesp_url):
def upload(file, ip, username, password):
"""Upload firmware to EMS-ESP device."""
# Print welcome message
print()
print("EMS-ESP Firmware Upload")
# Validate file exists
file_path = Path(file)
if not file_path.exists():
print_fail(f"File not found: {file}")
return
# Setup URLs and headers
emsesp_url = f"http://{ip}"
host_ip = ip
parsed_url = urlparse(f"http://{ip}")
host_name = parsed_url.hostname or ip
try:
host_ip = resolve_ipv4(host_name)
except OSError as e:
print_fail(f"Could not resolve {host_name} to IPv4: {e}")
return
# Use a session for connection pooling and persistence
if host_ip != host_name:
print_success(f"Resolved {host_name} -> {host_ip}")
emsesp_url = f"http://{host_ip}"
session = requests.Session()
session.mount('http://', FastHTTPAdapter(pool_connections=2, pool_maxsize=2))
try:
# Authentication
signon_url = f"{emsesp_url}/rest/signIn"
signon_headers = create_base_headers(host_ip, emsesp_url)
signon_headers['Content-Type'] = 'application/json'
username_password = {
"username": username,
"password": password
}
response = session.post(signon_url, json=username_password, headers=signon_headers)
response = session.post(
signon_url,
json={"username": username, "password": password},
headers=signon_headers,
timeout=15)
if response.status_code != 200:
print_fail(f"Authentication failed (code {response.status_code})")
@@ -95,15 +125,12 @@ def upload(file, ip, username, password):
print_success("Authentication successful")
access_token = response.json().get('access_token')
# Calculate MD5 hash
print("Calculating MD5 hash...")
md5 = calculate_md5(file_path)
firmware = file_path.read_bytes()
md5 = hashlib.md5(firmware).hexdigest()
# Start the upload
with open(file_path, 'rb') as firmware:
encoder = MultipartEncoder(fields={
'MD5': md5,
'file': (file, firmware, 'application/octet-stream')
'file': (file_path.name, firmware, 'application/octet-stream')
})
bar = tqdm(
@@ -112,7 +139,8 @@ def upload(file, ip, username, password):
dynamic_ncols=True,
unit='B',
unit_scale=True,
unit_divisor=1024
unit_divisor=1024,
mininterval=0.2
)
monitor = MultipartEncoderMonitor(
@@ -127,24 +155,32 @@ def upload(file, ip, username, password):
})
upload_url = f"{emsesp_url}/rest/uploadFile"
response = session.post(upload_url, data=monitor, headers=post_headers)
started = time.perf_counter()
response = session.post(
upload_url, data=monitor, headers=post_headers, timeout=None)
elapsed = time.perf_counter() - started
bar.close()
time.sleep(0.1)
if response.status_code != 200:
print_fail(f"Upload failed (code {response.status_code})")
else:
print_success("Upload successful. Rebooting device.")
kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0
print_success(
f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.")
restart_headers = create_base_headers(host_ip, emsesp_url)
restart_headers.update({
'Content-Type': 'application/json',
'Authorization': f'Bearer {access_token}'
})
restart_url = f"{emsesp_url}/api/system/restart"
response = session.get(restart_url, headers=restart_headers)
try:
response = session.get(
restart_url, headers=restart_headers, timeout=10)
if response.status_code != 200:
print_fail(f"Restart failed (code {response.status_code})")
except requests.RequestException:
pass
except requests.RequestException as e:
print_fail(f"Network error: {e}")
@@ -161,7 +197,6 @@ def upload(file, ip, username, password):
print()
# main
parser = argparse.ArgumentParser(description="EMS-ESP Firmware Upload")
parser.add_argument("-f", "--file", metavar="FILE",
required=True, type=str, help="firmware file")
src/core/analogsensor.h
+1 -1
View File
@@ -21,7 +21,7 @@
#include "helpers.h"
#include "mqtt.h"
#include "console.h"
#include "emsesp_console.h"
#include <uuid/log.h>
src/core/command.h
+1 -1
View File
@@ -21,7 +21,7 @@
#include <map>
#include "console.h"
#include "emsesp_console.h"
#include <esp32-psram.h>
using uuid::console::Shell;
src/core/console.cpp
+1 -1
View File
@@ -17,7 +17,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "console.h"
#include "emsesp_console.h"
#include "console_stream.h"
#include "emsesp.h"
src/core/console_stream.h
+1 -1
View File
@@ -20,7 +20,7 @@
#include <uuid/console.h>
#include "console.h"
#include "emsesp_console.h"
namespace emsesp {
src/core/default_settings.h
+1 -1
View File
@@ -304,7 +304,7 @@ enum {
#define STRINGIZE(s) #s
#endif
#define ARDUINO_VERSION_STR(major, minor, patch) "Tasmota Arduino v" STRINGIZE(major) "." STRINGIZE(minor) "." STRINGIZE(patch)
#define ARDUINO_VERSION_STR(major, minor, patch) "Arduino v" STRINGIZE(major) "." STRINGIZE(minor) "." STRINGIZE(patch)
#define ARDUINO_VERSION ARDUINO_VERSION_STR(ESP_ARDUINO_VERSION_MAJOR, ESP_ARDUINO_VERSION_MINOR, ESP_ARDUINO_VERSION_PATCH)
#endif
src/core/emsesp.h
+2 -2
View File
@@ -66,10 +66,10 @@
#include "mqtt.h"
#include "modbus.h"
#include "system.h"
#include "network.h"
#include "emsesp_network.h"
#include "temperaturesensor.h"
#include "analogsensor.h"
#include "console.h"
#include "emsesp_console.h"
#include "console_stream.h"
#include "shower.h"
#include "roomcontrol.h"
src/core/console.h → src/core/emsesp_console.h
View File
src/core/httpClient.h → src/core/emsesp_http_client.h
View File
src/core/network.h → src/core/emsesp_network.h
View File
src/core/httpClient.cpp
+1 -1
View File
@@ -18,7 +18,7 @@
#include "emsesp.h"
#include "httpClient.h"
#include "emsesp_http_client.h"
#include <WiFiClient.h>
#include <ESP_SSLClient.h>
src/core/mqtt.h
+1 -1
View File
@@ -23,7 +23,7 @@
#include "helpers.h"
#include "system.h"
#include "console.h"
#include "emsesp_console.h"
#include "command.h"
#include "emsdevicevalue.h"
#include <esp32-psram.h>
src/core/network.cpp
+1 -1
View File
@@ -16,7 +16,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "network.h"
#include "emsesp_network.h"
#include "emsesp.h"
src/core/shuntingYard.cpp
+1 -1
View File
@@ -22,7 +22,7 @@
#include "shuntingYard.h"
#include "httpClient.h"
#include "emsesp_http_client.h"
namespace emsesp {
src/core/system.cpp
+2 -2
View File
@@ -17,7 +17,7 @@
*/
#include "system.h"
#include "network.h"
#include "emsesp_network.h"
#include "emsesp.h" // for send_raw_telegram() command
#ifndef EMSESP_STANDALONE
@@ -34,7 +34,7 @@
#include "firmwareVersion.h"
#include "shuntingYard.h" // for compute() used by the message and sendmail commands
#include "httpClient.h" // for the shared TLS client settings
#include "emsesp_http_client.h" // for the shared TLS client settings
#if defined(EMSESP_TEST)
#include "../test/test.h"
src/core/system.h
+1 -1
View File
@@ -23,7 +23,7 @@
#include <ArduinoJson.h>
#include "helpers.h"
#include "console.h"
#include "emsesp_console.h"
#include "mqtt.h"
#include "telegram.h"
#include "led.h"
src/core/temperaturesensor.h
+1 -1
View File
@@ -23,7 +23,7 @@
#include "helpers.h"
#include "mqtt.h"
#include "console.h"
#include "emsesp_console.h"
#include <uuid/log.h>
src/emsesp_version.h
+1 -1
View File
@@ -1 +1 @@
#define EMSESP_APP_VERSION "3.9.0-dev.17"
#define EMSESP_APP_VERSION "3.9.0-dev.18"
src/test/test.cpp
+1 -1
View File
@@ -458,7 +458,7 @@ void Test::run_test(uuid::console::Shell & shell, const std::string & cmd, const
// S3 has 16MB flash
// EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_0-dev_32-ESP32S3-16MB+.bin"); // S3
// Test for 4MB Tasmota builds
// Test for 4MB builds
// EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_2-dev_8-ESP32-4MB.bin"); // S32
EMSESP::system_.uploadFirmwareURL("https://github.com/emsesp/EMS-ESP32/releases/download/latest/EMS-ESP-3_7_2-dev_8-ESP32-16MB.bin"); // E32
src/web/WebCommandService.cpp
+1 -1
View File
@@ -20,7 +20,7 @@
#include "WebCommandService.h"
#include "shuntingYard.h"
#include "httpClient.h"
#include "emsesp_http_client.h"
namespace emsesp {
src/web/WebStatusService.cpp
+1 -1
View File
@@ -21,7 +21,7 @@
#ifndef EMSESP_STANDALONE
#include <esp_ota_ops.h>
#endif
#include "httpClient.h"
#include "emsesp_http_client.h"
namespace emsesp {