andry/EMS-ESP32
1
0
Fork 0
mirror of https://github.com/emsesp/EMS-ESP32.git synced 2026-10-28 11:14:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

test new esp32-arduino-lib-builder packages

Browse Source
This commit is contained in:
proddy
2026-10-09 17:47:01 +02:00
parent 673c7462f5
commit e3b1eb6c51
7 changed files with 482 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dev_release.yml
+206 -27
View File
@@ -11,30 +11,209 @@ on:
permissions: permissions:
contents: write contents: write
# Parallel graph (wall-clock is ~max of the firmware jobs, not the sum):
# webui ──────┐
# ├─► firmware (s_4M / s_16M / s_16M_P / s3_16M_P) ─┐
# modbus ─────┤ ├─► release
# └─► standalone ───────────────────────────────────┘
#
# Extra -j on a single runner does not help: PlatformIO already compiles each
# environment with all CPUs. The win is separate GitHub runners.
jobs: jobs:
pre-release: webui:
name: 'Build Dev Release' name: Build webUI
if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32' if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32'
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository
- name: Install python 3.13 uses: actions/checkout@v7
uses: actions/setup-python@v7
- name: Install pnpm
uses: pnpm/action-setup@v4
with: with:
python-version: '3.13' package_json_file: interface/package.json
- name: Install Node.js 26 - name: Install Node.js 26
uses: actions/setup-node@v7 uses: actions/setup-node@v7
with: with:
node-version: 26 node-version: 26
- name: Cache pnpm store
uses: actions/cache@v4
with:
path: ~/.local/share/pnpm/store
key: ${{ runner.os }}-pnpm-${{ hashFiles('interface/package.json') }}
restore-keys: |
${{ runner.os }}-pnpm-
- name: Build webUI
env:
CI: true
working-directory: interface
run: |
pnpm install
pnpm build-webUI
- name: Upload WWWData.h
uses: actions/upload-artifact@v4
with:
name: wwwdata
path: src/ESP32React/WWWData.h
if-no-files-found: error
modbus:
name: Build modbus
if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "latest"
enable-cache: false
- name: Install pioarduino
run: uv pip install --system pioarduino==6.2.0
- name: Build modbus
run: platformio run -e build_modbus
- name: Upload modbus header
uses: actions/upload-artifact@v4
with:
name: modbus-header
path: src/core/modbus_entity_parameters.hpp
if-no-files-found: error
standalone:
name: Build standalone
needs: modbus
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "latest"
enable-cache: false
- name: Install pioarduino
run: uv pip install --system pioarduino==6.2.0
- name: Restore generated modbus header
uses: actions/download-artifact@v4
with:
name: modbus-header
path: src/core
- name: Build standalone
run: platformio run -e build_standalone
- name: Upload generated docs
uses: actions/upload-artifact@v4
with:
name: generated-docs
path: |
docs/dump_entities.csv
docs/dump_telegrams.csv
docs/Modbus-Entity-Registers.md
if-no-files-found: error
firmware:
name: Firmware ${{ matrix.env }}
needs: [webui, modbus]
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
# Keep in sync with platformio.ini default_envs
env: [s_4M, s_16M, s_16M_P, s3_16M_P]
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Install python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "latest"
enable-cache: false
- name: Cache PlatformIO packages
uses: actions/cache@v4
with:
path: |
~/.platformio/.cache
~/.platformio/packages
~/.platformio/platforms
key: ${{ runner.os }}-pio-${{ matrix.env }}-${{ hashFiles('platformio.ini') }}
restore-keys: |
${{ runner.os }}-pio-${{ matrix.env }}-
${{ runner.os }}-pio-
- name: Install pioarduino
run: uv pip install --system pioarduino==6.2.0 intelhex
- name: Restore WWWData.h
uses: actions/download-artifact@v4
with:
name: wwwdata
path: src/ESP32React
- name: Restore generated modbus header
uses: actions/download-artifact@v4
with:
name: modbus-header
path: src/core
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: firmware-${{ matrix.env }}
restore-keys: firmware-${{ matrix.env }}-
max-size: 1G
- name: Build ${{ matrix.env }}
env:
CCACHE_COMPILERCHECK: content
run: platformio run -e ${{ matrix.env }}
- name: Upload firmware
uses: actions/upload-artifact@v4
with:
name: firmware-${{ matrix.env }}
path: build/firmware
if-no-files-found: error
release:
name: Create GitHub Release
needs: [firmware, standalone]
runs-on: ubuntu-latest
steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v7 uses: actions/checkout@v7
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Install pnpm
run: npm install -g pnpm
- name: Get the EMS-ESP version - name: Get the EMS-ESP version
id: build_info id: build_info
@@ -49,27 +228,27 @@ jobs:
prev=`git rev-parse -q --verify 'refs/tags/latest^{commit}' || git describe --tags --abbrev=0 --match 'v*'` prev=`git rev-parse -q --verify 'refs/tags/latest^{commit}' || git describe --tags --abbrev=0 --match 'v*'`
echo "REF=$prev" >> $GITHUB_OUTPUT echo "REF=$prev" >> $GITHUB_OUTPUT
- name: Install PlatformIO - name: Restore generated modbus header
run: | uses: actions/download-artifact@v4
python -m pip install --upgrade pip with:
pip install -U platformio name: modbus-header
python -m pip install intelhex path: src/core
- name: Build webUI - name: Restore generated docs
run: | uses: actions/download-artifact@v4
platformio run -e build-webUI with:
name: generated-docs
path: .
- name: Build modbus - name: Restore firmware binaries
run: | uses: actions/download-artifact@v4
platformio run -e build_modbus with:
pattern: firmware-*
merge-multiple: true
path: build/firmware
- name: Build standalone - name: List firmware files
run: | run: ls -la build/firmware
platformio run -e build_standalone
- name: Build all PIO target environments, from default_envs
run: |
platformio run
- name: Commit the generated files - name: Commit the generated files
uses: stefanzweifel/git-auto-commit-action@v7 uses: stefanzweifel/git-auto-commit-action@v7
interface/package.json
+1 -1
View File
@@ -31,7 +31,7 @@
"@table-library/react-table-library": "4.1.15", "@table-library/react-table-library": "4.1.15",
"alova": "^3.5.5", "alova": "^3.5.5",
"jwt-decode": "^4.0.0", "jwt-decode": "^4.0.0",
"preact": "^11.0.0", "preact": "^11.0.1",
"react": "^19.3.0", "react": "^19.3.0",
"react-dom": "^19.3.0", "react-dom": "^19.3.0",
"react-icons": "^5.7.0", "react-icons": "^5.7.0",
interface/pnpm-lock.yaml
Generated
+21 -21
View File
@@ -191,8 +191,8 @@ importers:
specifier: ^4.0.0 specifier: ^4.0.0
version: 4.0.0 version: 4.0.0
preact: preact:
specifier: ^11.0.0 specifier: ^11.0.1
version: 11.0.0 version: 11.0.1
react: react:
specifier: ^19.3.0 specifier: ^19.3.0
version: 19.3.0 version: 19.3.0
@@ -214,7 +214,7 @@ importers:
version: 10.0.1(eslint@10.12.0(supports-color@10.2.2)) version: 10.0.1(eslint@10.12.0(supports-color@10.2.2))
'@preact/preset-vite': '@preact/preset-vite':
specifier: ^2.10.6 specifier: ^2.10.6
version: 2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2)) version: 2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))
'@trivago/prettier-plugin-sort-imports': '@trivago/prettier-plugin-sort-imports':
specifier: ^6.0.2 specifier: ^6.0.2
version: 6.0.2(prettier@3.9.9)(supports-color@10.2.2) version: 6.0.2(prettier@3.9.9)(supports-color@10.2.2)
@@ -1338,8 +1338,8 @@ packages:
duplexer3@0.1.5: duplexer3@0.1.5:
resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==} resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==}
electron-to-chromium@1.5.449: electron-to-chromium@1.5.451:
resolution: {integrity: sha512-dKeYk/3SFQz+nXi5whp3OuJzeWABEG4zCtkWJ9+Vby1E1iFM2y0gElRtQvNXRPJTsj8WAF4HMP7hvP790QtJ9g==} resolution: {integrity: sha512-vUNKPrUzw8ByEMOxloq0a71iD/mPuPxVkbDm3G5YhnxTzOw/I6qAg2MUS0cYj/c267lWmw26yW4Vv7foxTxPLg==}
emoji-regex@10.6.0: emoji-regex@10.6.0:
resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==}
@@ -2531,8 +2531,8 @@ packages:
resolution: {integrity: sha512-C+y9x90UElAddDZmV4qOx9W53B61PO7cIqWz2dQsWlwswuq4mr8NEwytdGKboYbQlGZ3awrkTeNvcZiZNHnQ8A==} resolution: {integrity: sha512-C+y9x90UElAddDZmV4qOx9W53B61PO7cIqWz2dQsWlwswuq4mr8NEwytdGKboYbQlGZ3awrkTeNvcZiZNHnQ8A==}
engines: {node: '>=20'} engines: {node: '>=20'}
preact@11.0.0: preact@11.0.1:
resolution: {integrity: sha512-LRTLKuSe6VYPhvBDjwFaX+nuZnpsw6IN2GSboisAvhGsncylBmrbDCERhgKDn6yP2mAwbGvsB5OVyTe33M8gBg==} resolution: {integrity: sha512-TNNwEq++jYNHy747LziCjPRVgEwnLSOIOAw/oYcjI3Tdnubx2EXtt76I0an8PATvUWtx/DX1kFIg1eFN+O/fkw==}
peerDependencies: peerDependencies:
preact-render-to-string: '>=6.7.0' preact-render-to-string: '>=6.7.0'
peerDependenciesMeta: peerDependenciesMeta:
@@ -3015,8 +3015,8 @@ packages:
resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==}
engines: {node: '>= 10.0.0'} engines: {node: '>= 10.0.0'}
update-browserslist-db@1.3.3: update-browserslist-db@1.3.4:
resolution: {integrity: sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==} resolution: {integrity: sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==}
hasBin: true hasBin: true
peerDependencies: peerDependencies:
browserslist: '>= 4.21.0' browserslist: '>= 4.21.0'
@@ -3591,12 +3591,12 @@ snapshots:
'@popperjs/core@2.11.8': {} '@popperjs/core@2.11.8': {}
'@preact/preset-vite@2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))': '@preact/preset-vite@2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))':
dependencies: dependencies:
'@babel/core': 7.29.7(supports-color@10.2.2) '@babel/core': 7.29.7(supports-color@10.2.2)
'@babel/plugin-transform-react-jsx': 7.29.7(@babel/core@7.29.7(supports-color@10.2.2))(supports-color@10.2.2) '@babel/plugin-transform-react-jsx': 7.29.7(@babel/core@7.29.7(supports-color@10.2.2))(supports-color@10.2.2)
'@babel/plugin-transform-react-jsx-development': 7.29.7(@babel/core@7.29.7(supports-color@10.2.2))(supports-color@10.2.2) '@babel/plugin-transform-react-jsx-development': 7.29.7(@babel/core@7.29.7(supports-color@10.2.2))(supports-color@10.2.2)
'@prefresh/vite': 2.4.12(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2)) '@prefresh/vite': 2.4.12(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))
'@rollup/pluginutils': 5.4.0 '@rollup/pluginutils': 5.4.0
babel-plugin-transform-hook-names: 1.0.2(@babel/core@7.29.7(supports-color@10.2.2)) babel-plugin-transform-hook-names: 1.0.2(@babel/core@7.29.7(supports-color@10.2.2))
debug: 4.4.3(supports-color@10.2.2) debug: 4.4.3(supports-color@10.2.2)
@@ -3614,20 +3614,20 @@ snapshots:
dependencies: dependencies:
'@babel/core': 7.29.7(supports-color@10.2.2) '@babel/core': 7.29.7(supports-color@10.2.2)
'@prefresh/core@1.5.11(preact@11.0.0)': '@prefresh/core@1.5.11(preact@11.0.1)':
dependencies: dependencies:
preact: 11.0.0 preact: 11.0.1
'@prefresh/utils@1.2.1': {} '@prefresh/utils@1.2.1': {}
'@prefresh/vite@2.4.12(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))': '@prefresh/vite@2.4.12(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))':
dependencies: dependencies:
'@babel/core': 7.29.7(supports-color@10.2.2) '@babel/core': 7.29.7(supports-color@10.2.2)
'@prefresh/babel-plugin': 0.5.4(@babel/core@7.29.7(supports-color@10.2.2)) '@prefresh/babel-plugin': 0.5.4(@babel/core@7.29.7(supports-color@10.2.2))
'@prefresh/core': 1.5.11(preact@11.0.0) '@prefresh/core': 1.5.11(preact@11.0.1)
'@prefresh/utils': 1.2.1 '@prefresh/utils': 1.2.1
'@rollup/pluginutils': 4.2.1 '@rollup/pluginutils': 4.2.1
preact: 11.0.0 preact: 11.0.1
vite: 8.3.3(@types/node@26.6.4)(terser@5.51.2) vite: 8.3.3(@types/node@26.6.4)(terser@5.51.2)
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
@@ -4005,9 +4005,9 @@ snapshots:
dependencies: dependencies:
baseline-browser-mapping: 2.11.27 baseline-browser-mapping: 2.11.27
caniuse-lite: 1.0.30001815 caniuse-lite: 1.0.30001815
electron-to-chromium: 1.5.449 electron-to-chromium: 1.5.451
node-releases: 2.0.57 node-releases: 2.0.57
update-browserslist-db: 1.3.3(browserslist@4.29.3) update-browserslist-db: 1.3.4(browserslist@4.29.3)
buffer-alloc-unsafe@1.1.0: {} buffer-alloc-unsafe@1.1.0: {}
@@ -4370,7 +4370,7 @@ snapshots:
duplexer3@0.1.5: {} duplexer3@0.1.5: {}
electron-to-chromium@1.5.449: {} electron-to-chromium@1.5.451: {}
emoji-regex@10.6.0: {} emoji-regex@10.6.0: {}
@@ -5501,7 +5501,7 @@ snapshots:
powershell-utils@0.2.1: {} powershell-utils@0.2.1: {}
preact@11.0.0: {} preact@11.0.1: {}
prelude-ls@1.2.1: {} prelude-ls@1.2.1: {}
@@ -5953,7 +5953,7 @@ snapshots:
universalify@2.0.1: {} universalify@2.0.1: {}
update-browserslist-db@1.3.3(browserslist@4.29.3): update-browserslist-db@1.3.4(browserslist@4.29.3):
dependencies: dependencies:
browserslist: 4.29.3 browserslist: 4.29.3
escalade: 3.2.0 escalade: 3.2.0
platformio.ini
+17 -6
View File
@@ -67,14 +67,14 @@ board_upload.flash_size = 4MB
board_build.app_partition_name = app0 board_build.app_partition_name = app0
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5 platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5
; 16MB Flash variants ; 16MB Flash variants - using EMS-ESP Arduino/IDF libs (not Tasmota)
[espressif32_base_16M] [espressif32_base_16M]
framework = arduino framework = arduino
board_build.partitions = partitions/esp32_partition_16M.csv board_build.partitions = partitions/esp32_partition_16M.csv
board_upload.flash_size = 16MB board_upload.flash_size = 16MB
board_build.app_partition_name = app0 board_build.app_partition_name = app0
platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5 platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5
; platform_packages = framework-arduinoespressif32 @ https://github.com/emsesp/esp32-arduino-lib-builder/releases/download/0810-2237-5.5.5/framework-arduinoespressif32-v5.5.5-a63b839f.tar.xz platform_packages = framework-arduinoespressif32 @ https://github.com/emsesp/esp32-arduino-lib-builder/releases/download/0810-2237-5.5.5/framework-arduinoespressif32-v5.5.5-a63b839f.tar.xz
; 32MB Flash variants ; 32MB Flash variants
[espressif32_base_32M] [espressif32_base_32M]
@@ -90,6 +90,7 @@ build_flags =
build_unflags = build_unflags =
${common.unbuild_flags} ${common.unbuild_flags}
extra_scripts = extra_scripts =
pre:scripts/enable_ccache.py ; uses ccache when it is installed
post:scripts/rename_fw.py ; renames the firmware .bin file post:scripts/rename_fw.py ; renames the firmware .bin file
monitor_speed = 115200 monitor_speed = 115200
monitor_filters = direct, esp32_exception_decoder monitor_filters = direct, esp32_exception_decoder
@@ -133,22 +134,32 @@ extends = espressif32_base_16M
board = s_16M_P board = s_16M_P
[env:s_16M_P_custom] [env:s_16M_P_custom]
; same as s_16M_P, but the Arduino/IDF libs are rebuilt with HybridCompile using the ; same as s_16M_P, but the Arduino/IDF libs are rebuilt with HybridCompile using the custom_sdkconfig below
; custom_sdkconfig below, to compare flash size and free heap against stock s_16M_P.
; first build takes 10-20 min (IDF compiled from source), later builds are cached.
; building s_16M_P and this env alternately triggers a rebuild each time.
extends = env:s_16M_P extends = env:s_16M_P
custom_sdkconfig = custom_sdkconfig =
'# CONFIG_BT_ENABLED is not set' '# CONFIG_BT_ENABLED is not set'
'# CONFIG_LWIP_PPP_SUPPORT is not set' '# CONFIG_LWIP_PPP_SUPPORT is not set'
'# CONFIG_LWIP_IP_FORWARD is not set' '# CONFIG_LWIP_IP_FORWARD is not set'
'# CONFIG_LWIP_IPV4_NAPT is not set' '# CONFIG_LWIP_IPV4_NAPT is not set'
'# CONFIG_LWIP_IPV4_NAPT_PORTMAP is not set'
'# CONFIG_ULP_COPROC_ENABLED is not set' '# CONFIG_ULP_COPROC_ENABLED is not set'
'# CONFIG_ETH_USE_SPI_ETHERNET is not set' '# CONFIG_ETH_USE_SPI_ETHERNET is not set'
'# CONFIG_ETH_SPI_ETHERNET_W5500 is not set' '# CONFIG_ETH_SPI_ETHERNET_W5500 is not set'
'# CONFIG_ETH_SPI_ETHERNET_DM9051 is not set' '# CONFIG_ETH_SPI_ETHERNET_DM9051 is not set'
'# CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL is not set' '# CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL is not set'
'# CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS is not set' '# CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS is not set'
'# CONFIG_FREERTOS_USE_TRACE_FACILITY is not set'
'# CONFIG_FREERTOS_ENABLE_TASK_SNAPSHOT is not set'
'# CONFIG_ESP_MODEM_USE_PPP_MODE is not set'
'# CONFIG_ESP_GDBSTUB_ENABLED is not set'
'# CONFIG_MDNS_ENABLE_CONSOLE_CLI is not set'
'# CONFIG_MDNS_ENABLE_BROWSE is not set'
'# CONFIG_I2S_ISR_IRAM_SAFE is not set'
; UART ISR must live in IRAM. Tasmota puts ringbuf ISRs in flash, which
; forces CONFIG_UART_ISR_IN_IRAM=n and panics (Cache error) when the EMS
; or console UART fires during a flash read (WiFi stop / mDNS / NVS).
'# CONFIG_RINGBUF_PLACE_ISR_FUNCTIONS_INTO_FLASH is not set'
'CONFIG_UART_ISR_IN_IRAM=y'
[env:s3_16M_P] [env:s3_16M_P]
; 16MB ESP32-S3 - with PSRAM - like a BBQKees S3 ; 16MB ESP32-S3 - with PSRAM - like a BBQKees S3
scripts/enable_ccache.py
+27
View File
@@ -0,0 +1,27 @@
# Wrap the C/C++ compilers with ccache when it is on PATH.
# No-op locally unless ccache is installed; CI installs it via ccache-action.
Import("env")
import shutil
ccache = shutil.which("ccache")
if ccache:
def wrap(tool):
cmd = env.get(tool)
if not cmd:
return
resolved = env.subst(cmd) if isinstance(cmd, str) else cmd
if isinstance(resolved, str):
if resolved.startswith("ccache"):
return
env[tool] = f"{ccache} {resolved}"
return
if isinstance(resolved, (list, tuple)):
if resolved and "ccache" in str(resolved[0]):
return
env[tool] = [ccache, *resolved]
wrap("CC")
wrap("CXX")
print("ccache enabled")
scripts/upload.py
+107 -45
View File
@@ -14,12 +14,17 @@
# and # and
# extra_scripts = scripts/upload.py # extra_scripts = scripts/upload.py
# #
# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency
# and can stall on IPv6 before falling back to IPv4.
import hashlib
import os
import socket
import time
from urllib.parse import urlparse
import requests import requests
import hashlib from requests.adapters import HTTPAdapter
from urllib.parse import urlparse
import time
import os
Import("env") Import("env")
@@ -35,6 +40,10 @@ except ImportError:
from tqdm import tqdm from tqdm import tqdm
from termcolor import cprint from termcolor import cprint
# http.client/urllib3 default is 8–16 KiB, which starves a LAN TCP window.
UPLOAD_BLOCK_SIZE = 64 * 1024
UPLOAD_SNDBUF = 256 * 1024
def print_success(x): def print_success(x):
cprint(x, 'green') cprint(x, 'green')
@@ -44,6 +53,43 @@ def print_fail(x):
cprint(f'Error: {x}', 'red') cprint(f'Error: {x}', 'red')
def resolve_ipv4(host):
"""Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request."""
if host.startswith('['):
return host
if ':' in host and host.count(':') == 1:
host = host.rsplit(':', 1)[0]
try:
socket.inet_pton(socket.AF_INET, host)
return host
except OSError:
pass
infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM)
if not infos:
raise socket.gaierror(f'No IPv4 address for {host}')
return infos[0][4][0]
class FastHTTPAdapter(HTTPAdapter):
"""Larger write chunks and socket send buffer for streaming POSTs."""
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
socket_options = [
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
(socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF),
]
pool_kwargs.setdefault('socket_options', socket_options)
pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE)
super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
def make_session():
session = requests.Session()
adapter = FastHTTPAdapter(pool_connections=2, pool_maxsize=2)
session.mount('http://', adapter)
return session
def build_headers(host_ip, emsesp_url, content_type='application/json', access_token=None, extra_headers=None): def build_headers(host_ip, emsesp_url, content_type='application/json', access_token=None, extra_headers=None):
"""Build common HTTP headers with optional overrides.""" """Build common HTTP headers with optional overrides."""
headers = { headers = {
@@ -51,18 +97,17 @@ def build_headers(host_ip, emsesp_url, content_type='application/json', access_t
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0',
'Accept': '*/*', 'Accept': '*/*',
'Accept-Language': 'en-US', 'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip, deflate',
'Referer': emsesp_url, 'Referer': emsesp_url,
'Content-Type': content_type, 'Content-Type': content_type,
'Connection': 'keep-alive' 'Connection': 'keep-alive'
} }
if access_token: if access_token:
headers['Authorization'] = f'Bearer {access_token}' headers['Authorization'] = f'Bearer {access_token}'
if extra_headers: if extra_headers:
headers.update(extra_headers) headers.update(extra_headers)
return headers return headers
@@ -79,44 +124,52 @@ def on_upload(source, target, env):
username = env.GetProjectOption('custom_username') username = env.GetProjectOption('custom_username')
password = env.GetProjectOption('custom_password') password = env.GetProjectOption('custom_password')
emsesp_ip = env.GetProjectOption('custom_emsesp_ip') emsesp_ip = env.GetProjectOption('custom_emsesp_ip')
except Exception as e: except Exception:
print_fail(f'Missing settings. Add these to your pio_local.ini file:\n\ncustom_username=username\ncustom_password=password\ncustom_emsesp_ip=ems-esp.local\n') print_fail(f'Missing settings. Add these to your pio_local.ini file:\n\ncustom_username=username\ncustom_password=password\ncustom_emsesp_ip=ems-esp.local\n')
return return
emsesp_url = f"http://{emsesp_ip}" parsed_url = urlparse(f"http://{emsesp_ip}")
parsed_url = urlparse(emsesp_url) host_name = parsed_url.hostname or emsesp_ip
host_ip = parsed_url.netloc try:
host_ip = resolve_ipv4(host_name)
signon_url = f"{emsesp_url}/rest/signIn" except OSError as e:
signon_headers = build_headers(host_ip, emsesp_url) print_fail(f"Could not resolve {host_name} to IPv4: {e}")
username_password = {
"username": username,
"password": password
}
response = requests.post(
signon_url, json=username_password, headers=signon_headers)
if response.status_code != 200:
print_fail("Authentication with EMS-ESP failed (code " +
str(response.status_code) + ")")
return return
print_success("Authentication with EMS-ESP successful") if host_ip != host_name:
access_token = response.json().get('access_token') print_success(f"Resolved {host_name} -> {host_ip}")
# start the upload emsesp_url = f"http://{host_ip}"
firmware_path = str(source[0]) session = make_session()
with open(firmware_path, 'rb') as firmware: try:
md5 = hashlib.md5(firmware.read()).hexdigest() signon_url = f"{emsesp_url}/rest/signIn"
signon_headers = build_headers(host_ip, emsesp_url)
firmware.seek(0) response = session.post(
signon_url,
json={"username": username, "password": password},
headers=signon_headers,
timeout=15)
if response.status_code != 200:
print_fail("Authentication with EMS-ESP failed (code " +
str(response.status_code) + ")")
return
print_success("Authentication with EMS-ESP successful")
access_token = response.json().get('access_token')
firmware_path = str(source[0])
with open(firmware_path, 'rb') as firmware_file:
firmware = firmware_file.read()
md5 = hashlib.md5(firmware).hexdigest()
filename = os.path.basename(firmware_path)
encoder = MultipartEncoder(fields={ encoder = MultipartEncoder(fields={
'MD5': md5, 'MD5': md5,
'file': (firmware_path, firmware, 'application/octet-stream')} 'file': (filename, firmware, 'application/octet-stream')}
) )
bar = tqdm(desc='Upload Progress', bar = tqdm(desc='Upload Progress',
@@ -124,7 +177,8 @@ def on_upload(source, target, env):
dynamic_ncols=True, dynamic_ncols=True,
unit='B', unit='B',
unit_scale=True, unit_scale=True,
unit_divisor=1024 unit_divisor=1024,
mininterval=0.2
) )
monitor = MultipartEncoderMonitor( monitor = MultipartEncoderMonitor(
@@ -142,27 +196,35 @@ def on_upload(source, target, env):
) )
upload_url = f"{emsesp_url}/rest/uploadFile" upload_url = f"{emsesp_url}/rest/uploadFile"
started = time.perf_counter()
response = requests.post( response = session.post(
upload_url, data=monitor, headers=post_headers) upload_url, data=monitor, headers=post_headers, timeout=None)
elapsed = time.perf_counter() - started
bar.close() bar.close()
time.sleep(0.1)
print() print()
if response.status_code != 200: if response.status_code != 200:
print_fail(f"Upload failed (code {response.status_code}).") print_fail(f"Upload failed (code {response.status_code}).")
else: else:
print_success("Upload successful. Rebooting device.") kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0
print_success(
f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.")
restart_headers = build_headers( restart_headers = build_headers(
host_ip, emsesp_url, access_token=access_token) host_ip, emsesp_url, access_token=access_token)
restart_url = f"{emsesp_url}/api/system/restart" restart_url = f"{emsesp_url}/api/system/restart"
response = requests.get(restart_url, headers=restart_headers) try:
if response.status_code != 200: response = session.get(
print_fail(f"Restart failed (code {response.status_code})") restart_url, headers=restart_headers, timeout=10)
if response.status_code != 200:
print_fail(f"Restart failed (code {response.status_code})")
except requests.RequestException:
# Device often drops the connection as it reboots — that's OK.
pass
print() print()
finally:
session.close()
if env.get('UPLOAD_PROTOCOL') == 'custom': if env.get('UPLOAD_PROTOCOL') == 'custom':
scripts/upload_cli.py
+103 -68
View File
@@ -9,21 +9,27 @@
# #
# Run using for example: # Run using for example:
# python3 upload_cli.py -i 10.10.10.175 -f ../build/firmware/EMS-ESP-3_7_0-dev_34-ESP32S3-16MB+.bin # python3 upload_cli.py -i 10.10.10.175 -f ../build/firmware/EMS-ESP-3_7_0-dev_34-ESP32S3-16MB+.bin
#
# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency
# and can stall on IPv6 before falling back to IPv4.
import argparse import argparse
import hashlib import hashlib
import socket
import sys import sys
import time import time
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse
import requests import requests
from requests.adapters import HTTPAdapter
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
from termcolor import cprint from termcolor import cprint
from tqdm import tqdm from tqdm import tqdm
# Constants
USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0' USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0'
CHUNK_SIZE = 8192 # 8KB chunks for MD5 calculation UPLOAD_BLOCK_SIZE = 64 * 1024
UPLOAD_SNDBUF = 256 * 1024
def print_success(x): def print_success(x):
@@ -34,13 +40,34 @@ def print_fail(x):
return cprint(x, 'red') return cprint(x, 'red')
def calculate_md5(file_path): def resolve_ipv4(host):
"""Calculate MD5 hash of a file in chunks for memory efficiency.""" """Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request."""
md5_hash = hashlib.md5() if host.startswith('['):
with open(file_path, 'rb') as f: return host
for chunk in iter(lambda: f.read(CHUNK_SIZE), b''): if ':' in host and host.count(':') == 1:
md5_hash.update(chunk) host = host.rsplit(':', 1)[0]
return md5_hash.hexdigest() try:
socket.inet_pton(socket.AF_INET, host)
return host
except OSError:
pass
infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM)
if not infos:
raise socket.gaierror(f'No IPv4 address for {host}')
return infos[0][4][0]
class FastHTTPAdapter(HTTPAdapter):
"""Larger write chunks and socket send buffer for streaming POSTs."""
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
socket_options = [
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
(socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF),
]
pool_kwargs.setdefault('socket_options', socket_options)
pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE)
super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
def create_base_headers(host_ip, emsesp_url): def create_base_headers(host_ip, emsesp_url):
@@ -50,7 +77,6 @@ def create_base_headers(host_ip, emsesp_url):
'User-Agent': USER_AGENT, 'User-Agent': USER_AGENT,
'Accept': '*/*', 'Accept': '*/*',
'Accept-Language': 'en-US', 'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip, deflate',
'Referer': emsesp_url, 'Referer': emsesp_url,
'Connection': 'keep-alive' 'Connection': 'keep-alive'
} }
@@ -58,35 +84,39 @@ def create_base_headers(host_ip, emsesp_url):
def upload(file, ip, username, password): def upload(file, ip, username, password):
"""Upload firmware to EMS-ESP device.""" """Upload firmware to EMS-ESP device."""
# Print welcome message
print() print()
print("EMS-ESP Firmware Upload") print("EMS-ESP Firmware Upload")
# Validate file exists
file_path = Path(file) file_path = Path(file)
if not file_path.exists(): if not file_path.exists():
print_fail(f"File not found: {file}") print_fail(f"File not found: {file}")
return return
# Setup URLs and headers parsed_url = urlparse(f"http://{ip}")
emsesp_url = f"http://{ip}" host_name = parsed_url.hostname or ip
host_ip = ip try:
host_ip = resolve_ipv4(host_name)
# Use a session for connection pooling and persistence except OSError as e:
print_fail(f"Could not resolve {host_name} to IPv4: {e}")
return
if host_ip != host_name:
print_success(f"Resolved {host_name} -> {host_ip}")
emsesp_url = f"http://{host_ip}"
session = requests.Session() session = requests.Session()
session.mount('http://', FastHTTPAdapter(pool_connections=2, pool_maxsize=2))
try: try:
# Authentication
signon_url = f"{emsesp_url}/rest/signIn" signon_url = f"{emsesp_url}/rest/signIn"
signon_headers = create_base_headers(host_ip, emsesp_url) signon_headers = create_base_headers(host_ip, emsesp_url)
signon_headers['Content-Type'] = 'application/json' signon_headers['Content-Type'] = 'application/json'
username_password = { response = session.post(
"username": username, signon_url,
"password": password json={"username": username, "password": password},
} headers=signon_headers,
timeout=15)
response = session.post(signon_url, json=username_password, headers=signon_headers)
if response.status_code != 200: if response.status_code != 200:
print_fail(f"Authentication failed (code {response.status_code})") print_fail(f"Authentication failed (code {response.status_code})")
@@ -95,56 +125,62 @@ def upload(file, ip, username, password):
print_success("Authentication successful") print_success("Authentication successful")
access_token = response.json().get('access_token') access_token = response.json().get('access_token')
# Calculate MD5 hash firmware = file_path.read_bytes()
print("Calculating MD5 hash...") md5 = hashlib.md5(firmware).hexdigest()
md5 = calculate_md5(file_path)
# Start the upload encoder = MultipartEncoder(fields={
with open(file_path, 'rb') as firmware: 'MD5': md5,
encoder = MultipartEncoder(fields={ 'file': (file_path.name, firmware, 'application/octet-stream')
'MD5': md5, })
'file': (file, firmware, 'application/octet-stream')
})
bar = tqdm( bar = tqdm(
desc='Upload Progress', desc='Upload Progress',
total=encoder.len, total=encoder.len,
dynamic_ncols=True, dynamic_ncols=True,
unit='B', unit='B',
unit_scale=True, unit_scale=True,
unit_divisor=1024 unit_divisor=1024,
) mininterval=0.2
)
monitor = MultipartEncoderMonitor( monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n)) encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n))
post_headers = create_base_headers(host_ip, emsesp_url) post_headers = create_base_headers(host_ip, emsesp_url)
post_headers.update({ post_headers.update({
'Content-Type': monitor.content_type, 'Content-Type': monitor.content_type,
'Content-Length': str(monitor.len), 'Content-Length': str(monitor.len),
'Origin': emsesp_url, 'Origin': emsesp_url,
'Authorization': f'Bearer {access_token}'
})
upload_url = f"{emsesp_url}/rest/uploadFile"
started = time.perf_counter()
response = session.post(
upload_url, data=monitor, headers=post_headers, timeout=None)
elapsed = time.perf_counter() - started
bar.close()
if response.status_code != 200:
print_fail(f"Upload failed (code {response.status_code})")
else:
kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0
print_success(
f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.")
restart_headers = create_base_headers(host_ip, emsesp_url)
restart_headers.update({
'Content-Type': 'application/json',
'Authorization': f'Bearer {access_token}' 'Authorization': f'Bearer {access_token}'
}) })
restart_url = f"{emsesp_url}/api/system/restart"
upload_url = f"{emsesp_url}/rest/uploadFile" try:
response = session.post(upload_url, data=monitor, headers=post_headers) response = session.get(
restart_url, headers=restart_headers, timeout=10)
bar.close()
time.sleep(0.1)
if response.status_code != 200:
print_fail(f"Upload failed (code {response.status_code})")
else:
print_success("Upload successful. Rebooting device.")
restart_headers = create_base_headers(host_ip, emsesp_url)
restart_headers.update({
'Content-Type': 'application/json',
'Authorization': f'Bearer {access_token}'
})
restart_url = f"{emsesp_url}/api/system/restart"
response = session.get(restart_url, headers=restart_headers)
if response.status_code != 200: if response.status_code != 200:
print_fail(f"Restart failed (code {response.status_code})") print_fail(f"Restart failed (code {response.status_code})")
except requests.RequestException:
pass
except requests.RequestException as e: except requests.RequestException as e:
print_fail(f"Network error: {e}") print_fail(f"Network error: {e}")
@@ -161,7 +197,6 @@ def upload(file, ip, username, password):
print() print()
# main
parser = argparse.ArgumentParser(description="EMS-ESP Firmware Upload") parser = argparse.ArgumentParser(description="EMS-ESP Firmware Upload")
parser.add_argument("-f", "--file", metavar="FILE", parser.add_argument("-f", "--file", metavar="FILE",
required=True, type=str, help="firmware file") required=True, type=str, help="firmware file")