From e3b1eb6c51bb2e05a73e3c83be8397308176634b Mon Sep 17 00:00:00 2001 From: proddy Date: Fri, 9 Oct 2026 17:47:01 +0200 Subject: [PATCH] test new esp32-arduino-lib-builder packages --- .github/workflows/dev_release.yml | 233 ++++++++++++++++++++++++++---- interface/package.json | 2 +- interface/pnpm-lock.yaml | 42 +++--- platformio.ini | 23 ++- scripts/enable_ccache.py | 27 ++++ scripts/upload.py | 152 +++++++++++++------ scripts/upload_cli.py | 171 +++++++++++++--------- 7 files changed, 482 insertions(+), 168 deletions(-) create mode 100644 scripts/enable_ccache.py diff --git a/.github/workflows/dev_release.yml b/.github/workflows/dev_release.yml index f518634e4..a9811b2b3 100644 --- a/.github/workflows/dev_release.yml +++ b/.github/workflows/dev_release.yml @@ -11,30 +11,209 @@ on: permissions: contents: write +# Parallel graph (wall-clock is ~max of the firmware jobs, not the sum): +# webui ──────┐ +# ├─► firmware (s_4M / s_16M / s_16M_P / s3_16M_P) ─┐ +# modbus ─────┤ ├─► release +# └─► standalone ───────────────────────────────────┘ +# +# Extra -j on a single runner does not help: PlatformIO already compiles each +# environment with all CPUs. The win is separate GitHub runners. + jobs: - pre-release: - name: 'Build Dev Release' + webui: + name: Build webUI if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32' runs-on: ubuntu-latest steps: - - - name: Install python 3.13 - uses: actions/setup-python@v7 + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install pnpm + uses: pnpm/action-setup@v4 with: - python-version: '3.13' + package_json_file: interface/package.json - name: Install Node.js 26 uses: actions/setup-node@v7 with: node-version: 26 + - name: Cache pnpm store + uses: actions/cache@v4 + with: + path: ~/.local/share/pnpm/store + key: ${{ runner.os }}-pnpm-${{ hashFiles('interface/package.json') }} + restore-keys: | + ${{ runner.os }}-pnpm- + + - name: Build webUI + env: + CI: true + working-directory: interface + run: | + pnpm install + pnpm build-webUI + + - name: Upload WWWData.h + uses: actions/upload-artifact@v4 + with: + name: wwwdata + path: src/ESP32React/WWWData.h + if-no-files-found: error + + modbus: + name: Build modbus + if: github.event_name == 'workflow_dispatch' || github.repository == 'emsesp/EMS-ESP32' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + enable-cache: false + + - name: Install pioarduino + run: uv pip install --system pioarduino==6.2.0 + + - name: Build modbus + run: platformio run -e build_modbus + + - name: Upload modbus header + uses: actions/upload-artifact@v4 + with: + name: modbus-header + path: src/core/modbus_entity_parameters.hpp + if-no-files-found: error + + standalone: + name: Build standalone + needs: modbus + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + enable-cache: false + + - name: Install pioarduino + run: uv pip install --system pioarduino==6.2.0 + + - name: Restore generated modbus header + uses: actions/download-artifact@v4 + with: + name: modbus-header + path: src/core + + - name: Build standalone + run: platformio run -e build_standalone + + - name: Upload generated docs + uses: actions/upload-artifact@v4 + with: + name: generated-docs + path: | + docs/dump_entities.csv + docs/dump_telegrams.csv + docs/Modbus-Entity-Registers.md + if-no-files-found: error + + firmware: + name: Firmware ${{ matrix.env }} + needs: [webui, modbus] + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + # Keep in sync with platformio.ini default_envs + env: [s_4M, s_16M, s_16M_P, s3_16M_P] + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + enable-cache: false + + - name: Cache PlatformIO packages + uses: actions/cache@v4 + with: + path: | + ~/.platformio/.cache + ~/.platformio/packages + ~/.platformio/platforms + key: ${{ runner.os }}-pio-${{ matrix.env }}-${{ hashFiles('platformio.ini') }} + restore-keys: | + ${{ runner.os }}-pio-${{ matrix.env }}- + ${{ runner.os }}-pio- + + - name: Install pioarduino + run: uv pip install --system pioarduino==6.2.0 intelhex + + - name: Restore WWWData.h + uses: actions/download-artifact@v4 + with: + name: wwwdata + path: src/ESP32React + + - name: Restore generated modbus header + uses: actions/download-artifact@v4 + with: + name: modbus-header + path: src/core + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: firmware-${{ matrix.env }} + restore-keys: firmware-${{ matrix.env }}- + max-size: 1G + + - name: Build ${{ matrix.env }} + env: + CCACHE_COMPILERCHECK: content + run: platformio run -e ${{ matrix.env }} + + - name: Upload firmware + uses: actions/upload-artifact@v4 + with: + name: firmware-${{ matrix.env }} + path: build/firmware + if-no-files-found: error + + release: + name: Create GitHub Release + needs: [firmware, standalone] + runs-on: ubuntu-latest + steps: - name: Checkout repository uses: actions/checkout@v7 with: fetch-depth: 0 - - - name: Install pnpm - run: npm install -g pnpm - name: Get the EMS-ESP version id: build_info @@ -49,27 +228,27 @@ jobs: prev=`git rev-parse -q --verify 'refs/tags/latest^{commit}' || git describe --tags --abbrev=0 --match 'v*'` echo "REF=$prev" >> $GITHUB_OUTPUT - - name: Install PlatformIO - run: | - python -m pip install --upgrade pip - pip install -U platformio - python -m pip install intelhex + - name: Restore generated modbus header + uses: actions/download-artifact@v4 + with: + name: modbus-header + path: src/core - - name: Build webUI - run: | - platformio run -e build-webUI + - name: Restore generated docs + uses: actions/download-artifact@v4 + with: + name: generated-docs + path: . - - name: Build modbus - run: | - platformio run -e build_modbus + - name: Restore firmware binaries + uses: actions/download-artifact@v4 + with: + pattern: firmware-* + merge-multiple: true + path: build/firmware - - name: Build standalone - run: | - platformio run -e build_standalone - - - name: Build all PIO target environments, from default_envs - run: | - platformio run + - name: List firmware files + run: ls -la build/firmware - name: Commit the generated files uses: stefanzweifel/git-auto-commit-action@v7 diff --git a/interface/package.json b/interface/package.json index 95c0e9e8b..05cda3a78 100644 --- a/interface/package.json +++ b/interface/package.json @@ -31,7 +31,7 @@ "@table-library/react-table-library": "4.1.15", "alova": "^3.5.5", "jwt-decode": "^4.0.0", - "preact": "^11.0.0", + "preact": "^11.0.1", "react": "^19.3.0", "react-dom": "^19.3.0", "react-icons": "^5.7.0", diff --git a/interface/pnpm-lock.yaml b/interface/pnpm-lock.yaml index 91d2b5913..fc19efc83 100644 --- a/interface/pnpm-lock.yaml +++ b/interface/pnpm-lock.yaml @@ -191,8 +191,8 @@ importers: specifier: ^4.0.0 version: 4.0.0 preact: - specifier: ^11.0.0 - version: 11.0.0 + specifier: ^11.0.1 + version: 11.0.1 react: specifier: ^19.3.0 version: 19.3.0 @@ -214,7 +214,7 @@ importers: version: 10.0.1(eslint@10.12.0(supports-color@10.2.2)) '@preact/preset-vite': specifier: ^2.10.6 - version: 2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2)) + version: 2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2)) '@trivago/prettier-plugin-sort-imports': specifier: ^6.0.2 version: 6.0.2(prettier@3.9.9)(supports-color@10.2.2) @@ -1338,8 +1338,8 @@ packages: duplexer3@0.1.5: resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==} - electron-to-chromium@1.5.449: - resolution: {integrity: sha512-dKeYk/3SFQz+nXi5whp3OuJzeWABEG4zCtkWJ9+Vby1E1iFM2y0gElRtQvNXRPJTsj8WAF4HMP7hvP790QtJ9g==} + electron-to-chromium@1.5.451: + resolution: {integrity: sha512-vUNKPrUzw8ByEMOxloq0a71iD/mPuPxVkbDm3G5YhnxTzOw/I6qAg2MUS0cYj/c267lWmw26yW4Vv7foxTxPLg==} emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} @@ -2531,8 +2531,8 @@ packages: resolution: {integrity: sha512-C+y9x90UElAddDZmV4qOx9W53B61PO7cIqWz2dQsWlwswuq4mr8NEwytdGKboYbQlGZ3awrkTeNvcZiZNHnQ8A==} engines: {node: '>=20'} - preact@11.0.0: - resolution: {integrity: sha512-LRTLKuSe6VYPhvBDjwFaX+nuZnpsw6IN2GSboisAvhGsncylBmrbDCERhgKDn6yP2mAwbGvsB5OVyTe33M8gBg==} + preact@11.0.1: + resolution: {integrity: sha512-TNNwEq++jYNHy747LziCjPRVgEwnLSOIOAw/oYcjI3Tdnubx2EXtt76I0an8PATvUWtx/DX1kFIg1eFN+O/fkw==} peerDependencies: preact-render-to-string: '>=6.7.0' peerDependenciesMeta: @@ -3015,8 +3015,8 @@ packages: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} - update-browserslist-db@1.3.3: - resolution: {integrity: sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==} + update-browserslist-db@1.3.4: + resolution: {integrity: sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==} hasBin: true peerDependencies: browserslist: '>= 4.21.0' @@ -3591,12 +3591,12 @@ snapshots: '@popperjs/core@2.11.8': {} - '@preact/preset-vite@2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))': + '@preact/preset-vite@2.10.6(@babel/core@7.29.7(supports-color@10.2.2))(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))': dependencies: '@babel/core': 7.29.7(supports-color@10.2.2) '@babel/plugin-transform-react-jsx': 7.29.7(@babel/core@7.29.7(supports-color@10.2.2))(supports-color@10.2.2) '@babel/plugin-transform-react-jsx-development': 7.29.7(@babel/core@7.29.7(supports-color@10.2.2))(supports-color@10.2.2) - '@prefresh/vite': 2.4.12(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2)) + '@prefresh/vite': 2.4.12(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2)) '@rollup/pluginutils': 5.4.0 babel-plugin-transform-hook-names: 1.0.2(@babel/core@7.29.7(supports-color@10.2.2)) debug: 4.4.3(supports-color@10.2.2) @@ -3614,20 +3614,20 @@ snapshots: dependencies: '@babel/core': 7.29.7(supports-color@10.2.2) - '@prefresh/core@1.5.11(preact@11.0.0)': + '@prefresh/core@1.5.11(preact@11.0.1)': dependencies: - preact: 11.0.0 + preact: 11.0.1 '@prefresh/utils@1.2.1': {} - '@prefresh/vite@2.4.12(preact@11.0.0)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))': + '@prefresh/vite@2.4.12(preact@11.0.1)(supports-color@10.2.2)(vite@8.3.3(@types/node@26.6.4)(terser@5.51.2))': dependencies: '@babel/core': 7.29.7(supports-color@10.2.2) '@prefresh/babel-plugin': 0.5.4(@babel/core@7.29.7(supports-color@10.2.2)) - '@prefresh/core': 1.5.11(preact@11.0.0) + '@prefresh/core': 1.5.11(preact@11.0.1) '@prefresh/utils': 1.2.1 '@rollup/pluginutils': 4.2.1 - preact: 11.0.0 + preact: 11.0.1 vite: 8.3.3(@types/node@26.6.4)(terser@5.51.2) transitivePeerDependencies: - supports-color @@ -4005,9 +4005,9 @@ snapshots: dependencies: baseline-browser-mapping: 2.11.27 caniuse-lite: 1.0.30001815 - electron-to-chromium: 1.5.449 + electron-to-chromium: 1.5.451 node-releases: 2.0.57 - update-browserslist-db: 1.3.3(browserslist@4.29.3) + update-browserslist-db: 1.3.4(browserslist@4.29.3) buffer-alloc-unsafe@1.1.0: {} @@ -4370,7 +4370,7 @@ snapshots: duplexer3@0.1.5: {} - electron-to-chromium@1.5.449: {} + electron-to-chromium@1.5.451: {} emoji-regex@10.6.0: {} @@ -5501,7 +5501,7 @@ snapshots: powershell-utils@0.2.1: {} - preact@11.0.0: {} + preact@11.0.1: {} prelude-ls@1.2.1: {} @@ -5953,7 +5953,7 @@ snapshots: universalify@2.0.1: {} - update-browserslist-db@1.3.3(browserslist@4.29.3): + update-browserslist-db@1.3.4(browserslist@4.29.3): dependencies: browserslist: 4.29.3 escalade: 3.2.0 diff --git a/platformio.ini b/platformio.ini index 1f7fc3038..623058d2f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -67,14 +67,14 @@ board_upload.flash_size = 4MB board_build.app_partition_name = app0 platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5 -; 16MB Flash variants +; 16MB Flash variants - using EMS-ESP Arduino/IDF libs (not Tasmota) [espressif32_base_16M] framework = arduino board_build.partitions = partitions/esp32_partition_16M.csv board_upload.flash_size = 16MB board_build.app_partition_name = app0 platform = https://github.com/tasmota/platform-espressif32/releases/download/2026.09.50/platform-espressif32.zip ; Tasmota Arduino Core 3.3.12 based on IDF 5.5.5 -; platform_packages = framework-arduinoespressif32 @ https://github.com/emsesp/esp32-arduino-lib-builder/releases/download/0810-2237-5.5.5/framework-arduinoespressif32-v5.5.5-a63b839f.tar.xz +platform_packages = framework-arduinoespressif32 @ https://github.com/emsesp/esp32-arduino-lib-builder/releases/download/0810-2237-5.5.5/framework-arduinoespressif32-v5.5.5-a63b839f.tar.xz ; 32MB Flash variants [espressif32_base_32M] @@ -90,6 +90,7 @@ build_flags = build_unflags = ${common.unbuild_flags} extra_scripts = + pre:scripts/enable_ccache.py ; uses ccache when it is installed post:scripts/rename_fw.py ; renames the firmware .bin file monitor_speed = 115200 monitor_filters = direct, esp32_exception_decoder @@ -133,22 +134,32 @@ extends = espressif32_base_16M board = s_16M_P [env:s_16M_P_custom] -; same as s_16M_P, but the Arduino/IDF libs are rebuilt with HybridCompile using the -; custom_sdkconfig below, to compare flash size and free heap against stock s_16M_P. -; first build takes 10-20 min (IDF compiled from source), later builds are cached. -; building s_16M_P and this env alternately triggers a rebuild each time. +; same as s_16M_P, but the Arduino/IDF libs are rebuilt with HybridCompile using the custom_sdkconfig below extends = env:s_16M_P custom_sdkconfig = '# CONFIG_BT_ENABLED is not set' '# CONFIG_LWIP_PPP_SUPPORT is not set' '# CONFIG_LWIP_IP_FORWARD is not set' '# CONFIG_LWIP_IPV4_NAPT is not set' + '# CONFIG_LWIP_IPV4_NAPT_PORTMAP is not set' '# CONFIG_ULP_COPROC_ENABLED is not set' '# CONFIG_ETH_USE_SPI_ETHERNET is not set' '# CONFIG_ETH_SPI_ETHERNET_W5500 is not set' '# CONFIG_ETH_SPI_ETHERNET_DM9051 is not set' '# CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL is not set' '# CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS is not set' + '# CONFIG_FREERTOS_USE_TRACE_FACILITY is not set' + '# CONFIG_FREERTOS_ENABLE_TASK_SNAPSHOT is not set' + '# CONFIG_ESP_MODEM_USE_PPP_MODE is not set' + '# CONFIG_ESP_GDBSTUB_ENABLED is not set' + '# CONFIG_MDNS_ENABLE_CONSOLE_CLI is not set' + '# CONFIG_MDNS_ENABLE_BROWSE is not set' + '# CONFIG_I2S_ISR_IRAM_SAFE is not set' + ; UART ISR must live in IRAM. Tasmota puts ringbuf ISRs in flash, which + ; forces CONFIG_UART_ISR_IN_IRAM=n and panics (Cache error) when the EMS + ; or console UART fires during a flash read (WiFi stop / mDNS / NVS). + '# CONFIG_RINGBUF_PLACE_ISR_FUNCTIONS_INTO_FLASH is not set' + 'CONFIG_UART_ISR_IN_IRAM=y' [env:s3_16M_P] ; 16MB ESP32-S3 - with PSRAM - like a BBQKees S3 diff --git a/scripts/enable_ccache.py b/scripts/enable_ccache.py new file mode 100644 index 000000000..2bf5d1684 --- /dev/null +++ b/scripts/enable_ccache.py @@ -0,0 +1,27 @@ +# Wrap the C/C++ compilers with ccache when it is on PATH. +# No-op locally unless ccache is installed; CI installs it via ccache-action. +Import("env") + +import shutil + +ccache = shutil.which("ccache") +if ccache: + + def wrap(tool): + cmd = env.get(tool) + if not cmd: + return + resolved = env.subst(cmd) if isinstance(cmd, str) else cmd + if isinstance(resolved, str): + if resolved.startswith("ccache"): + return + env[tool] = f"{ccache} {resolved}" + return + if isinstance(resolved, (list, tuple)): + if resolved and "ccache" in str(resolved[0]): + return + env[tool] = [ccache, *resolved] + + wrap("CC") + wrap("CXX") + print("ccache enabled") diff --git a/scripts/upload.py b/scripts/upload.py index 19be74710..2d228b1fd 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -14,12 +14,17 @@ # and # extra_scripts = scripts/upload.py # +# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency +# and can stall on IPv6 before falling back to IPv4. + +import hashlib +import os +import socket +import time +from urllib.parse import urlparse import requests -import hashlib -from urllib.parse import urlparse -import time -import os +from requests.adapters import HTTPAdapter Import("env") @@ -35,6 +40,10 @@ except ImportError: from tqdm import tqdm from termcolor import cprint +# http.client/urllib3 default is 8–16 KiB, which starves a LAN TCP window. +UPLOAD_BLOCK_SIZE = 64 * 1024 +UPLOAD_SNDBUF = 256 * 1024 + def print_success(x): cprint(x, 'green') @@ -44,6 +53,43 @@ def print_fail(x): cprint(f'Error: {x}', 'red') +def resolve_ipv4(host): + """Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request.""" + if host.startswith('['): + return host + if ':' in host and host.count(':') == 1: + host = host.rsplit(':', 1)[0] + try: + socket.inet_pton(socket.AF_INET, host) + return host + except OSError: + pass + infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM) + if not infos: + raise socket.gaierror(f'No IPv4 address for {host}') + return infos[0][4][0] + + +class FastHTTPAdapter(HTTPAdapter): + """Larger write chunks and socket send buffer for streaming POSTs.""" + + def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs): + socket_options = [ + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1), + (socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF), + ] + pool_kwargs.setdefault('socket_options', socket_options) + pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE) + super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs) + + +def make_session(): + session = requests.Session() + adapter = FastHTTPAdapter(pool_connections=2, pool_maxsize=2) + session.mount('http://', adapter) + return session + + def build_headers(host_ip, emsesp_url, content_type='application/json', access_token=None, extra_headers=None): """Build common HTTP headers with optional overrides.""" headers = { @@ -51,18 +97,17 @@ def build_headers(host_ip, emsesp_url, content_type='application/json', access_t 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0', 'Accept': '*/*', 'Accept-Language': 'en-US', - 'Accept-Encoding': 'gzip, deflate', 'Referer': emsesp_url, 'Content-Type': content_type, 'Connection': 'keep-alive' } - + if access_token: headers['Authorization'] = f'Bearer {access_token}' - + if extra_headers: headers.update(extra_headers) - + return headers @@ -79,44 +124,52 @@ def on_upload(source, target, env): username = env.GetProjectOption('custom_username') password = env.GetProjectOption('custom_password') emsesp_ip = env.GetProjectOption('custom_emsesp_ip') - except Exception as e: + except Exception: print_fail(f'Missing settings. Add these to your pio_local.ini file:\n\ncustom_username=username\ncustom_password=password\ncustom_emsesp_ip=ems-esp.local\n') return - emsesp_url = f"http://{emsesp_ip}" - parsed_url = urlparse(emsesp_url) - host_ip = parsed_url.netloc - - signon_url = f"{emsesp_url}/rest/signIn" - signon_headers = build_headers(host_ip, emsesp_url) - - username_password = { - "username": username, - "password": password - } - - response = requests.post( - signon_url, json=username_password, headers=signon_headers) - - if response.status_code != 200: - print_fail("Authentication with EMS-ESP failed (code " + - str(response.status_code) + ")") + parsed_url = urlparse(f"http://{emsesp_ip}") + host_name = parsed_url.hostname or emsesp_ip + try: + host_ip = resolve_ipv4(host_name) + except OSError as e: + print_fail(f"Could not resolve {host_name} to IPv4: {e}") return - print_success("Authentication with EMS-ESP successful") - access_token = response.json().get('access_token') + if host_ip != host_name: + print_success(f"Resolved {host_name} -> {host_ip}") - # start the upload - firmware_path = str(source[0]) + emsesp_url = f"http://{host_ip}" + session = make_session() - with open(firmware_path, 'rb') as firmware: - md5 = hashlib.md5(firmware.read()).hexdigest() + try: + signon_url = f"{emsesp_url}/rest/signIn" + signon_headers = build_headers(host_ip, emsesp_url) - firmware.seek(0) + response = session.post( + signon_url, + json={"username": username, "password": password}, + headers=signon_headers, + timeout=15) + + if response.status_code != 200: + print_fail("Authentication with EMS-ESP failed (code " + + str(response.status_code) + ")") + return + + print_success("Authentication with EMS-ESP successful") + access_token = response.json().get('access_token') + + firmware_path = str(source[0]) + with open(firmware_path, 'rb') as firmware_file: + firmware = firmware_file.read() + + md5 = hashlib.md5(firmware).hexdigest() + filename = os.path.basename(firmware_path) encoder = MultipartEncoder(fields={ 'MD5': md5, - 'file': (firmware_path, firmware, 'application/octet-stream')} + 'file': (filename, firmware, 'application/octet-stream')} ) bar = tqdm(desc='Upload Progress', @@ -124,7 +177,8 @@ def on_upload(source, target, env): dynamic_ncols=True, unit='B', unit_scale=True, - unit_divisor=1024 + unit_divisor=1024, + mininterval=0.2 ) monitor = MultipartEncoderMonitor( @@ -142,27 +196,35 @@ def on_upload(source, target, env): ) upload_url = f"{emsesp_url}/rest/uploadFile" - - response = requests.post( - upload_url, data=monitor, headers=post_headers) + started = time.perf_counter() + response = session.post( + upload_url, data=monitor, headers=post_headers, timeout=None) + elapsed = time.perf_counter() - started bar.close() - time.sleep(0.1) - print() if response.status_code != 200: print_fail(f"Upload failed (code {response.status_code}).") else: - print_success("Upload successful. Rebooting device.") + kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0 + print_success( + f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.") restart_headers = build_headers( host_ip, emsesp_url, access_token=access_token) restart_url = f"{emsesp_url}/api/system/restart" - response = requests.get(restart_url, headers=restart_headers) - if response.status_code != 200: - print_fail(f"Restart failed (code {response.status_code})") + try: + response = session.get( + restart_url, headers=restart_headers, timeout=10) + if response.status_code != 200: + print_fail(f"Restart failed (code {response.status_code})") + except requests.RequestException: + # Device often drops the connection as it reboots — that's OK. + pass print() + finally: + session.close() if env.get('UPLOAD_PROTOCOL') == 'custom': diff --git a/scripts/upload_cli.py b/scripts/upload_cli.py index a6ed77681..c002009a2 100644 --- a/scripts/upload_cli.py +++ b/scripts/upload_cli.py @@ -9,21 +9,27 @@ # # Run using for example: # python3 upload_cli.py -i 10.10.10.175 -f ../build/firmware/EMS-ESP-3_7_0-dev_34-ESP32S3-16MB+.bin +# +# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency +# and can stall on IPv6 before falling back to IPv4. import argparse import hashlib +import socket import sys import time from pathlib import Path +from urllib.parse import urlparse import requests +from requests.adapters import HTTPAdapter from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor from termcolor import cprint from tqdm import tqdm -# Constants USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0' -CHUNK_SIZE = 8192 # 8KB chunks for MD5 calculation +UPLOAD_BLOCK_SIZE = 64 * 1024 +UPLOAD_SNDBUF = 256 * 1024 def print_success(x): @@ -34,13 +40,34 @@ def print_fail(x): return cprint(x, 'red') -def calculate_md5(file_path): - """Calculate MD5 hash of a file in chunks for memory efficiency.""" - md5_hash = hashlib.md5() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(CHUNK_SIZE), b''): - md5_hash.update(chunk) - return md5_hash.hexdigest() +def resolve_ipv4(host): + """Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request.""" + if host.startswith('['): + return host + if ':' in host and host.count(':') == 1: + host = host.rsplit(':', 1)[0] + try: + socket.inet_pton(socket.AF_INET, host) + return host + except OSError: + pass + infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM) + if not infos: + raise socket.gaierror(f'No IPv4 address for {host}') + return infos[0][4][0] + + +class FastHTTPAdapter(HTTPAdapter): + """Larger write chunks and socket send buffer for streaming POSTs.""" + + def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs): + socket_options = [ + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1), + (socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF), + ] + pool_kwargs.setdefault('socket_options', socket_options) + pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE) + super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs) def create_base_headers(host_ip, emsesp_url): @@ -50,7 +77,6 @@ def create_base_headers(host_ip, emsesp_url): 'User-Agent': USER_AGENT, 'Accept': '*/*', 'Accept-Language': 'en-US', - 'Accept-Encoding': 'gzip, deflate', 'Referer': emsesp_url, 'Connection': 'keep-alive' } @@ -58,35 +84,39 @@ def create_base_headers(host_ip, emsesp_url): def upload(file, ip, username, password): """Upload firmware to EMS-ESP device.""" - # Print welcome message print() print("EMS-ESP Firmware Upload") - # Validate file exists file_path = Path(file) if not file_path.exists(): print_fail(f"File not found: {file}") return - # Setup URLs and headers - emsesp_url = f"http://{ip}" - host_ip = ip - - # Use a session for connection pooling and persistence + parsed_url = urlparse(f"http://{ip}") + host_name = parsed_url.hostname or ip + try: + host_ip = resolve_ipv4(host_name) + except OSError as e: + print_fail(f"Could not resolve {host_name} to IPv4: {e}") + return + + if host_ip != host_name: + print_success(f"Resolved {host_name} -> {host_ip}") + + emsesp_url = f"http://{host_ip}" session = requests.Session() - + session.mount('http://', FastHTTPAdapter(pool_connections=2, pool_maxsize=2)) + try: - # Authentication signon_url = f"{emsesp_url}/rest/signIn" signon_headers = create_base_headers(host_ip, emsesp_url) signon_headers['Content-Type'] = 'application/json' - username_password = { - "username": username, - "password": password - } - - response = session.post(signon_url, json=username_password, headers=signon_headers) + response = session.post( + signon_url, + json={"username": username, "password": password}, + headers=signon_headers, + timeout=15) if response.status_code != 200: print_fail(f"Authentication failed (code {response.status_code})") @@ -95,56 +125,62 @@ def upload(file, ip, username, password): print_success("Authentication successful") access_token = response.json().get('access_token') - # Calculate MD5 hash - print("Calculating MD5 hash...") - md5 = calculate_md5(file_path) + firmware = file_path.read_bytes() + md5 = hashlib.md5(firmware).hexdigest() - # Start the upload - with open(file_path, 'rb') as firmware: - encoder = MultipartEncoder(fields={ - 'MD5': md5, - 'file': (file, firmware, 'application/octet-stream') - }) + encoder = MultipartEncoder(fields={ + 'MD5': md5, + 'file': (file_path.name, firmware, 'application/octet-stream') + }) - bar = tqdm( - desc='Upload Progress', - total=encoder.len, - dynamic_ncols=True, - unit='B', - unit_scale=True, - unit_divisor=1024 - ) + bar = tqdm( + desc='Upload Progress', + total=encoder.len, + dynamic_ncols=True, + unit='B', + unit_scale=True, + unit_divisor=1024, + mininterval=0.2 + ) - monitor = MultipartEncoderMonitor( - encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n)) + monitor = MultipartEncoderMonitor( + encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n)) - post_headers = create_base_headers(host_ip, emsesp_url) - post_headers.update({ - 'Content-Type': monitor.content_type, - 'Content-Length': str(monitor.len), - 'Origin': emsesp_url, + post_headers = create_base_headers(host_ip, emsesp_url) + post_headers.update({ + 'Content-Type': monitor.content_type, + 'Content-Length': str(monitor.len), + 'Origin': emsesp_url, + 'Authorization': f'Bearer {access_token}' + }) + + upload_url = f"{emsesp_url}/rest/uploadFile" + started = time.perf_counter() + response = session.post( + upload_url, data=monitor, headers=post_headers, timeout=None) + elapsed = time.perf_counter() - started + + bar.close() + + if response.status_code != 200: + print_fail(f"Upload failed (code {response.status_code})") + else: + kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0 + print_success( + f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.") + restart_headers = create_base_headers(host_ip, emsesp_url) + restart_headers.update({ + 'Content-Type': 'application/json', 'Authorization': f'Bearer {access_token}' }) - - upload_url = f"{emsesp_url}/rest/uploadFile" - response = session.post(upload_url, data=monitor, headers=post_headers) - - bar.close() - time.sleep(0.1) - - if response.status_code != 200: - print_fail(f"Upload failed (code {response.status_code})") - else: - print_success("Upload successful. Rebooting device.") - restart_headers = create_base_headers(host_ip, emsesp_url) - restart_headers.update({ - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {access_token}' - }) - restart_url = f"{emsesp_url}/api/system/restart" - response = session.get(restart_url, headers=restart_headers) + restart_url = f"{emsesp_url}/api/system/restart" + try: + response = session.get( + restart_url, headers=restart_headers, timeout=10) if response.status_code != 200: print_fail(f"Restart failed (code {response.status_code})") + except requests.RequestException: + pass except requests.RequestException as e: print_fail(f"Network error: {e}") @@ -161,7 +197,6 @@ def upload(file, ip, username, password): print() -# main parser = argparse.ArgumentParser(description="EMS-ESP Firmware Upload") parser.add_argument("-f", "--file", metavar="FILE", required=True, type=str, help="firmware file")