test new esp32-arduino-lib-builder packages

scripts/enable_ccache.py

@@ -0,0 +1,27 @@
+# Wrap the C/C++ compilers with ccache when it is on PATH.
+# No-op locally unless ccache is installed; CI installs it via ccache-action.
+Import("env")
+
+import shutil
+
+ccache = shutil.which("ccache")
+if ccache:
+
+    def wrap(tool):
+        cmd = env.get(tool)
+        if not cmd:
+            return
+        resolved = env.subst(cmd) if isinstance(cmd, str) else cmd
+        if isinstance(resolved, str):
+            if resolved.startswith("ccache"):
+                return
+            env[tool] = f"{ccache} {resolved}"
+            return
+        if isinstance(resolved, (list, tuple)):
+            if resolved and "ccache" in str(resolved[0]):
+                return
+            env[tool] = [ccache, *resolved]
+
+    wrap("CC")
+    wrap("CXX")
+    print("ccache enabled")

scripts/upload.py

@@ -14,12 +14,17 @@
 # and
 #  extra_scripts = scripts/upload.py
 #
+# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency
+# and can stall on IPv6 before falling back to IPv4.
+
+import hashlib
+import os
+import socket
+import time
+from urllib.parse import urlparse
 
 import requests
-import hashlib
-from urllib.parse import urlparse
-import time
-import os
+from requests.adapters import HTTPAdapter
 
 Import("env")
 
@@ -35,6 +40,10 @@ except ImportError:
     from tqdm import tqdm
     from termcolor import cprint
 
+# http.client/urllib3 default is 8–16 KiB, which starves a LAN TCP window.
+UPLOAD_BLOCK_SIZE = 64 * 1024
+UPLOAD_SNDBUF = 256 * 1024
+
 
 def print_success(x):
     cprint(x, 'green')
@@ -44,6 +53,43 @@ def print_fail(x):
     cprint(f'Error: {x}', 'red')
 
 
+def resolve_ipv4(host):
+    """Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request."""
+    if host.startswith('['):
+        return host
+    if ':' in host and host.count(':') == 1:
+        host = host.rsplit(':', 1)[0]
+    try:
+        socket.inet_pton(socket.AF_INET, host)
+        return host
+    except OSError:
+        pass
+    infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM)
+    if not infos:
+        raise socket.gaierror(f'No IPv4 address for {host}')
+    return infos[0][4][0]
+
+
+class FastHTTPAdapter(HTTPAdapter):
+    """Larger write chunks and socket send buffer for streaming POSTs."""
+
+    def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
+        socket_options = [
+            (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
+            (socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF),
+        ]
+        pool_kwargs.setdefault('socket_options', socket_options)
+        pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE)
+        super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
+
+
+def make_session():
+    session = requests.Session()
+    adapter = FastHTTPAdapter(pool_connections=2, pool_maxsize=2)
+    session.mount('http://', adapter)
+    return session
+
+
 def build_headers(host_ip, emsesp_url, content_type='application/json', access_token=None, extra_headers=None):
     """Build common HTTP headers with optional overrides."""
     headers = {
@@ -51,18 +97,17 @@ def build_headers(host_ip, emsesp_url, content_type='application/json', access_t
         'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0',
         'Accept': '*/*',
         'Accept-Language': 'en-US',
-        'Accept-Encoding': 'gzip, deflate',
         'Referer': emsesp_url,
         'Content-Type': content_type,
         'Connection': 'keep-alive'
     }
-    
+
     if access_token:
         headers['Authorization'] = f'Bearer {access_token}'
-    
+
     if extra_headers:
         headers.update(extra_headers)
-    
+
     return headers
 
 
@@ -79,44 +124,52 @@ def on_upload(source, target, env):
         username = env.GetProjectOption('custom_username')
         password = env.GetProjectOption('custom_password')
         emsesp_ip = env.GetProjectOption('custom_emsesp_ip')
-    except Exception as e:
+    except Exception:
         print_fail(f'Missing settings. Add these to your pio_local.ini file:\n\ncustom_username=username\ncustom_password=password\ncustom_emsesp_ip=ems-esp.local\n')
         return
 
-    emsesp_url = f"http://{emsesp_ip}"
-    parsed_url = urlparse(emsesp_url)
-    host_ip = parsed_url.netloc
-
-    signon_url = f"{emsesp_url}/rest/signIn"
-    signon_headers = build_headers(host_ip, emsesp_url)
-
-    username_password = {
-        "username": username,
-        "password": password
-    }
-
-    response = requests.post(
-        signon_url, json=username_password, headers=signon_headers)
-
-    if response.status_code != 200:
-        print_fail("Authentication with EMS-ESP failed (code " +
-                   str(response.status_code) + ")")
+    parsed_url = urlparse(f"http://{emsesp_ip}")
+    host_name = parsed_url.hostname or emsesp_ip
+    try:
+        host_ip = resolve_ipv4(host_name)
+    except OSError as e:
+        print_fail(f"Could not resolve {host_name} to IPv4: {e}")
         return
 
-    print_success("Authentication with EMS-ESP successful")
-    access_token = response.json().get('access_token')
+    if host_ip != host_name:
+        print_success(f"Resolved {host_name} -> {host_ip}")
 
-    # start the upload
-    firmware_path = str(source[0])
+    emsesp_url = f"http://{host_ip}"
+    session = make_session()
 
-    with open(firmware_path, 'rb') as firmware:
-        md5 = hashlib.md5(firmware.read()).hexdigest()
+    try:
+        signon_url = f"{emsesp_url}/rest/signIn"
+        signon_headers = build_headers(host_ip, emsesp_url)
 
-        firmware.seek(0)
+        response = session.post(
+            signon_url,
+            json={"username": username, "password": password},
+            headers=signon_headers,
+            timeout=15)
+
+        if response.status_code != 200:
+            print_fail("Authentication with EMS-ESP failed (code " +
+                       str(response.status_code) + ")")
+            return
+
+        print_success("Authentication with EMS-ESP successful")
+        access_token = response.json().get('access_token')
+
+        firmware_path = str(source[0])
+        with open(firmware_path, 'rb') as firmware_file:
+            firmware = firmware_file.read()
+
+        md5 = hashlib.md5(firmware).hexdigest()
+        filename = os.path.basename(firmware_path)
 
         encoder = MultipartEncoder(fields={
             'MD5': md5,
-            'file': (firmware_path, firmware, 'application/octet-stream')}
+            'file': (filename, firmware, 'application/octet-stream')}
         )
 
         bar = tqdm(desc='Upload Progress',
@@ -124,7 +177,8 @@ def on_upload(source, target, env):
                    dynamic_ncols=True,
                    unit='B',
                    unit_scale=True,
-                   unit_divisor=1024
+                   unit_divisor=1024,
+                   mininterval=0.2
                    )
 
         monitor = MultipartEncoderMonitor(
@@ -142,27 +196,35 @@ def on_upload(source, target, env):
         )
 
         upload_url = f"{emsesp_url}/rest/uploadFile"
-
-        response = requests.post(
-            upload_url, data=monitor, headers=post_headers)
+        started = time.perf_counter()
+        response = session.post(
+            upload_url, data=monitor, headers=post_headers, timeout=None)
+        elapsed = time.perf_counter() - started
 
         bar.close()
-        time.sleep(0.1)
-
         print()
 
         if response.status_code != 200:
             print_fail(f"Upload failed (code {response.status_code}).")
         else:
-            print_success("Upload successful. Rebooting device.")
+            kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0
+            print_success(
+                f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.")
             restart_headers = build_headers(
                 host_ip, emsesp_url, access_token=access_token)
             restart_url = f"{emsesp_url}/api/system/restart"
-            response = requests.get(restart_url, headers=restart_headers)
-            if response.status_code != 200:
-                print_fail(f"Restart failed (code {response.status_code})")
+            try:
+                response = session.get(
+                    restart_url, headers=restart_headers, timeout=10)
+                if response.status_code != 200:
+                    print_fail(f"Restart failed (code {response.status_code})")
+            except requests.RequestException:
+                # Device often drops the connection as it reboots — that's OK.
+                pass
 
         print()
+    finally:
+        session.close()
 
 
 if env.get('UPLOAD_PROTOCOL') == 'custom':

scripts/upload_cli.py

@@ -9,21 +9,27 @@
 #
 # Run using for example:
 #  python3 upload_cli.py -i 10.10.10.175 -f ../build/firmware/EMS-ESP-3_7_0-dev_34-ESP32S3-16MB+.bin
+#
+# Prefer a numeric IP over .local — mDNS (especially under WSL2) adds latency
+# and can stall on IPv6 before falling back to IPv4.
 
 import argparse
 import hashlib
+import socket
 import sys
 import time
 from pathlib import Path
+from urllib.parse import urlparse
 
 import requests
+from requests.adapters import HTTPAdapter
 from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
 from termcolor import cprint
 from tqdm import tqdm
 
-# Constants
 USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0'
-CHUNK_SIZE = 8192  # 8KB chunks for MD5 calculation
+UPLOAD_BLOCK_SIZE = 64 * 1024
+UPLOAD_SNDBUF = 256 * 1024
 
 
 def print_success(x):
@@ -34,13 +40,34 @@ def print_fail(x):
     return cprint(x, 'red')
 
 
-def calculate_md5(file_path):
-    """Calculate MD5 hash of a file in chunks for memory efficiency."""
-    md5_hash = hashlib.md5()
-    with open(file_path, 'rb') as f:
-        for chunk in iter(lambda: f.read(CHUNK_SIZE), b''):
-            md5_hash.update(chunk)
-    return md5_hash.hexdigest()
+def resolve_ipv4(host):
+    """Resolve hostname to IPv4 so we skip mDNS/IPv6 retries on every request."""
+    if host.startswith('['):
+        return host
+    if ':' in host and host.count(':') == 1:
+        host = host.rsplit(':', 1)[0]
+    try:
+        socket.inet_pton(socket.AF_INET, host)
+        return host
+    except OSError:
+        pass
+    infos = socket.getaddrinfo(host, 80, socket.AF_INET, socket.SOCK_STREAM)
+    if not infos:
+        raise socket.gaierror(f'No IPv4 address for {host}')
+    return infos[0][4][0]
+
+
+class FastHTTPAdapter(HTTPAdapter):
+    """Larger write chunks and socket send buffer for streaming POSTs."""
+
+    def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
+        socket_options = [
+            (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
+            (socket.SOL_SOCKET, socket.SO_SNDBUF, UPLOAD_SNDBUF),
+        ]
+        pool_kwargs.setdefault('socket_options', socket_options)
+        pool_kwargs.setdefault('blocksize', UPLOAD_BLOCK_SIZE)
+        super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
 
 
 def create_base_headers(host_ip, emsesp_url):
@@ -50,7 +77,6 @@ def create_base_headers(host_ip, emsesp_url):
         'User-Agent': USER_AGENT,
         'Accept': '*/*',
         'Accept-Language': 'en-US',
-        'Accept-Encoding': 'gzip, deflate',
         'Referer': emsesp_url,
         'Connection': 'keep-alive'
     }
@@ -58,35 +84,39 @@ def create_base_headers(host_ip, emsesp_url):
 
 def upload(file, ip, username, password):
     """Upload firmware to EMS-ESP device."""
-    # Print welcome message
     print()
     print("EMS-ESP Firmware Upload")
 
-    # Validate file exists
     file_path = Path(file)
     if not file_path.exists():
         print_fail(f"File not found: {file}")
         return
 
-    # Setup URLs and headers
-    emsesp_url = f"http://{ip}"
-    host_ip = ip
-    
-    # Use a session for connection pooling and persistence
+    parsed_url = urlparse(f"http://{ip}")
+    host_name = parsed_url.hostname or ip
+    try:
+        host_ip = resolve_ipv4(host_name)
+    except OSError as e:
+        print_fail(f"Could not resolve {host_name} to IPv4: {e}")
+        return
+
+    if host_ip != host_name:
+        print_success(f"Resolved {host_name} -> {host_ip}")
+
+    emsesp_url = f"http://{host_ip}"
     session = requests.Session()
-    
+    session.mount('http://', FastHTTPAdapter(pool_connections=2, pool_maxsize=2))
+
     try:
-        # Authentication
         signon_url = f"{emsesp_url}/rest/signIn"
         signon_headers = create_base_headers(host_ip, emsesp_url)
         signon_headers['Content-Type'] = 'application/json'
 
-        username_password = {
-            "username": username,
-            "password": password
-        }
-
-        response = session.post(signon_url, json=username_password, headers=signon_headers)
+        response = session.post(
+            signon_url,
+            json={"username": username, "password": password},
+            headers=signon_headers,
+            timeout=15)
 
         if response.status_code != 200:
             print_fail(f"Authentication failed (code {response.status_code})")
@@ -95,56 +125,62 @@ def upload(file, ip, username, password):
         print_success("Authentication successful")
         access_token = response.json().get('access_token')
 
-        # Calculate MD5 hash
-        print("Calculating MD5 hash...")
-        md5 = calculate_md5(file_path)
+        firmware = file_path.read_bytes()
+        md5 = hashlib.md5(firmware).hexdigest()
 
-        # Start the upload
-        with open(file_path, 'rb') as firmware:
-            encoder = MultipartEncoder(fields={
-                'MD5': md5,
-                'file': (file, firmware, 'application/octet-stream')
-            })
+        encoder = MultipartEncoder(fields={
+            'MD5': md5,
+            'file': (file_path.name, firmware, 'application/octet-stream')
+        })
 
-            bar = tqdm(
-                desc='Upload Progress',
-                total=encoder.len,
-                dynamic_ncols=True,
-                unit='B',
-                unit_scale=True,
-                unit_divisor=1024
-            )
+        bar = tqdm(
+            desc='Upload Progress',
+            total=encoder.len,
+            dynamic_ncols=True,
+            unit='B',
+            unit_scale=True,
+            unit_divisor=1024,
+            mininterval=0.2
+        )
 
-            monitor = MultipartEncoderMonitor(
-                encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n))
+        monitor = MultipartEncoderMonitor(
+            encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n))
 
-            post_headers = create_base_headers(host_ip, emsesp_url)
-            post_headers.update({
-                'Content-Type': monitor.content_type,
-                'Content-Length': str(monitor.len),
-                'Origin': emsesp_url,
+        post_headers = create_base_headers(host_ip, emsesp_url)
+        post_headers.update({
+            'Content-Type': monitor.content_type,
+            'Content-Length': str(monitor.len),
+            'Origin': emsesp_url,
+            'Authorization': f'Bearer {access_token}'
+        })
+
+        upload_url = f"{emsesp_url}/rest/uploadFile"
+        started = time.perf_counter()
+        response = session.post(
+            upload_url, data=monitor, headers=post_headers, timeout=None)
+        elapsed = time.perf_counter() - started
+
+        bar.close()
+
+        if response.status_code != 200:
+            print_fail(f"Upload failed (code {response.status_code})")
+        else:
+            kibs = (encoder.len / 1024) / elapsed if elapsed > 0 else 0
+            print_success(
+                f"Upload successful ({kibs:.0f} KiB/s in {elapsed:.1f}s). Rebooting device.")
+            restart_headers = create_base_headers(host_ip, emsesp_url)
+            restart_headers.update({
+                'Content-Type': 'application/json',
                 'Authorization': f'Bearer {access_token}'
             })
-
-            upload_url = f"{emsesp_url}/rest/uploadFile"
-            response = session.post(upload_url, data=monitor, headers=post_headers)
-
-            bar.close()
-            time.sleep(0.1)
-
-            if response.status_code != 200:
-                print_fail(f"Upload failed (code {response.status_code})")
-            else:
-                print_success("Upload successful. Rebooting device.")
-                restart_headers = create_base_headers(host_ip, emsesp_url)
-                restart_headers.update({
-                    'Content-Type': 'application/json',
-                    'Authorization': f'Bearer {access_token}'
-                })
-                restart_url = f"{emsesp_url}/api/system/restart"
-                response = session.get(restart_url, headers=restart_headers)
+            restart_url = f"{emsesp_url}/api/system/restart"
+            try:
+                response = session.get(
+                    restart_url, headers=restart_headers, timeout=10)
                 if response.status_code != 200:
                     print_fail(f"Restart failed (code {response.status_code})")
+            except requests.RequestException:
+                pass
 
     except requests.RequestException as e:
         print_fail(f"Network error: {e}")
@@ -161,7 +197,6 @@ def upload(file, ip, username, password):
     print()
 
 
-# main
 parser = argparse.ArgumentParser(description="EMS-ESP Firmware Upload")
 parser.add_argument("-f", "--file", metavar="FILE",
                     required=True, type=str, help="firmware file")