diff --git a/CHANGELOG_LATEST.md b/CHANGELOG_LATEST.md index 3e03cf0e0..a75000324 100644 --- a/CHANGELOG_LATEST.md +++ b/CHANGELOG_LATEST.md @@ -17,6 +17,7 @@ - heatpump entities `fan` and `shutdown` [#1690](https://github.com/emsesp/EMS-ESP32/discussions/1690) - mqtt HA-mode 3 for v3.6 compatible HA entities, set on update v3.6->v3.7 - HP input states [#1723](https://github.com/emsesp/EMS-ESP32/discussions/1723) +- Added scripts for OTA (scripts/upload.py and upload_cli.py) [#1738](https://github.com/emsesp/EMS-ESP32/issues/1738) ## Fixed @@ -31,3 +32,4 @@ - Refresh UI - moving settings to one location [#1665](https://github.com/emsesp/EMS-ESP32/issues/1665) - rename DeviceValueTypes, add UINT32 for custom entities - dynamic register dhw circuits for thermostat +- removed OTA feature [#1738](https://github.com/emsesp/EMS-ESP32/issues/1738) diff --git a/factory_settings.ini b/factory_settings.ini index b423605ca..de76cc0f2 100644 --- a/factory_settings.ini +++ b/factory_settings.ini @@ -25,11 +25,6 @@ build_flags = -D FACTORY_NTP_TIME_ZONE_FORMAT=\"CET-1CEST,M3.5.0,M10.5.0/3\" -D FACTORY_NTP_SERVER=\"time.google.com\" - ; OTA settings - -D FACTORY_OTA_PORT=8266 - -D FACTORY_OTA_PASSWORD=\"ems-esp-neo\" - -D FACTORY_OTA_ENABLED=false - ; MQTT settings -D FACTORY_MQTT_ENABLED=false -D FACTORY_MQTT_HOST=\"\" diff --git a/interface/src/AuthenticatedRouting.tsx b/interface/src/AuthenticatedRouting.tsx index 664643a15..00ca0d836 100644 --- a/interface/src/AuthenticatedRouting.tsx +++ b/interface/src/AuthenticatedRouting.tsx @@ -8,7 +8,6 @@ import AccessPoint from 'framework/ap/AccessPoint'; import Mqtt from 'framework/mqtt/Mqtt'; import Network from 'framework/network/Network'; import NetworkTime from 'framework/ntp/NetworkTime'; -import OTASettings from 'framework/ota/OTASettings'; import Security from 'framework/security/Security'; import ESPSystemStatus from 'framework/system/ESPSystemStatus'; import System from 'framework/system/System'; @@ -43,7 +42,6 @@ const AuthenticatedRouting: FC = () => { } /> } /> } /> - } /> } /> alovaInstance.Post('/rest/restart'); export const partition = () => alovaInstance.Post('/rest/partition'); export const factoryReset = () => alovaInstance.Post('/rest/factoryReset'); -// OTA -export const readOTASettings = () => - alovaInstance.Get(`/rest/otaSettings`); -export const updateOTASettings = (data: OTASettings) => - alovaInstance.Post('/rest/otaSettings', data); - // SystemLog export const readLogSettings = () => alovaInstance.Get(`/rest/logSettings`); diff --git a/interface/src/framework/Settings.tsx b/interface/src/framework/Settings.tsx index e6e09a34e..6164406ad 100644 --- a/interface/src/framework/Settings.tsx +++ b/interface/src/framework/Settings.tsx @@ -3,7 +3,6 @@ import { toast } from 'react-toastify'; import AccessTimeIcon from '@mui/icons-material/AccessTime'; import CancelIcon from '@mui/icons-material/Cancel'; -import CastIcon from '@mui/icons-material/Cast'; import DeviceHubIcon from '@mui/icons-material/DeviceHub'; import ImportExportIcon from '@mui/icons-material/ImportExport'; import LockIcon from '@mui/icons-material/Lock'; @@ -212,13 +211,7 @@ const Settings: FC = () => { text={LL.CONFIGURE('MQTT')} to="mqtt" /> - + { - const { - loadData, - saveData, - saving, - updateDataValue, - data, - origData, - dirtyFlags, - setDirtyFlags, - blocker, - errorMessage - } = useRest({ - read: SystemApi.readOTASettings, - update: SystemApi.updateOTASettings - }); - - const { LL } = useI18nContext(); - - const updateFormValue = updateValueDirty( - origData, - dirtyFlags, - setDirtyFlags, - updateDataValue - ); - - const [fieldErrors, setFieldErrors] = useState(); - - const content = () => { - if (!data) { - return ; - } - - const validateAndSubmit = async () => { - try { - setFieldErrors(undefined); - await validate(OTA_SETTINGS_VALIDATOR, data); - await saveData(); - } catch (error) { - setFieldErrors(error as ValidateFieldsError); - } - }; - - useLayoutTitle('OTA'); - - return ( - <> - - } - label={LL.ENABLE_OTA()} - /> - - - {dirtyFlags && dirtyFlags.length !== 0 && ( - - - - - )} - - ); - }; - - return ( - - {blocker ? : null} - {content()} - - ); -}; - -export default OTASettings; diff --git a/interface/src/framework/system/SystemStatus.tsx b/interface/src/framework/system/SystemStatus.tsx index 8a83ef884..1b023cde6 100644 --- a/interface/src/framework/system/SystemStatus.tsx +++ b/interface/src/framework/system/SystemStatus.tsx @@ -4,7 +4,6 @@ import { toast } from 'react-toastify'; import AccessTimeIcon from '@mui/icons-material/AccessTime'; import BuildIcon from '@mui/icons-material/Build'; import CancelIcon from '@mui/icons-material/Cancel'; -import CastIcon from '@mui/icons-material/Cast'; import DeviceHubIcon from '@mui/icons-material/DeviceHub'; import DirectionsBusIcon from '@mui/icons-material/DirectionsBus'; import MemoryIcon from '@mui/icons-material/Memory'; @@ -277,20 +276,10 @@ const SystemStatus: FC = () => { /> - - - on(SIGN_IN_PATH, [this](AsyncWebServerRequest * request, JsonVariant json) { signIn(request, json); }); } -/** - * Verifies that the request supplied a valid JWT. - */ +// Verifies that the request supplied a valid JWT. void AuthenticationService::verifyAuthorization(AsyncWebServerRequest * request) { Authentication authentication = _securityManager->authenticateRequest(request); request->send(authentication.authenticated ? 200 : 401); } -/** - * Signs in a user if the username and password match. Provides a JWT to be used in the Authorization header in - * subsequent requests. - */ +// Signs in a user if the username and password match. Provides a JWT to be used in the Authorization header in subsequent requests. void AuthenticationService::signIn(AsyncWebServerRequest * request, JsonVariant json) { if (json.is()) { String username = json["username"]; @@ -33,6 +28,7 @@ void AuthenticationService::signIn(AsyncWebServerRequest * request, JsonVariant return; } } + AsyncWebServerResponse * response = request->beginResponse(401); request->send(response); } diff --git a/lib/framework/ESP8266React.cpp b/lib/framework/ESP8266React.cpp index be7baf8f1..12036b759 100644 --- a/lib/framework/ESP8266React.cpp +++ b/lib/framework/ESP8266React.cpp @@ -11,7 +11,6 @@ ESP8266React::ESP8266React(AsyncWebServer * server, FS * fs) , _apStatus(server, &_securitySettingsService, &_apSettingsService) , _ntpSettingsService(server, fs, &_securitySettingsService) , _ntpStatus(server, &_securitySettingsService) - , _otaSettingsService(server, fs, &_securitySettingsService) , _uploadFileService(server, &_securitySettingsService) , _mqttSettingsService(server, fs, &_securitySettingsService) , _mqttStatus(server, &_mqttSettingsService, &_securitySettingsService) @@ -76,7 +75,6 @@ void ESP8266React::begin() { }); _apSettingsService.begin(); _ntpSettingsService.begin(); - _otaSettingsService.begin(); _mqttSettingsService.begin(); _securitySettingsService.begin(); } @@ -84,6 +82,5 @@ void ESP8266React::begin() { void ESP8266React::loop() { _networkSettingsService.loop(); _apSettingsService.loop(); - _otaSettingsService.loop(); _mqttSettingsService.loop(); } \ No newline at end of file diff --git a/lib/framework/ESP8266React.h b/lib/framework/ESP8266React.h index 3ec012f10..1fdd69e5d 100644 --- a/lib/framework/ESP8266React.h +++ b/lib/framework/ESP8266React.h @@ -9,7 +9,6 @@ #include "MqttStatus.h" #include "NTPSettingsService.h" #include "NTPStatus.h" -#include "OTASettingsService.h" #include "UploadFileService.h" #include "RestartService.h" #include "SecuritySettingsService.h" @@ -48,10 +47,6 @@ class ESP8266React { return &_ntpSettingsService; } - StatefulService * getOTASettingsService() { - return &_otaSettingsService; - } - StatefulService * getMqttSettingsService() { return &_mqttSettingsService; } @@ -88,7 +83,6 @@ class ESP8266React { APStatus _apStatus; NTPSettingsService _ntpSettingsService; NTPStatus _ntpStatus; - OTASettingsService _otaSettingsService; UploadFileService _uploadFileService; MqttSettingsService _mqttSettingsService; MqttStatus _mqttStatus; diff --git a/lib/framework/Features.h b/lib/framework/Features.h index 1629b39ef..d5cccc997 100644 --- a/lib/framework/Features.h +++ b/lib/framework/Features.h @@ -21,15 +21,9 @@ #define FT_NTP 1 #endif -// ota feature on by default -#ifndef FT_OTA -#define FT_OTA 1 -#endif - // upload firmware/file feature on by default #ifndef FT_UPLOAD_FIRMWARE #define FT_UPLOAD_FIRMWARE 1 #endif - #endif diff --git a/lib/framework/OTASettingsService.cpp b/lib/framework/OTASettingsService.cpp deleted file mode 100644 index 61673092c..000000000 --- a/lib/framework/OTASettingsService.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "OTASettingsService.h" - -#include "../../src/emsesp_stub.hpp" - -OTASettingsService::OTASettingsService(AsyncWebServer * server, FS * fs, SecurityManager * securityManager) - : _httpEndpoint(OTASettings::read, OTASettings::update, this, server, OTA_SETTINGS_SERVICE_PATH, securityManager) - , _fsPersistence(OTASettings::read, OTASettings::update, this, fs, OTA_SETTINGS_FILE) - , _arduinoOTA(nullptr) { - WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { WiFiEvent(event); }); - addUpdateHandler([this] { configureArduinoOTA(); }, false); -} - -void OTASettingsService::begin() { - _fsPersistence.readFromFS(); - configureArduinoOTA(); -} - -void OTASettingsService::loop() { - if (_state.enabled && _arduinoOTA) { - _arduinoOTA->handle(); - } -} - -void OTASettingsService::configureArduinoOTA() { - if (_arduinoOTA) { - _arduinoOTA->end(); - delete _arduinoOTA; - _arduinoOTA = nullptr; - } - - if (_state.enabled) { - _arduinoOTA = new ArduinoOTAClass; - _arduinoOTA->setPort(static_cast(_state.port)); - _arduinoOTA->setPassword(_state.password.c_str()); - - _arduinoOTA->onStart([] { emsesp::EMSESP::system_.upload_status(true); }); - _arduinoOTA->onEnd([] { emsesp::EMSESP::system_.upload_status(false); }); - - _arduinoOTA->onProgress([](unsigned int progress, unsigned int total) { - // Serial.printf("Progress: %u%%\r\n", (progress / (total / 100))); - }); - _arduinoOTA->onError([](ota_error_t error) { - /* -#if defined(EMSESP_DEBUG) - Serial.printf("Error[%u]: ", error); - if (error == OTA_AUTH_ERROR) - Serial.println("Auth Failed"); - else if (error == OTA_BEGIN_ERROR) - Serial.println("Begin Failed"); - else if (error == OTA_CONNECT_ERROR) - Serial.println("Connect Failed"); - else if (error == OTA_RECEIVE_ERROR) - Serial.println("Receive Failed"); - else if (error == OTA_END_ERROR) - Serial.println("End Failed"); -#endif -*/ - }); - - _arduinoOTA->setMdnsEnabled(false); // disable as handled in NetworkSettingsService.cpp. https://github.com/emsesp/EMS-ESP32/issues/161 - _arduinoOTA->begin(); - } -} - -void OTASettingsService::WiFiEvent(WiFiEvent_t event) { - switch (event) { - case ARDUINO_EVENT_WIFI_STA_GOT_IP: - case ARDUINO_EVENT_ETH_GOT_IP: - configureArduinoOTA(); - break; - default: - break; - } -} - -void OTASettings::read(OTASettings & settings, JsonObject root) { - root["enabled"] = settings.enabled; - root["port"] = settings.port; - root["password"] = settings.password; -} - -StateUpdateResult OTASettings::update(JsonObject root, OTASettings & settings) { - settings.enabled = root["enabled"] | FACTORY_OTA_ENABLED; - settings.port = root["port"] | FACTORY_OTA_PORT; - settings.password = root["password"] | FACTORY_OTA_PASSWORD; - return StateUpdateResult::CHANGED; -} \ No newline at end of file diff --git a/lib/framework/OTASettingsService.h b/lib/framework/OTASettingsService.h deleted file mode 100644 index 41183dac6..000000000 --- a/lib/framework/OTASettingsService.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef OTASettingsService_h -#define OTASettingsService_h - -#include "HttpEndpoint.h" -#include "FSPersistence.h" - -#include -#include - -#ifndef FACTORY_OTA_PORT -#define FACTORY_OTA_PORT 8266 -#endif - -#ifndef FACTORY_OTA_PASSWORD -#define FACTORY_OTA_PASSWORD "ems-esp-neo" -#endif - -#ifndef FACTORY_OTA_ENABLED -#define FACTORY_OTA_ENABLED false -#endif - -#define OTA_SETTINGS_FILE "/config/otaSettings.json" -#define OTA_SETTINGS_SERVICE_PATH "/rest/otaSettings" - -class OTASettings { - public: - bool enabled; - int port; - String password; - - static void read(OTASettings & settings, JsonObject root); - static StateUpdateResult update(JsonObject root, OTASettings & settings); -}; - -class OTASettingsService : public StatefulService { - public: - OTASettingsService(AsyncWebServer * server, FS * fs, SecurityManager * securityManager); - - void begin(); - void loop(); - - private: - HttpEndpoint _httpEndpoint; - FSPersistence _fsPersistence; - ArduinoOTAClass * _arduinoOTA; - - void configureArduinoOTA(); - void WiFiEvent(WiFiEvent_t event); -}; - -#endif diff --git a/lib/framework/SecuritySettingsService.h b/lib/framework/SecuritySettingsService.h index 91f3f4b3b..11cdebbf4 100644 --- a/lib/framework/SecuritySettingsService.h +++ b/lib/framework/SecuritySettingsService.h @@ -71,7 +71,6 @@ class SecuritySettingsService final : public StatefulService, void begin(); - // Functions to implement SecurityManager Authentication authenticate(const String & username, const String & password) override; Authentication authenticateRequest(AsyncWebServerRequest * request) override; String generateJWT(const User * user) override; @@ -88,15 +87,9 @@ class SecuritySettingsService final : public StatefulService, void configureJWTHandler(); - /* - * Lookup the user by JWT - */ - Authentication authenticateJWT(String & jwt); + Authentication authenticateJWT(String & jwt); // Lookup the user by JWT - /* - * Verify the payload is correct - */ - boolean validatePayload(JsonObject parsedPayload, const User * user); + boolean validatePayload(JsonObject parsedPayload, const User * user); // Verify the payload is correct }; #endif \ No newline at end of file diff --git a/lib/framework/UploadFileService.cpp b/lib/framework/UploadFileService.cpp index 13a8e5a09..3ab3428c1 100644 --- a/lib/framework/UploadFileService.cpp +++ b/lib/framework/UploadFileService.cpp @@ -1,4 +1,5 @@ #include "UploadFileService.h" + #include "../../src/emsesp_stub.hpp" #include @@ -84,6 +85,7 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri Update.setMD5(_md5.data()); _md5.front() = '\0'; } + // emsesp::EMSESP::system_.upload_status(true); // force just in case, this is stop UART, MQTT and other services request->onDisconnect([this] { handleEarlyDisconnect(); }); // success, let's make sure we end the update if the client hangs up } else { handleError(request, 507); // failed to begin, send an error response Insufficient Storage @@ -101,11 +103,11 @@ void UploadFileService::handleUpload(AsyncWebServerRequest * request, const Stri } } else if (!request->_tempObject) { // if we haven't delt with an error, continue with the firmware update if (Update.write(data, len) != len) { - handleError(request, 500); + handleError(request, 500); // internal error, failed return; } if (final && !Update.end(true)) { - handleError(request, 500); + handleError(request, 500); // internal error, failed } } } diff --git a/lib_standalone/ESP8266React.h b/lib_standalone/ESP8266React.h index 9f7590ee1..6c11c7cdc 100644 --- a/lib_standalone/ESP8266React.h +++ b/lib_standalone/ESP8266React.h @@ -20,7 +20,6 @@ #define NETWORK_SETTINGS_FILE "/config/networkSettings.json" #define NTP_SETTINGS_FILE "/config/ntpSettings.json" #define EMSESP_SETTINGS_FILE "/config/emsespSettings.json" -#define OTA_SETTINGS_FILE "/config/otaSettings.json" class DummySettings { public: @@ -77,8 +76,8 @@ class DummySettings { uint8_t provisionMode = 0; uint32_t publish_time_water = 0; - static void read(DummySettings & settings, JsonObject root) {}; - static void read(DummySettings & settings) {}; + static void read(DummySettings & settings, JsonObject root){}; + static void read(DummySettings & settings){}; static StateUpdateResult update(JsonObject root, DummySettings & settings) { return StateUpdateResult::CHANGED; @@ -97,10 +96,8 @@ class DummySettingsService : public StatefulService { #define SecuritySettings DummySettings #define MqttSettings DummySettings #define NTPSettings DummySettings -#define OTASettings DummySettings #define APSettings DummySettings - class ESP8266React { public: ESP8266React(AsyncWebServer * server, FS * fs) @@ -110,7 +107,7 @@ class ESP8266React { void begin() { _mqttClient = new espMqttClient(); }; - void loop() {}; + void loop(){}; SecurityManager * getSecurityManager() { return &_securitySettingsService; @@ -145,10 +142,6 @@ class ESP8266React { return &_settings; } - StatefulService * getOTASettingsService() { - return &_settings; - } - StatefulService * getAPSettingsService() { return &_settings; } diff --git a/lib_standalone/Features.h b/lib_standalone/Features.h index 526806569..e5c05d101 100644 --- a/lib_standalone/Features.h +++ b/lib_standalone/Features.h @@ -21,11 +21,6 @@ #define FT_NTP 0 #endif -// mqtt feature on by default -#ifndef FT_OTA -#define FT_OTA 0 -#endif - // upload firmware/file feature off by default #ifndef FT_UPLOAD_FIRMWARE #define FT_UPLOAD_FIRMWARE 0 diff --git a/mock-api/rest_server.ts b/mock-api/rest_server.ts index 5d0e5f9f1..99505053e 100644 --- a/mock-api/rest_server.ts +++ b/mock-api/rest_server.ts @@ -291,14 +291,6 @@ const list_networks = { ] }; -// OTA -const OTA_SETTINGS_ENDPOINT = REST_ENDPOINT_ROOT + 'otaSettings'; -let ota_settings = { - enabled: false, - port: 8266, - password: 'ems-esp-neo' -}; - // MQTT const MQTT_SETTINGS_ENDPOINT = REST_ENDPOINT_ROOT + 'mqttSettings'; const MQTT_STATUS_ENDPOINT = REST_ENDPOINT_ROOT + 'mqttStatus'; @@ -389,7 +381,6 @@ const system_status = { num_analogs: 1, free_heap: 143, ntp_status: 2, - ota_status: false, mqtt_status: true, ap_status: false }; @@ -2339,15 +2330,6 @@ router return status(200); }); -// OTA -router - .get(OTA_SETTINGS_ENDPOINT, () => ota_settings) - .post(OTA_SETTINGS_ENDPOINT, async (request: any) => { - ota_settings = await request.json(); - console.log('ota settings saved', ota_settings); - return status(200); - }); - // MQTT router .get(MQTT_SETTINGS_ENDPOINT, () => mqtt_settings) diff --git a/pio_local.ini_example b/pio_local.ini_example index d111668b4..b056295cc 100644 --- a/pio_local.ini_example +++ b/pio_local.ini_example @@ -23,21 +23,24 @@ default_envs = esp32_4M ; default_envs = debug ; default_envs = custom +[env] +; upload settings +upload_protocol = custom +custom_emsesp_ip = 10.10.10.173 +custom_username = admin +custom_password = admin +upload_port = /dev/ttyUSB* + [env:esp32_4M] -; if using OTA enter your details below -; upload_protocol = espota -; upload_flags = -; --port=8266 -; --auth=ems-esp-neo -; upload_port = ems-esp.local -; for USB, here are some examples: -; upload_port = /dev/ttyUSB* -; upload_port = COM5 extra_scripts = pre:scripts/build_interface.py ; comment out if you don't want to re-build the WebUI each time scripts/rename_fw.py + scripts/upload.py -[env:esp32_16M] +[env:lolin_s3] +extra_scripts = +; pre:scripts/build_interface.py ; comment out if you don't want to re-build the WebUI each time + scripts/rename_fw.py [env:custom] ; use for basic ESP boards with 4MB flash @@ -75,12 +78,6 @@ build_flags = -D CONFIG_ASYNC_TCP_STACK_SIZE=8192 '-DEMSESP_DEFAULT_BOARD_PROFILE="Test"' -[env:lolin_s3] -upload_port = /dev/ttyUSB0 -extra_scripts = - pre:scripts/build_interface.py ; comment out if you don't want to re-build the WebUI each time - scripts/rename_fw.py - ; pio run -e debug ; or from Visual Studio Code do PIO -> Project Tasks -> debug -> General -> Upload and Monitor ; options for debugging are: EMSESP_DEBUG EMSESP_UART_DEBUG EMSESP_DEBUG_SENSOR diff --git a/scripts/espota.py b/scripts/espota.py deleted file mode 100755 index d6b5a8086..000000000 --- a/scripts/espota.py +++ /dev/null @@ -1,343 +0,0 @@ -#!/usr/bin/env python3 -# -# Original espota.py by Ivan Grokhotkov: -# https://gist.github.com/igrr/d35ab8446922179dc58c -# -# Modified since 2015-09-18 from Pascal Gollor (https://github.com/pgollor) -# Modified since 2015-11-09 from Hristo Gochkov (https://github.com/me-no-dev) -# Modified since 2016-01-03 from Matthew O'Gorman (https://githumb.com/mogorman) -# -# This script will push an OTA update to the ESP -# use it like: python3 espota.py -i -I -p -P [-a password] -f -# Or to upload SPIFFS image: -# python3 espota.py -i -I -p -P [-a password] -s -f -# -# Changes -# 2015-09-18: -# - Add option parser. -# - Add logging. -# - Send command to controller to differ between flashing and transmitting SPIFFS image. -# -# Changes -# 2015-11-09: -# - Added digest authentication -# - Enhanced error tracking and reporting -# -# Changes -# 2016-01-03: -# - Added more options to parser. -# - -from __future__ import print_function -import socket -import sys -import os -import optparse -import logging -import hashlib -import random - -# Commands -FLASH = 0 -SPIFFS = 100 -AUTH = 200 -PROGRESS = False -# update_progress() : Displays or updates a console progress bar -## Accepts a float between 0 and 1. Any int will be converted to a float. -## A value under 0 represents a 'halt'. -## A value at 1 or bigger represents 100% -def update_progress(progress): - if (PROGRESS): - barLength = 60 # Modify this to change the length of the progress bar - status = "" - if isinstance(progress, int): - progress = float(progress) - if not isinstance(progress, float): - progress = 0 - status = "error: progress var must be float\r\n" - if progress < 0: - progress = 0 - status = "Halt...\r\n" - if progress >= 1: - progress = 1 - status = "Done...\r\n" - block = int(round(barLength*progress)) - text = "\rUploading: [{0}] {1}% {2}".format( "="*block + " "*(barLength-block), int(progress*100), status) - sys.stderr.write(text) - sys.stderr.flush() - else: - sys.stderr.write('.') - sys.stderr.flush() - -def serve(remoteAddr, localAddr, remotePort, localPort, password, filename, command = FLASH): - # Create a TCP/IP socket - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = (localAddr, localPort) - logging.info('Starting on %s:%s', str(server_address[0]), str(server_address[1])) - try: - sock.bind(server_address) - sock.listen(1) - except: - logging.error("Listen Failed") - return 1 - - # Check whether Signed Update is used. - if ( os.path.isfile(filename + '.signed') ): - filename = filename + '.signed' - file_check_msg = 'Detected Signed Update. %s will be uploaded instead.' % (filename) - sys.stderr.write(file_check_msg + '\n') - sys.stderr.flush() - logging.info(file_check_msg) - - content_size = os.path.getsize(filename) - f = open(filename,'rb') - file_md5 = hashlib.md5(f.read()).hexdigest() - f.close() - logging.info('Upload size: %d', content_size) - message = '%d %d %d %s\n' % (command, localPort, content_size, file_md5) - - # Wait for a connection - logging.info('Sending invitation to: %s', remoteAddr) - sock2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - remote_address = (remoteAddr, int(remotePort)) - sent = sock2.sendto(message.encode(), remote_address) - sock2.settimeout(10) - try: - data = sock2.recv(128).decode() - except: - logging.error('No Answer') - sock2.close() - return 1 - if (data != "OK"): - if(data.startswith('AUTH')): - nonce = data.split()[1] - cnonce_text = '%s%u%s%s' % (filename, content_size, file_md5, remoteAddr) - cnonce = hashlib.md5(cnonce_text.encode()).hexdigest() - passmd5 = hashlib.md5(password.encode()).hexdigest() - result_text = '%s:%s:%s' % (passmd5 ,nonce, cnonce) - result = hashlib.md5(result_text.encode()).hexdigest() - sys.stderr.write('Authenticating...') - sys.stderr.flush() - message = '%d %s %s\n' % (AUTH, cnonce, result) - sock2.sendto(message.encode(), remote_address) - sock2.settimeout(10) - try: - data = sock2.recv(32).decode() - except: - sys.stderr.write('FAIL\n') - logging.error('No Answer to our Authentication') - sock2.close() - return 1 - if (data != "OK"): - sys.stderr.write('FAIL\n') - logging.error('%s', data) - sock2.close() - sys.exit(1) - return 1 - sys.stderr.write('OK\n') - else: - logging.error('Bad Answer: %s', data) - sock2.close() - return 1 - sock2.close() - - logging.info('Waiting for device...') - try: - sock.settimeout(10) - connection, client_address = sock.accept() - sock.settimeout(None) - connection.settimeout(None) - except: - logging.error('No response from device') - sock.close() - return 1 - - received_ok = False - - try: - f = open(filename, "rb") - if (PROGRESS): - update_progress(0) - else: - sys.stderr.write('Uploading') - sys.stderr.flush() - offset = 0 - while True: - chunk = f.read(1460) - if not chunk: break - offset += len(chunk) - update_progress(offset/float(content_size)) - connection.settimeout(10) - try: - connection.sendall(chunk) - if connection.recv(32).decode().find('O') >= 0: - # connection will receive only digits or 'OK' - received_ok = True - except: - sys.stderr.write('\n') - logging.error('Error Uploading') - connection.close() - f.close() - sock.close() - return 1 - - sys.stderr.write('\n') - logging.info('Waiting for result...') - # libraries/ArduinoOTA/ArduinoOTA.cpp L311 L320 - # only sends digits or 'OK'. We must not not close - # the connection before receiving the 'O' of 'OK' - try: - connection.settimeout(60) - received_ok = False - received_error = False - while not (received_ok or received_error): - reply = connection.recv(64).decode() - # Look for either the "E" in ERROR or the "O" in OK response - # Check for "E" first, since both strings contain "O" - if reply.find('E') >= 0: - sys.stderr.write('\n') - logging.error('%s', reply) - received_error = True - elif reply.find('O') >= 0: - logging.info('Result: OK') - received_ok = True - connection.close() - f.close() - sock.close() - if received_ok: - return 0 - return 1 - except: - logging.error('No Result!') - connection.close() - f.close() - sock.close() - return 1 - - finally: - connection.close() - f.close() - - sock.close() - return 1 -# end serve - - -def parser(unparsed_args): - parser = optparse.OptionParser( - usage = "%prog [options]", - description = "Transmit image over the air to the esp8266 module with OTA support." - ) - - # destination ip and port - group = optparse.OptionGroup(parser, "Destination") - group.add_option("-i", "--ip", - dest = "esp_ip", - action = "store", - help = "ESP8266 IP Address.", - default = False - ) - group.add_option("-I", "--host_ip", - dest = "host_ip", - action = "store", - help = "Host IP Address.", - default = "0.0.0.0" - ) - group.add_option("-p", "--port", - dest = "esp_port", - type = "int", - help = "ESP8266 ota Port. Default 8266", - default = 8266 - ) - group.add_option("-P", "--host_port", - dest = "host_port", - type = "int", - help = "Host server ota Port. Default random 10000-60000", - default = random.randint(10000,60000) - ) - parser.add_option_group(group) - - # auth - group = optparse.OptionGroup(parser, "Authentication") - group.add_option("-a", "--auth", - dest = "auth", - help = "Set authentication password.", - action = "store", - default = "" - ) - parser.add_option_group(group) - - # image - group = optparse.OptionGroup(parser, "Image") - group.add_option("-f", "--file", - dest = "image", - help = "Image file.", - metavar="FILE", - default = None - ) - group.add_option("-s", "--spiffs", - dest = "spiffs", - action = "store_true", - help = "Use this option to transmit a SPIFFS image and do not flash the module.", - default = False - ) - parser.add_option_group(group) - - # output group - group = optparse.OptionGroup(parser, "Output") - group.add_option("-d", "--debug", - dest = "debug", - help = "Show debug output. And override loglevel with debug.", - action = "store_true", - default = False - ) - group.add_option("-r", "--progress", - dest = "progress", - help = "Show progress output. Does not work for ArduinoIDE", - action = "store_true", - default = False - ) - parser.add_option_group(group) - - (options, args) = parser.parse_args(unparsed_args) - - return options -# end parser - - -def main(args): - # get options - options = parser(args) - - # adapt log level - loglevel = logging.WARNING - if (options.debug): - loglevel = logging.DEBUG - # end if - - # logging - logging.basicConfig(level = loglevel, format = '%(asctime)-8s [%(levelname)s]: %(message)s', datefmt = '%H:%M:%S') - - logging.debug("Options: %s", str(options)) - - # check options - global PROGRESS - PROGRESS = options.progress - if (not options.esp_ip or not options.image): - logging.critical("Not enough arguments.") - - return 1 - # end if - - command = FLASH - if (options.spiffs): - command = SPIFFS - # end if - - return serve(options.esp_ip, options.host_ip, options.esp_port, options.host_port, options.auth, options.image, command) -# end main - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) -# end if diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 000000000..b05375d1e --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,6 @@ +requests +termcolor +requests_toolbelt +tqdm + + diff --git a/scripts/upload.py b/scripts/upload.py new file mode 100644 index 000000000..61fff81bf --- /dev/null +++ b/scripts/upload.py @@ -0,0 +1,130 @@ + +# Modified from https://github.com/ayushsharma82/ElegantOTA + +# This is called during the platformIO upload process +# To use create a pio_local.ini file in the project root and add the following: +# [env] +# upload_protocol = custom +# custom_emsesp_ip = 10.10.10.173 +# custom_username = admin +# custom_password = admin +# +# and +# extra_scripts = scripts/upload.py + +import requests +import hashlib +from urllib.parse import urlparse +import time +Import("env") + +try: + from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor + from tqdm import tqdm + from termcolor import cprint +except ImportError: + env.Execute("$PYTHONEXE -m pip install requests_toolbelt") + env.Execute("$PYTHONEXE -m pip install tqdm") + env.Execute("$PYTHONEXE -m pip install termcolor") + from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor + from tqdm import tqdm + from termcolor import cprint + +def print_success(x): return cprint(x, 'green') +def print_fail(x): return cprint(x, 'red') + +def on_upload(source, target, env): + + # first check authentication + try: + username = env.GetProjectOption('custom_username') + password = env.GetProjectOption('custom_password') + except: + print('No authentication settings specified. Please, add these to your pio_local.ini file: \n\ncustom_username=username\ncustom_password=password\n') + return + + emsesp_url = "http://" + env.GetProjectOption('custom_emsesp_ip') + parsed_url = urlparse(emsesp_url) + host_ip = parsed_url.netloc + + signon_url = f"{emsesp_url}/rest/signIn" + + signon_headers = { + 'Host': host_ip, + 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0', + 'Accept': '*/*', + 'Accept-Language': 'de,en-US;q=0.7,en;q=0.3', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': f'{emsesp_url}', + 'Content-Type': 'application/json', + 'Connection': 'keep-alive' + } + + username_password = { + "username": username, + "password": password + } + + response = requests.post(signon_url, json=username_password, headers=signon_headers, auth=None) + + if response.status_code != 200: + print_fail("Authentication failed (code " + str(response.status_code) + ")") + return + + print_success("Authentication successful") + access_token = response.json().get('access_token') + + # start the upload + firmware_path = str(source[0]) + + with open(firmware_path, 'rb') as firmware: + md5 = hashlib.md5(firmware.read()).hexdigest() + + firmware.seek(0) + + encoder = MultipartEncoder(fields={ + 'MD5': md5, + 'file': (firmware_path, firmware, 'application/octet-stream')} + ) + + bar = tqdm(desc='Upload Progress', + total=encoder.len, + dynamic_ncols=True, + unit='B', + unit_scale=True, + unit_divisor=1024 + ) + + monitor = MultipartEncoderMonitor(encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n)) + + post_headers = { + 'Host': host_ip, + 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0', + 'Accept': '*/*', + 'Accept-Language': 'de,en-US;q=0.7,en;q=0.3', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': f'{emsesp_url}', + 'Connection': 'keep-alive', + 'Content-Type': monitor.content_type, + 'Content-Length': str(monitor.len), + 'Origin': f'{emsesp_url}', + 'Authorization': 'Bearer ' + f'{access_token}' + } + + upload_url = f"{emsesp_url}/rest/uploadFile" + + response = requests.post(upload_url, data=monitor, headers=post_headers, auth=None) + + bar.close() + time.sleep(0.1) + + print() + + if response.status_code != 200: + print_fail("Upload failed (code " + response.status.code + ").") + else: + print_success("Upload successful.") + + print() + +env.Replace(UPLOADCMD=on_upload) \ No newline at end of file diff --git a/scripts/upload_cli.py b/scripts/upload_cli.py new file mode 100644 index 000000000..8a6a5836e --- /dev/null +++ b/scripts/upload_cli.py @@ -0,0 +1,119 @@ + +# Modified from https://github.com/ayushsharma82/ElegantOTA +# +# Requires Python (sudo apt install python3-pip) +# `python3 -m venv venv` to create the virtual environment +# `source ./venv/bin/activate` to enter it +# `pip install -r requirements.txt` to install the libraries +# +# Run using for example: +# python3 upload_cli.py -i 10.10.10.173 -f ../build/firmware/EMS-ESP-3_7_0-dev_8-ESP32_4M.bin + +import argparse +import requests +import hashlib +from urllib.parse import urlparse +import time +from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor +from tqdm import tqdm +from termcolor import cprint + +def print_success(x): return cprint(x, 'green') +def print_fail(x): return cprint(x, 'red') + +def upload(file, ip, username, password): + + # Print welcome message + print() + print("EMS-ESP Firmware Upload") + + # first check authentication + emsesp_url = "http://" + f'{ip}' + parsed_url = urlparse(emsesp_url) + host_ip = parsed_url.netloc + + signon_url = f"{emsesp_url}/rest/signIn" + + signon_headers = { + 'Host': host_ip, + 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0', + 'Accept': '*/*', + 'Accept-Language': 'de,en-US;q=0.7,en;q=0.3', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': f'{emsesp_url}', + 'Content-Type': 'application/json', + 'Connection': 'keep-alive' + } + + username_password = { + "username": username, + "password": password + } + + response = requests.post(signon_url, json=username_password, headers=signon_headers, auth=None) + + if response.status_code != 200: + print_fail("Authentication failed (code " + str(response.status_code) + ")") + return + + print_success("Authentication successful") + access_token = response.json().get('access_token') + + # start the upload + with open(file, 'rb') as firmware: + md5 = hashlib.md5(firmware.read()).hexdigest() + + firmware.seek(0) + + encoder = MultipartEncoder(fields={ + 'MD5': md5, + 'file': (file, firmware, 'application/octet-stream')} + ) + + bar = tqdm(desc='Upload Progress', + total=encoder.len, + dynamic_ncols=True, + unit='B', + unit_scale=True, + unit_divisor=1024 + ) + + monitor = MultipartEncoderMonitor(encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n)) + + post_headers = { + 'Host': host_ip, + 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0', + 'Accept': '*/*', + 'Accept-Language': 'de,en-US;q=0.7,en;q=0.3', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': f'{emsesp_url}', + 'Connection': 'keep-alive', + 'Content-Type': monitor.content_type, + 'Content-Length': str(monitor.len), + 'Origin': f'{emsesp_url}', + 'Authorization': 'Bearer ' + f'{access_token}' + } + + upload_url = f"{emsesp_url}/rest/uploadFile" + + response = requests.post(upload_url, data=monitor, headers=post_headers, auth=None) + + bar.close() + time.sleep(0.1) + + if response.status_code != 200: + print_fail("Upload failed (code " + response.status.code + ").") + else: + print_success("Upload successful.") + + print() + +# main +parser = argparse.ArgumentParser(description="EMS-ESP Firmware Upload") +parser.add_argument("-f", "--file", metavar="FILE", required=True, type=str, help="firmware file") +parser.add_argument("-i", "--ip", metavar="IP", type=str, default="ems-esp.local", help="IP address of EMS-ESP") +parser.add_argument("-u", "--username", metavar="USERNAME", type=str, default="admin", help="admin user") +parser.add_argument("-p", "--password", metavar="PASSWORD", type=str, default="admin", help="admin password") +args = parser.parse_args() +upload(**vars(args)) + diff --git a/scripts/upload_esp32.py b/scripts/upload_esp32.py deleted file mode 100755 index 7afd4b095..000000000 --- a/scripts/upload_esp32.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -import subprocess -import os, argparse - -print("\n** Starting upload...") - -ap = argparse.ArgumentParser() -ap.add_argument("-p", "--port", required=True, help="port") -args = vars(ap.parse_args()) - -# esptool.py --chip esp32 --port "COM4" --baud 921600 --before default_reset --after hard_reset write_flash -z --flash_mode dio --flash_freq 40m --flash_size detect 0x1000 bootloader_dio_40m.bin 0x8000 partitions.bin 0xe000 boot_app0.bin 0x10000 EMS-ESP-dev-esp32.bin - -subprocess.call(["esptool.py", "--chip esp32", "-p", args['port'], "--baud", "921600", "--before", "default_reset", "--after", "hard_reset", "write_flash", "-z", "--flash_mode", "dio", "--flash_freq", "40m","--flash_size", "detect", "0x1000", "bootloader_dio_40m.bin", "0x8000", "partitions.bin","0xe000", "boot_app0.bin", "0x10000", "EMS-ESP-dev-esp32.bin"]) - -print("\n** Finished upload.") diff --git a/src/console.cpp b/src/console.cpp index 9c523f701..20271ccc1 100644 --- a/src/console.cpp +++ b/src/console.cpp @@ -364,11 +364,6 @@ static void setup_commands(std::shared_ptr & commands) { Settings.enabled = enable; return StateUpdateResult::CHANGED; }); - } else if (arguments.front() == "ota") { - to_app(shell).esp8266React.getOTASettingsService()->update([&](OTASettings & Settings) { - Settings.enabled = enable; - return StateUpdateResult::CHANGED; - }); } else if (arguments.front() == "ntp") { to_app(shell).esp8266React.getNTPSettingsService()->update([&](NTPSettings & Settings) { Settings.enabled = enable; diff --git a/src/locale_common.h b/src/locale_common.h index 5e0af068a..bec8b7eb9 100644 --- a/src/locale_common.h +++ b/src/locale_common.h @@ -160,7 +160,7 @@ MAKE_WORD_CUSTOM(new_password_prompt2, "Retype new password: ") MAKE_WORD_CUSTOM(password_prompt, "Password: ") MAKE_WORD_CUSTOM(unset, "") MAKE_WORD_CUSTOM(enable_mandatory, "") -MAKE_WORD_CUSTOM(service_mandatory, "") +MAKE_WORD_CUSTOM(service_mandatory, "") // more common names that don't need translations MAKE_NOTRANSLATION(1x3min, "1x3min") diff --git a/src/system.cpp b/src/system.cpp index 2e5804b2e..aa76da654 100644 --- a/src/system.cpp +++ b/src/system.cpp @@ -836,6 +836,9 @@ void System::commands_init() { } // uses LED to show system health +// 1 x flash = the EMS bus is not connected +// 2 x flash = the network (wifi or ethernet) is not connected +// 3 x flash = both EMS bus and network are failing. This is a critical error! void System::led_monitor() { // we only need to run the LED healthcheck if there are errors if (!healthcheck_ || !led_gpio_) { @@ -849,7 +852,6 @@ void System::led_monitor() { // first long pause before we start flashing if (led_long_timer_ && (uint32_t)(current_time - led_long_timer_) >= HEALTHCHECK_LED_LONG_DUARATION) { - // Serial.println("starting the flash check"); led_short_timer_ = current_time; // start the short timer led_long_timer_ = 0; // stop long timer led_flash_step_ = 1; // enable the short flash timer @@ -863,7 +865,6 @@ void System::led_monitor() { if (++led_flash_step_ == 8) { // reset the whole sequence - // Serial.println("resetting flash check"); led_long_timer_ = uuid::get_uptime(); led_flash_step_ = 0; #if defined(ARDUINO_LOLIN_C3_MINI) && !defined(BOARD_C3_MINI_V1) @@ -1100,7 +1101,6 @@ bool System::check_restore() { reboot_required |= saveSettings(NTP_SETTINGS_FILE, "NTP", input); reboot_required |= saveSettings(SECURITY_SETTINGS_FILE, "Security", input); reboot_required |= saveSettings(EMSESP_SETTINGS_FILE, "Settings", input); - reboot_required |= saveSettings(OTA_SETTINGS_FILE, "OTA", input); } else if (settings_type == "customizations") { // it's a customization file, just replace it and there's no need to reboot saveSettings(EMSESP_CUSTOMIZATION_FILE, "Customizations", input); @@ -1343,13 +1343,6 @@ bool System::command_info(const char * value, const int8_t id, JsonObject output node["tz label"] = settings.tzLabel; // node["tz format"] = settings.tzFormat; }); - - // OTA status - node = output["OTA Info"].to(); - EMSESP::esp8266React.getOTASettingsService()->read([&](OTASettings & settings) { - node["enabled"] = settings.enabled; - node["port"] = settings.port; - }); #endif // MQTT Status diff --git a/src/version.h b/src/version.h index ad1f990b9..25521abd3 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define EMSESP_APP_VERSION "3.7.0-dev.8" +#define EMSESP_APP_VERSION "3.7.0-dev.9" diff --git a/src/web/WebAPIService.cpp b/src/web/WebAPIService.cpp index 3a242d01e..a7e2e83eb 100644 --- a/src/web/WebAPIService.cpp +++ b/src/web/WebAPIService.cpp @@ -181,7 +181,6 @@ void WebAPIService::getSettings(AsyncWebServerRequest * request) { System::extractSettings(AP_SETTINGS_FILE, "AP", root); System::extractSettings(MQTT_SETTINGS_FILE, "MQTT", root); System::extractSettings(NTP_SETTINGS_FILE, "NTP", root); - System::extractSettings(OTA_SETTINGS_FILE, "OTA", root); System::extractSettings(SECURITY_SETTINGS_FILE, "Security", root); System::extractSettings(EMSESP_SETTINGS_FILE, "Settings", root); diff --git a/src/web/WebStatusService.cpp b/src/web/WebStatusService.cpp index 7916b2470..3d988176a 100644 --- a/src/web/WebStatusService.cpp +++ b/src/web/WebStatusService.cpp @@ -53,8 +53,7 @@ void WebStatusService::systemStatus(AsyncWebServerRequest * request) { root["num_analogs"] = EMSESP::analogsensor_.no_sensors(); root["free_heap"] = EMSESP::system_.getHeapMem(); root["uptime"] = uuid::get_uptime_sec(); - EMSESP::esp8266React.getOTASettingsService()->read([root](OTASettings & otaSettings) { root["ota_status"] = otaSettings.enabled; }); - root["mqtt_status"] = EMSESP::mqtt_.connected(); + root["mqtt_status"] = EMSESP::mqtt_.connected(); #ifndef EMSESP_STANDALONE root["ntp_status"] = [] { diff --git a/test/standalone_file_export/emsesp_settings.json b/test/standalone_file_export/emsesp_settings.json index 4dc16697e..988fe1bbe 100644 --- a/test/standalone_file_export/emsesp_settings.json +++ b/test/standalone_file_export/emsesp_settings.json @@ -62,11 +62,6 @@ "tz_label": "Europe/Amsterdam", "tz_format": "CET-1CEST,M3.5.0,M10.5.0/3" }, - "OTA": { - "enabled": false, - "port": 8266, - "password": "ems-esp-neo" - }, "Security": { "jwt_secret": "ems-esp-neo", "users": [