Merge pull request #3009 from MichaelDvP/core3

https client for scheduler/shuntingYard

lib/espMqttClient/src/Transport/ClientPosix.cpp

@@ -8,7 +8,7 @@ the LICENSE file.
 
 #include "ClientPosix.h"
 
-#if defined(__linux__)
+#if defined(__linux__) || defined(__APPLE__)
 
 namespace espMqttClientInternals {
 

lib/espMqttClient/src/Transport/ClientPosix.h

@@ -8,7 +8,7 @@ the LICENSE file.
 
 #pragma once
 
-#if defined(__linux__)
+#if defined(__linux__) || defined(__APPLE__)
 
 #include <stdio.h>
 #include <stdlib.h>

lib/espMqttClient/src/espMqttClient.cpp

@@ -69,7 +69,7 @@ espMqttClientSecure & espMqttClientSecure::setPreSharedKey(const char * pskIdent
 
 #endif
 
-#if defined(__linux__)
+#if defined(__linux__) || defined(__APPLE__)
 espMqttClient::espMqttClient()
     : MqttClientSetup(espMqttClientTypes::UseInternalTask::NO)
     , _client() {

lib/espMqttClient/src/espMqttClient.h

@@ -14,7 +14,7 @@ the LICENSE file.
 #if defined(ARDUINO_ARCH_ESP8266) || defined(ARDUINO_ARCH_ESP32)
 #include "Transport/ClientSync.h"
 #include "Transport/ClientSecureSync.h"
-#elif defined(__linux__)
+#elif defined(__linux__) || defined(__APPLE__)
 #include "Transport/ClientPosix.h"
 #endif
 
@@ -74,7 +74,7 @@ class espMqttClientSecure : public MqttClientSetup<espMqttClientSecure> {
 
 #endif
 
-#if defined(__linux__)
+#if defined(__linux__) || defined(__APPLE__)
 class espMqttClient : public MqttClientSetup<espMqttClient> {
   public:
     espMqttClient();

src/core/shuntingYard.cpp

@@ -380,11 +380,15 @@ std::string commands(std::string & expr, bool quotes) {
             if (return_code != CommandRet::OK && return_code != CommandRet::NO_VALUE) {
                 return expr = "";
             }
-
+            std::string data;
-            std::string data = output["api_data"] | "";
+            if (output["api_data"].is<std::string>()) {
-            if (!isnum(data) && quotes) {
+                data = output["api_data"].as<std::string>();
-                data.insert(data.begin(), '"');
+                if (!isnum(data) && quotes) {
-                data.insert(data.end(), '"');
+                    data.insert(data.begin(), '"');
+                    data.insert(data.end(), '"');
+                }
+            } else {
+                serializeJson(output, data);
             }
             expr.replace(f, l, data);
             e        = f + data.length();
@@ -700,7 +704,6 @@ std::string compute(const std::string & expr) {
         std::string  cmd = expr_new.substr(f, e - f).c_str();
         JsonDocument doc;
         if (DeserializationError::Ok == deserializeJson(doc, cmd)) {
-            HTTPClient  http;
             std::string url, header_s, value_s, method_s, key_s, keys_s;
             // search keys lower case
             for (JsonPair p : doc.as<JsonObject>()) {
@@ -720,56 +723,149 @@ std::string compute(const std::string & expr) {
                     keys_s = p.key().c_str();
                 }
             }
-            if (http.begin(url.c_str())) {
+            bool        content_set = false;
-                int httpResult = 0;
+            std::string value       = doc[value_s] | "";
-                for (JsonPair p : doc[header_s].as<JsonObject>()) {
+            std::string method      = doc[method_s] | "GET";
-                    http.addHeader(p.key().c_str(), p.value().as<std::string>().c_str());
+            if (value.length()) {
+                method = "POST";
+            }
+            std::string result;
+            int         httpResult = 0;
+#ifndef NO_TLS_SUPPORT
+            if (Helpers::toLower(url.c_str()).starts_with("https://")) {
+                WiFiClient *    basic_client = new WiFiClient;
+                ESP_SSLClient * ssl_client   = new ESP_SSLClient;
+                ssl_client->setInsecure(); // with root CA we should set here: ssl_client->setCACert(rootCACert);
+                ssl_client->setBufferSizes(1024, 1024);
+                ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds)
+                url.replace(0, 8, "");
+                std::string host  = url;
+                auto        index = url.find_first_of('/');
+                if (index != std::string::npos) {
+                    host = url.substr(0, index);
+                    url.replace(0, index, "");
                 }
-                std::string value  = doc[value_s] | "";
+                /*
-                std::string method = doc[method_s] | "get";
+                index = host.find_first_of('@');
-
+                std::string auth;
-                // if there is data, force a POST
+                if (index != std::string::npos) {
-                if (value.length() || Helpers::toLower(method) == "post") {
+                    auth = base64::encode(host.substr(0, index));
-                    if (value.find_first_of('{') != std::string::npos) {
+                    host.replace(0, index, "");
-                        http.addHeader(asyncsrv::T_Content_Type, asyncsrv::T_application_json, false); // auto-set to JSON
-                    }
-                    httpResult = http.POST(value.c_str());
-                } else {
-                    httpResult = http.GET(); // normal GET
                 }
-
+                */
-                if (httpResult > 0) {
+                ssl_client->setClient(basic_client);
-                    std::string  result = http.getString().c_str();
+                if (ssl_client->connect(host.c_str(), 443)) {
-                    std::string  key    = doc[key_s] | "";
+                    if (value.length() || Helpers::toLower(method) == "post") {
-                    JsonDocument keys_doc; // JsonDocument to hold "keys" after doc is parsed with HTTP body
+                        ssl_client->print("POST ");
-                    if (doc[keys_s].is<JsonArray>()) {
+                        ssl_client->print(url.c_str());
-                        keys_doc.set(doc[keys_s].as<JsonArray>());
+                        ssl_client->println(" HTTP/1.1");
-                    }
+                        ssl_client->print("Host: ");
-                    JsonArray keys = keys_doc.as<JsonArray>();
+                        ssl_client->println(host.c_str());
-
+                        for (JsonPair p : doc[header_s].as<JsonObject>()) {
-                    if (key.length() || !keys.isNull()) {
+                            content_set |= (emsesp::Helpers::toLower(p.key().c_str()) == "content-type");
-                        doc.clear();
+                            ssl_client->print(p.key().c_str());
-                        if (DeserializationError::Ok == deserializeJson(doc, result)) {
+                            ssl_client->print(": ");
-                            if (key.length()) {
+                            ssl_client->println(p.value().as<std::string>().c_str());
-                                result = doc[key.c_str()].as<std::string>();
+                        }
+                        if (!content_set) {
+                            ssl_client->print("Content-Type: ");
+                            if (value.starts_with('{')) {
+                                ssl_client->println(asyncsrv::T_application_json);
                             } else {
-                                JsonVariant json = doc.as<JsonVariant>();
+                                ssl_client->println(asyncsrv::T_text_plain);
-                                for (JsonVariant keys_key : keys) {
-                                    if (keys_key.is<std::string>() && json.is<JsonObject>()) {
-                                        json = json[keys_key.as<std::string>()].as<JsonVariant>();
-                                    } else if (keys_key.is<int>() && json.is<JsonArray>()) {
-                                        json = json[keys_key.as<int>()].as<JsonVariant>();
-                                    } else {
-                                        break; // type mismatch
-                                    }
-                                }
-                                result = json.as<std::string>();
                             }
                         }
+                        ssl_client->print("Content-Length: ");
+                        ssl_client->println(value.length());
+                        ssl_client->println("Connection: close");
+                        ssl_client->print("\r\n");
+                        ssl_client->print(value.c_str());
+                    } else {
+                        ssl_client->print("GET ");
+                        ssl_client->print(url.c_str());
+                        ssl_client->println(" HTTP/1.1");
+                        ssl_client->print("Host: ");
+                        ssl_client->println(host.c_str());
+                        for (JsonPair p : doc[header_s].as<JsonObject>()) {
+                            ssl_client->print(p.key().c_str());
+                            ssl_client->print(": ");
+                            ssl_client->println(p.value().as<std::string>().c_str());
+                        }
+                        ssl_client->println("Connection: close");
+                    }
+                    auto ms = millis();
+                    while (!ssl_client->available() && millis() - ms < 3000) {
+                        delay(0);
+                    }
+                    while (ssl_client->available()) {
+                        result += (char)ssl_client->read();
+                    }
+                    ssl_client->stop();
+                    index = result.find_first_of(' ');
+                    if (index != std::string::npos) {
+                        httpResult = stoi(result.substr(index + 1, 3));
+                    }
+                    index = result.find("\r\n\r\n");
+                    if (index != std::string::npos) {
+                        result.replace(0, index + 4, "");
                     }
-                    expr_new.replace(f, e - f, result.c_str());
                 }
-                http.end();
+                delete ssl_client;
+                delete basic_client;
+            } else
+#endif
+                if (Helpers::toLower(url.c_str()).starts_with("http://")) {
+                HTTPClient * http = new HTTPClient;
+                if (http->begin(url.c_str())) {
+                    for (JsonPair p : doc[header_s].as<JsonObject>()) {
+                        http->addHeader(p.key().c_str(), p.value().as<std::string>().c_str());
+                        content_set |= (emsesp::Helpers::toLower(p.key().c_str()) == "content-type");
+                    }
+                    if (value.length() || Helpers::toLower(method) == "post") {
+                        if (!content_set) {
+                            http->addHeader("Content-Type", value.starts_with('{') ? asyncsrv::T_application_json : asyncsrv::T_text_plain);
+                        }
+                        httpResult = http->POST(value.c_str());
+                    } else {
+                        httpResult = http->GET(); // normal GET
+                    }
+                    if (httpResult > 0) {
+                        result = http->getString().c_str();
+                    }
+                }
+                http->end();
+                delete http;
+            }
+            if (httpResult == 200) {
+                std::string  key = doc[key_s] | "";
+                JsonDocument keys_doc; // JsonDocument to hold "keys" after doc is parsed with HTTP body
+                if (doc[keys_s].is<JsonArray>()) {
+                    keys_doc.set(doc[keys_s].as<JsonArray>());
+                }
+                JsonArray keys = keys_doc.as<JsonArray>();
+                if (key.length() || !keys.isNull()) {
+                    doc.clear();
+                    if (DeserializationError::Ok == deserializeJson(doc, result)) {
+                        if (key.length()) {
+                            result = doc[key.c_str()].as<std::string>();
+                        } else {
+                            JsonVariant json = doc.as<JsonVariant>();
+                            for (JsonVariant keys_key : keys) {
+                                if (keys_key.is<std::string>() && json.is<JsonObject>()) {
+                                    json = json[keys_key.as<std::string>()].as<JsonVariant>();
+                                } else if (keys_key.is<int>() && json.is<JsonArray>()) {
+                                    json = json[keys_key.as<int>()].as<JsonVariant>();
+                                } else {
+                                    break; // type mismatch
+                                }
+                            }
+                            result = json.as<std::string>();
+                        }
+                    }
+                }
+                expr_new.replace(f, e - f, result);
+            } else if (httpResult != 0) {
+                EMSESP::logger().warning("URL command failed with https code: %d, response: %s", httpResult, result.c_str());
             }
         }
         f = expr_new.find_first_of('{', e);

src/emsesp_version.h

@@ -1 +1 @@
-#define EMSESP_APP_VERSION "3.8.2-dev.C13"
+#define EMSESP_APP_VERSION "3.9.0-dev.0"

src/web/WebSchedulerService.cpp

@@ -363,39 +363,139 @@ bool WebSchedulerService::command(const char * name, const std::string & command
             commands(s, false);
             url.replace(q + 1, l, s);
         }
-        if (http.begin(url.c_str())) {
+        std::string value  = doc["value"] | data;   // extract value if its in the command, or take the data
-            // add any given headers
+        std::string method = doc["method"] | "GET"; // default GET
-            for (JsonPair p : doc["header"].as<JsonObject>()) {
+        commands(value, false);
-                http.addHeader(p.key().c_str(), p.value().as<String>().c_str());
+        if (value.length()) {
+            method = "POST";
+        }
+        std::string result;
+        int         httpResult = 0;
+#ifndef NO_TLS_SUPPORT
+        if (Helpers::toLower(url.c_str()).starts_with("https://")) {
+            WiFiClient *    basic_client = new WiFiClient;
+            ESP_SSLClient * ssl_client   = new ESP_SSLClient;
+            ssl_client->setInsecure(); // with root CA we should set here: ssl_client->setCACert(rootCACert);
+            ssl_client->setBufferSizes(1024, 1024);
+            ssl_client->setSessionTimeout(120); // Set the timeout in seconds (>=120 seconds)
+            url.replace(0, 8, "");
+            std::string host  = url;
+            auto        index = url.find_first_of('/');
+            if (index != std::string::npos) {
+                host = url.substr(0, index);
+                url.replace(0, index, "");
             }
-            std::string value  = doc["value"] | data.c_str(); // extract value if its in the command, or take the data
+            // EMSESP::logger().debug("Host: %s, URL: %s", host.c_str(), url.c_str());
-            std::string method = doc["method"] | "GET";       // default GET
+            ssl_client->setClient(basic_client);
-
+            if (ssl_client->connect(host.c_str(), 443)) {
-            commands(value, false);
+                if (value.length() || Helpers::toLower(method) == "post") {
-            // if there is data, force a POST
+                    // EMSESP::logger().debug("POST %s HTTP/1.1", url.c_str());
-            int httpResult = 0;
+                    ssl_client->print("POST ");
-            if (value.length() || method == "post") { // we have all lowercase
+                    ssl_client->print(url.c_str());
-                if (value.find_first_of('{') != std::string::npos) {
+                    ssl_client->println(" HTTP/1.1");
-                    http.addHeader(asyncsrv::T_Content_Type, asyncsrv::T_application_json, false); // auto-set to JSON
+                    ssl_client->print("Host: ");
+                    ssl_client->println(host.c_str());
+                    bool content_set = false;
+                    for (JsonPair p : doc["header"].as<JsonObject>()) {
+                        content_set |= (emsesp::Helpers::toLower(p.key().c_str()) == "content-type");
+                        ssl_client->print(p.key().c_str());
+                        ssl_client->print(": ");
+                        ssl_client->println(p.value().as<std::string>().c_str());
+                    }
+                    if (!content_set) {
+                        ssl_client->print("Content-Type: ");
+                        if (value.starts_with('{')) {
+                            ssl_client->println(asyncsrv::T_application_json);
+                        } else {
+                            ssl_client->println(asyncsrv::T_text_plain);
+                        }
+                    }
+                    ssl_client->print("Content-Length: ");
+                    ssl_client->println(value.length());
+                    ssl_client->println("Connection: close");
+                    ssl_client->print("\r\n");
+                    ssl_client->print(value.c_str());
+                } else {
+                    // EMSESP::logger().debug("GET %s HTTP/1.1", url.c_str());
+                    ssl_client->print("GET ");
+                    ssl_client->print(url.c_str());
+                    ssl_client->println(" HTTP/1.1");
+                    ssl_client->print("Host: ");
+                    ssl_client->println(host.c_str());
+                    for (JsonPair p : doc["header"].as<JsonObject>()) {
+                        ssl_client->print(p.key().c_str());
+                        ssl_client->print(": ");
+                        ssl_client->println(p.value().as<std::string>().c_str());
+                    }
+                    ssl_client->println("Connection: close");
+                }
+                auto ms = millis();
+                while (ssl_client->connected() && !ssl_client->available() && millis() - ms < 3000) {
+                    delay(0);
+                }
+                while (ssl_client->available()) {
+                    result += (char)ssl_client->read();
+                }
+                ssl_client->stop();
+                // EMSESP::logger().debug("HTTPS response: %s", result.c_str());
+                index = result.find_first_of(' ');
+                if (index != std::string::npos) {
+                    httpResult = stoi(result.substr(index + 1, 3));
+                    // EMSESP::logger().debug("HTTPS code: %i", httpResult);
+                }
+                index = result.find("\r\n\r\n");
+                if (index != std::string::npos) {
+                    result.replace(0, index + 4, "");
+                    // EMSESP::logger().debug("HTTPS response: %s", result.c_str());
                 }
-                httpResult = http.POST(value.c_str());
             } else {
-                httpResult = http.GET(); // normal GET
+                EMSESP::logger().warning("HTTPS connection failed");
             }
-
+            delete ssl_client;
-            http.end();
+            delete basic_client;
-
             // check HTTP return code
             if (httpResult != 200) {
-                char error[100];
+                EMSESP::logger().warning("Schedule '%s': URL command failed with http code %d", name, httpResult);
-                snprintf(error, sizeof(error), "Schedule %s: URL command failed with http code %d", name, httpResult);
+                return false;
-                EMSESP::logger().warning(error);
+            }
+            return true;
+        } else
+#endif
+        if (Helpers::toLower(url.c_str()).starts_with("http://")) {
+            HTTPClient * http = new HTTPClient;
+            if (http->begin(url.c_str())) {
+                bool content_set = false;
+                for (JsonPair p : doc["header"].as<JsonObject>()) {
+                    http->addHeader(p.key().c_str(), p.value().as<std::string>().c_str());
+                    content_set |= p.key() == "content-type";
+                }
+                // if there is data, force a POST
+                if (Helpers::toLower(method) == "post") { // we have all lowercase
+                    if (!content_set) {
+                        // http->addHeader("Content-Type", value.find_first_of('{') != std::string::npos ? "application/json" : "text/plain");
+                        if (value.starts_with('{')) {
+                            http->addHeader(asyncsrv::T_Content_Type, asyncsrv::T_application_json, false); // auto-set to JSON
+                        } else {
+                            http->addHeader(asyncsrv::T_Content_Type, asyncsrv::T_text_plain, false); // auto-set to JSON
+                        }
+                    }
+                    httpResult = http->POST(value.c_str());
+                } else {
+                    httpResult = http->GET(); // normal GET
+                    if (httpResult > 0) {
+                        result = http->getString().c_str();
+                    }
+                }
+            }
+            http->end();
+            delete http;
+            // check HTTP return code
+            if (httpResult != 200) {
+                EMSESP::logger().warning("Schedule '%s': URL command failed with http code %d", name, httpResult);
                 return false;
             }
 #if defined(EMSESP_DEBUG)
-            char msg[100];
+            EMSESP::logger().debug("Schedule %s: URL '%s' command successful with http code %d", name, url.c_str(), httpResult);
-            snprintf(msg, sizeof(msg), "Schedule %s: URL command successful with http code %d", name, httpResult);
-            EMSESP::logger().debug(msg);
 #endif
             return true;
         }