Merge pull request #1339 from proddy/dev

update web building

.github/workflows/pre_release.yml

@@ -33,9 +33,9 @@ jobs:
         run: |
           cd interface
           yarn install
-          yarn run typesafe-i18n --no-watch
+          yarn typesafe-i18n --no-watch
           sed -i "s/= 'pl'/= 'en'/" ./src/i18n/i18n-util.ts
-          yarn run build
+          yarn build
 
       - name: Build firmware
         run: |

.github/workflows/tagged_release.yml

@@ -28,9 +28,9 @@ jobs:
         run: |
           cd interface
           yarn install
-          yarn run typesafe-i18n --no-watch
+          yarn typesafe-i18n --no-watch
           sed -i "s/= 'pl'/= 'en'/" ./src/i18n/i18n-util.ts
-          yarn run build
+          yarn build
 
       - name: Build firmware
         run: |

.github/workflows/test_release.yml

@@ -33,9 +33,9 @@ jobs:
         run: |
           cd interface
           yarn install
-          yarn run typesafe-i18n --no-watch
+          yarn typesafe-i18n --no-watch
           sed -i "s/= 'pl'/= 'en'/" ./src/i18n/i18n-util.ts
-          yarn run build
+          yarn build
 
       - name: Build firmware
         run: |

.gitignore

@@ -36,6 +36,8 @@ stats.html
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+yarn.lock
+interface/analyse.html
 
 # scripts
 test.sh
@@ -59,4 +61,3 @@ bw-output/
 # dump_entities.csv
 # dump_entities.xls*
 
-yarn.lock

interface/.yarnrc.yml

@@ -3,5 +3,3 @@ compressionLevel: mixed
 enableGlobalCache: false
 
 nodeLinker: node-modules
-
-yarnPath: .yarn/releases/yarn-4.0.0-rc.53.cjs

interface/package.json

@@ -1,29 +1,30 @@
 {
   "name": "EMS-ESP",
-  "version": "3.6",
+  "version": "3.6.3",
   "description": "build EMS-ESP WebUI",
   "homepage": "https://emsesp.github.io/docs",
   "author": "proddy",
   "license": "MIT",
   "private": true,
   "scripts": {
-    "dev": "vite",
     "build": "vite build",
-    "build-hosted": "vite build --mode hosted",
     "preview": "vite preview",
-    "preview-standalone": "npm-run-all -p preview typesafe-i18n mock-api",
+    "build-hosted": "typesafe-i18n --no-watch && vite build --mode hosted",
+    "preview-standalone": "typesafe-i18n --no-watch && vite build && concurrently -c \"auto\" \"npm:mock-api\" \"vite preview\"",
     "mock-api": "node --watch ../mock-api ../mock-api/server.js",
-    "standalone": "npm-run-all -p dev typesafe-i18n mock-api",
-    "typesafe-i18n": "typesafe-i18n",
+    "standalone": "concurrently -c \"auto\" \"typesafe-i18n\" \"npm:mock-api\" \"vite\"",
+    "typesafe-i18n": "typesafe-i18n --no-watch",
+    "webUI": "node progmem-generator.js",
     "format": "prettier --write '**/*.{ts,tsx,js,css,json,md}'",
     "lint": "eslint . --cache --fix"
   },
   "dependencies": {
     "@alova/adapter-xhr": "^1.0.1",
+    "@babel/core": "^7.23.2",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
-    "@mui/icons-material": "^5.14.13",
-    "@mui/material": "^5.14.13",
+    "@mui/icons-material": "^5.14.14",
+    "@mui/material": "^5.14.14",
     "@table-library/react-table-library": "4.1.7",
     "@types/lodash-es": "^4.17.9",
     "@types/node": "^20.8.6",
@@ -32,6 +33,7 @@
     "@types/react-router-dom": "^5.3.3",
     "alova": "^2.13.1",
     "async-validator": "^4.2.5",
+    "dev": "^0.1.3",
     "history": "^5.3.0",
     "jwt-decode": "^3.1.2",
     "lodash-es": "^4.17.21",
@@ -40,19 +42,18 @@
     "react-dom": "latest",
     "react-dropzone": "^14.2.3",
     "react-icons": "^4.11.0",
-    "react-router-dom": "^6.16.0",
+    "react-router-dom": "^6.17.0",
     "react-toastify": "^9.1.3",
     "sockette": "^2.0.6",
     "typesafe-i18n": "^5.26.2",
     "typescript": "^5.2.2"
   },
   "devDependencies": {
-    "@babel/core": "^7.23.2",
     "@preact/compat": "^17.1.2",
     "@preact/preset-vite": "^2.6.0",
-    "@types/babel__core": "^7",
-    "@typescript-eslint/eslint-plugin": "^6.7.5",
-    "@typescript-eslint/parser": "^6.7.5",
+    "@typescript-eslint/eslint-plugin": "^6.8.0",
+    "@typescript-eslint/parser": "^6.8.0",
+    "concurrently": "^8.2.1",
     "eslint": "^8.51.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^17.1.0",
@@ -64,13 +65,12 @@
     "eslint-plugin-prettier": "alpha",
     "eslint-plugin-react": "^7.33.2",
     "eslint-plugin-react-hooks": "^4.6.0",
-    "npm-run-all": "^4.1.5",
     "preact": "^10.18.1",
     "prettier": "^3.0.3",
     "rollup-plugin-visualizer": "^5.9.2",
-    "terser": "^5.21.0",
+    "terser": "^5.22.0",
     "vite": "^4.4.11",
-    "vite-plugin-svgr": "^4.1.0",
+    "vite-plugin-imagemin": "^0.6.1",
     "vite-tsconfig-paths": "^4.2.1"
   },
   "packageManager": "yarn@4.0.0-rc.53"

interface/progmem-generator.js

@@ -5,6 +5,23 @@ var mime = require('mime-types');
 
 const ARDUINO_INCLUDES = '#include <Arduino.h>\n\n';
 const INDENT = '  ';
+const outputPath = '../lib/framework/WWWData.h';
+const sourcePath = './dist';
+const bytesPerLine = 20;
+var totalSize = 0;
+
+const generateWWWClass = () =>
+  `typedef std::function<void(const String& uri, const String& contentType, const uint8_t * content, size_t len)> RouteRegistrationHandler;
+
+class WWWData {
+${indent}public:
+${indent.repeat(2)}static void registerRoutes(RouteRegistrationHandler handler) {
+${fileInfo
+  .map((file) => `${indent.repeat(3)}handler("${file.uri}", "${file.mimeType}", ${file.variable}, ${file.size});`)
+  .join('\n')}
+${indent.repeat(2)}}
+};
+`;
 
 function getFilesSync(dir, files = []) {
   readdirSync(dir, { withFileTypes: true }).forEach((entry) => {
@@ -18,10 +35,6 @@ function getFilesSync(dir, files = []) {
   return files;
 }
 
-// function coherseToBuffer(input) {
-//   return Buffer.isBuffer(input) ? input : Buffer.from(input);
-// }
-
 function cleanAndOpen(path) {
   if (existsSync(path)) {
     unlinkSync(path);
@@ -29,90 +42,58 @@ function cleanAndOpen(path) {
   return createWriteStream(path, { flags: 'w+' });
 }
 
-export default function ProgmemGenerator({ outputPath = './WWWData.h', bytesPerLine = 20 }) {
-  return {
-    name: 'ProgmemGenerator',
-    writeBundle: () => {
-      console.log('Generating ' + outputPath);
-      const includes = ARDUINO_INCLUDES;
-      const indent = INDENT;
-      const fileInfo = [];
-      const writeStream = cleanAndOpen(resolve(outputPath));
-
-      try {
-        const writeIncludes = () => {
-          writeStream.write(includes);
-        };
-
-        const writeFile = (relativeFilePath, buffer) => {
-          const variable = 'ESP_REACT_DATA_' + fileInfo.length;
-          const mimeType = mime.lookup(relativeFilePath);
-          var size = 0;
-          writeStream.write('const uint8_t ' + variable + '[] = {');
-          // const zipBuffer = zlib.brotliCompressSync(buffer, { quality: 1 });
-          const zipBuffer = zlib.gzipSync(buffer);
-          zipBuffer.forEach((b) => {
-            if (!(size % bytesPerLine)) {
-              writeStream.write('\n');
-              writeStream.write(indent);
-            }
-            writeStream.write('0x' + ('00' + b.toString(16).toUpperCase()).substr(-2) + ',');
-            size++;
-          });
-          if (size % bytesPerLine) {
-            writeStream.write('\n');
-          }
-          writeStream.write('};\n\n');
-          fileInfo.push({
-            uri: '/' + relativeFilePath.replace(sep, '/'),
-            mimeType,
-            variable,
-            size
-          });
-        };
-
-        const writeFiles = () => {
-          // process static files
-          const buildPath = resolve('build');
-          for (const filePath of getFilesSync(buildPath)) {
-            const readStream = readFileSync(filePath);
-            const relativeFilePath = relative(buildPath, filePath);
-            writeFile(relativeFilePath, readStream);
-          }
-
-          // process assets
-          // const { assets } = compilation;
-          // Object.keys(assets).forEach((relativeFilePath) => {
-          //   writeFile(relativeFilePath, coherseToBuffer(assets[relativeFilePath].source()));
-          // });
-        };
-
-        const generateWWWClass = () =>
-          `typedef std::function<void(const String& uri, const String& contentType, const uint8_t * content, size_t len)> RouteRegistrationHandler;
-
-class WWWData {
-${indent}public:
-${indent.repeat(2)}static void registerRoutes(RouteRegistrationHandler handler) {
-${fileInfo
-  .map((file) => `${indent.repeat(3)}handler("${file.uri}", "${file.mimeType}", ${file.variable}, ${file.size});`)
-  .join('\n')}
-${indent.repeat(2)}}
-};
-`;
-        const writeWWWClass = () => {
-          writeStream.write(generateWWWClass());
-        };
-
-        writeIncludes();
-        writeFiles();
-        writeWWWClass();
-
-        writeStream.on('finish', () => {
-          // callback();
-        });
-      } finally {
-        writeStream.end();
-      }
+const writeFile = (relativeFilePath, buffer) => {
+  const variable = 'ESP_REACT_DATA_' + fileInfo.length;
+  const mimeType = mime.lookup(relativeFilePath);
+  var size = 0;
+  writeStream.write('const uint8_t ' + variable + '[] = {');
+  // const zipBuffer = zlib.brotliCompressSync(buffer, { quality: 1 });
+  const zipBuffer = zlib.gzipSync(buffer, { level: 9 });
+  zipBuffer.forEach((b) => {
+    if (!(size % bytesPerLine)) {
+      writeStream.write('\n');
+      writeStream.write(indent);
     }
-  };
+    writeStream.write('0x' + ('00' + b.toString(16).toUpperCase()).slice(-2) + ',');
+    size++;
+  });
+  if (size % bytesPerLine) {
+    writeStream.write('\n');
+  }
+  writeStream.write('};\n\n');
+  fileInfo.push({
+    uri: '/' + relativeFilePath.replace(sep, '/'),
+    mimeType,
+    variable,
+    size
+  });
+
+  // console.log(relativeFilePath + ' (size ' + size + ' bytes)');
+  totalSize += size;
+};
+
+// start
+console.log('Generating ' + outputPath + ' from ' + sourcePath);
+const includes = ARDUINO_INCLUDES;
+const indent = INDENT;
+const fileInfo = [];
+const writeStream = cleanAndOpen(resolve(outputPath));
+
+// includes
+writeStream.write(includes);
+
+// process static files
+const buildPath = resolve(sourcePath);
+for (const filePath of getFilesSync(buildPath)) {
+  const readStream = readFileSync(filePath);
+  const relativeFilePath = relative(buildPath, filePath);
+  writeFile(relativeFilePath, readStream);
 }
+
+// add class
+writeStream.write(generateWWWClass());
+
+// end
+writeStream.end();
+
+console.log('Total size: ' + totalSize / 1000 + ' KB');

interface/tsconfig.json

@@ -3,7 +3,7 @@
     "target": "ESNext",
     "useDefineForClassFields": true,
     "lib": ["DOM", "DOM.Iterable", "ESNext"],
-    "types": ["vite/client", "vite-plugin-svgr/client", "node"],
+    "types": ["node"],
     "allowJs": false,
     "skipLibCheck": true,
     "esModuleInterop": false,
@@ -26,6 +26,6 @@
       "@/*": ["src/*"]
     }
   },
-  "include": ["src/**/*", "vite.config.ts", "progmem-generator.js"],
+  "include": ["src/**/*", "vite.config.ts"],
   "exclude": ["node_modules", "dist", "src/**/*.test.tsx", "src/**/*.test.ts"]
 }

interface/vite.config.ts

@@ -1,58 +1,18 @@
-import { defineConfig, type PluginOption } from 'vite';
+import { defineConfig } from 'vite';
 import viteTsconfigPaths from 'vite-tsconfig-paths';
-import svgrPlugin from 'vite-plugin-svgr';
-import { visualizer } from 'rollup-plugin-visualizer';
-import ProgmemGenerator from './progmem-generator';
 import preact from '@preact/preset-vite';
+import viteImagemin from 'vite-plugin-imagemin';
+import { visualizer } from 'rollup-plugin-visualizer';
 
 export default defineConfig(({ command, mode }) => {
-  if (mode === 'hosted') {
+  // standalone build for development - runs the server
+  if (command === 'serve') {
+    console.log('Preparing for standalone build with server, mode=' + mode);
     return {
-      // hosted, ignore all errors, output to dist
-      plugins: [preact(), viteTsconfigPaths(), svgrPlugin(), visualizer({ gzipSize: true }) as PluginOption]
-    };
-  } else {
-    // normal build
-    return {
-      plugins: [
-        preact(),
-        viteTsconfigPaths(),
-        svgrPlugin(),
-        ProgmemGenerator({ outputPath: '../lib/framework/WWWData.h', bytesPerLine: 20 })
-      ],
-
-      build: {
-        outDir: 'build',
-        chunkSizeWarningLimit: 1024,
-        sourcemap: false,
-        manifest: false,
-        minify: mode === 'development' ? false : 'terser',
-        rollupOptions: {
-          /**
-           * Ignore "use client" waning since we are not using SSR
-           */
-          onwarn(warning, warn) {
-            if (warning.code === 'MODULE_LEVEL_DIRECTIVE' && warning.message.includes(`"use client"`)) {
-              return;
-            }
-            warn(warning);
-          }
-        }
-      },
-
-      onwarn(warning, warn) {
-        if (warning.code === 'MODULE_LEVEL_DIRECTIVE') {
-          return;
-        }
-        warn(warning);
-      },
-
+      plugins: [preact(), viteTsconfigPaths()],
       server: {
         open: true,
-        port: 3000,
-        // watch: {
-        //   usePolling: true
-        // },
+        port: mode == 'production' ? 4173 : 3000,
         proxy: {
           '/rest': 'http://localhost:3080',
           '/api': {
@@ -69,4 +29,98 @@ export default defineConfig(({ command, mode }) => {
       }
     };
   }
+
+  // production build, both for hosted and building the firmware
+  if (command === 'build') {
+    console.log('Preparing for production build, mode is ' + mode);
+    return {
+      plugins: [
+        preact(),
+        viteTsconfigPaths(),
+        {
+          ...viteImagemin({
+            verbose: false,
+            gifsicle: {
+              optimizationLevel: 7,
+              interlaced: false
+            },
+            optipng: {
+              optimizationLevel: 7
+            },
+            mozjpeg: {
+              quality: 20
+            },
+            pngquant: {
+              quality: [0.8, 0.9],
+              speed: 4
+            },
+            svgo: {
+              plugins: [
+                {
+                  name: 'removeViewBox'
+                },
+                {
+                  name: 'removeEmptyAttrs',
+                  active: false
+                }
+              ]
+            }
+          }),
+          enforce: 'pre'
+        },
+        visualizer({
+          template: 'treemap', // or sunburst
+          open: false,
+          gzipSize: true,
+          brotliSize: true,
+          filename: 'analyse.html' // will be saved in project's root
+        })
+      ],
+
+      build: {
+        target: 'es2022',
+        outDir: 'dist',
+        reportCompressedSize: false,
+        chunkSizeWarningLimit: 1024,
+        sourcemap: false,
+        manifest: false,
+        minify: 'terser',
+        terserOptions: {
+          compress: {
+            passes: 4,
+            arrows: true,
+            drop_console: true,
+            drop_debugger: true,
+            sequences: true
+          },
+          mangle: {
+            toplevel: true,
+            module: true,
+            properties: {
+              regex: /^_/
+            }
+          },
+          ecma: 5,
+          enclose: false,
+          keep_classnames: false,
+          keep_fnames: false,
+          ie8: false,
+          module: false,
+          nameCache: null,
+          safari10: false,
+          toplevel: false
+        },
+
+        rollupOptions: {
+          // Ignore "use client" waning since we are not using SSR
+          onwarn(warning, warn) {
+            if (warning.code === 'MODULE_LEVEL_DIRECTIVE' && warning.message.includes(`"use client"`)) {
+              return;
+            }
+            warn(warning);
+          }
+        }
+      }
+    };
+  }
 });

mock-api/README.md

@@ -2,4 +2,4 @@
 
 When developing and testing the web interface, it's handy not to bother with re-flashing an ESP32 each time. The idea is to mimic the ESP using a mock/stub server that responds to the REST (HTTP POST & GET) and WebSocket calls.
 
-To use first make sure you have nodejs installed (>v18) then install yarn (`npm install -g yarn`). Now type `yarn` from this `mock-api` folder. To run EMS-ESP's WebUI navigate up to the `interface` folder and type `yarn run standalone`. This will start the mock API server on port 3080 using static dummy data from `mock-api/server.js` and also the web server which is at <http://localhost:3000>.
+To use first make sure you have nodejs installed (>v18) then install yarn (`npm install -g yarn`). Now type `yarn` from this `mock-api` folder. To run EMS-ESP's WebUI navigate up to the `interface` folder and type `yarn standalone`. This will start the mock API server on port 3080 using static dummy data from `mock-api/server.js` and also the web server which is at <http://localhost:3000>.

scripts/build_interface.py

@@ -1,61 +1,25 @@
 from pathlib import Path
-from shutil import copytree, rmtree, copyfileobj
 import os
-import gzip
-
-# brotli has better compression than gzip but requires https so leaving here for future
-# import brotli
 
 Import("env")
 
 
-def gzipFile(file):
-    with open(file, 'rb') as f_in:
-        with gzip.open(file + '.gz', 'wb') as f_out:
-            copyfileobj(f_in, f_out)
-    os.remove(file)
-
-    # brotli version:
-    # with open(file + '.br', 'wb') as f_out:
-    #     with open(file, 'rb') as f_in:
-    #         f_out.write(brotli.compress(f_in.read(), quality=11))
-    #     os.remove(file)
-
-
-def flagExists(flag):
-    buildFlags = env.ParseFlags(env["BUILD_FLAGS"])
-    for define in buildFlags.get("CPPDEFINES"):
-        if (define == flag or (isinstance(define, list) and define[0] == flag)):
-            return True
-
-
 def buildWeb():
     os.chdir("interface")
     print("Building web interface...")
     try:
         env.Execute("yarn")
-        env.Execute("yarn run typesafe-i18n --no-watch")
+        env.Execute("yarn typesafe-i18n --no-watch")
         with open("./src/i18n/i18n-util.ts") as r:
             text = r.read().replace("Locales = 'pl'", "Locales = 'en'")
         with open("./src/i18n/i18n-util.ts", "w") as w:
             w.write(text)
-        print("Setting locale to 'en'")
-        env.Execute("yarn run build")
-
-        buildPath = Path("build")
-        wwwPath = Path("../data/www")
-
-        if wwwPath.exists() and wwwPath.is_dir():
-            rmtree(wwwPath)
-
-        print("Copying web files from build to data/www...")
-        copytree(buildPath, wwwPath)
-        for currentpath, folders, files in os.walk(wwwPath):
-            for file in files:
-                gzipFile(os.path.join(currentpath, file))
-
+        print("Setting WebUI locale to 'en'")
+        env.Execute("yarn build")
+        env.Execute("yarn webUI")
     finally:
         os.chdir("..")
 
+
 if not (env.IsCleanTarget()):
     buildWeb()