diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 000000000..473529a9b --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,152 @@ +--- +Checks: >- + *, + -abseil-*, + -android-*, + -boost-*, + -bugprone-branch-clone, + -bugprone-narrowing-conversions, + -bugprone-signed-char-misuse, + -bugprone-too-small-loop-variable, + -cert-dcl50-cpp, + -cert-err58-cpp, + -cert-oop57-cpp, + -cert-str34-c, + -clang-analyzer-optin.cplusplus.UninitializedObject, + -clang-analyzer-osx.*, + -clang-diagnostic-delete-abstract-non-virtual-dtor, + -clang-diagnostic-delete-non-abstract-non-virtual-dtor, + -clang-diagnostic-shadow-field, + -clang-diagnostic-sign-compare, + -clang-diagnostic-unused-variable, + -clang-diagnostic-unused-const-variable, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-goto, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-init-variables, + -cppcoreguidelines-macro-usage, + -cppcoreguidelines-narrowing-conversions, + -cppcoreguidelines-non-private-member-variables-in-classes, + -cppcoreguidelines-owning-memory, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-const-cast, + -cppcoreguidelines-pro-type-cstyle-cast, + -cppcoreguidelines-pro-type-member-init, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-pro-type-static-cast-downcast, + -cppcoreguidelines-pro-type-union-access, + -cppcoreguidelines-pro-type-vararg, + -cppcoreguidelines-special-member-functions, + -fuchsia-default-arguments, + -fuchsia-multiple-inheritance, + -fuchsia-overloaded-operator, + -fuchsia-statically-constructed-objects, + -fuchsia-default-arguments-declarations, + -fuchsia-default-arguments-calls, + -google-build-using-namespace, + -google-explicit-constructor, + -google-readability-braces-around-statements, + -google-readability-casting, + -google-readability-todo, + -google-runtime-references, + -hicpp-*, + -llvm-else-after-return, + -llvm-header-guard, + -llvm-include-order, + -llvm-qualified-auto, + -llvmlibc-*, + -misc-non-private-member-variables-in-classes, + -misc-no-recursion, + -misc-unused-parameters, + -modernize-avoid-c-arrays, + -modernize-return-braced-init-list, + -modernize-use-auto, + -modernize-use-default-member-init, + -modernize-use-equals-default, + -modernize-use-trailing-return-type, + -mpi-*, + -objc-*, + -readability-braces-around-statements, + -readability-const-return-type, + -readability-convert-member-functions-to-static, + -readability-else-after-return, + -readability-implicit-bool-conversion, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-make-member-function-const, + -readability-named-parameter, + -readability-qualified-auto, + -readability-redundant-access-specifiers, + -readability-redundant-member-init, + -readability-redundant-string-init, + -readability-uppercase-literal-suffix, + -readability-use-anyofallof, + -warnings-as-errors +WarningsAsErrors: '*' +AnalyzeTemporaryDtors: false +FormatStyle: google +CheckOptions: + - key: google-readability-braces-around-statements.ShortStatementLines + value: '1' + - key: google-readability-function-size.StatementThreshold + value: '800' + - key: google-readability-namespace-comments.ShortNamespaceLines + value: '10' + - key: google-readability-namespace-comments.SpacesBeforeComments + value: '2' + - key: modernize-loop-convert.MaxCopySize + value: '16' + - key: modernize-loop-convert.MinConfidence + value: reasonable + - key: modernize-loop-convert.NamingStyle + value: CamelCase + - key: modernize-pass-by-value.IncludeStyle + value: llvm + - key: modernize-replace-auto-ptr.IncludeStyle + value: llvm + - key: modernize-use-nullptr.NullMacros + value: 'NULL' + - key: readability-identifier-naming.LocalVariableCase + value: 'lower_case' + - key: readability-identifier-naming.ClassCase + value: 'CamelCase' + - key: readability-identifier-naming.StructCase + value: 'CamelCase' + - key: readability-identifier-naming.EnumCase + value: 'CamelCase' + - key: readability-identifier-naming.EnumConstantCase + value: 'UPPER_CASE' + - key: readability-identifier-naming.StaticConstantCase + value: 'UPPER_CASE' + - key: readability-identifier-naming.StaticVariableCase + value: 'UPPER_CASE' + - key: readability-identifier-naming.GlobalConstantCase + value: 'UPPER_CASE' + - key: readability-identifier-naming.ParameterCase + value: 'lower_case' + - key: readability-identifier-naming.PrivateMemberPrefix + value: 'NO_PRIVATE_MEMBERS_ALWAYS_USE_PROTECTED' + - key: readability-identifier-naming.PrivateMethodPrefix + value: 'NO_PRIVATE_METHODS_ALWAYS_USE_PROTECTED' + - key: readability-identifier-naming.ClassMemberCase + value: 'lower_case' + - key: readability-identifier-naming.ClassMemberCase + value: 'lower_case' + - key: readability-identifier-naming.ProtectedMemberCase + value: 'lower_case' + - key: readability-identifier-naming.ProtectedMemberSuffix + value: '_' + - key: readability-identifier-naming.FunctionCase + value: 'lower_case' + - key: readability-identifier-naming.ClassMethodCase + value: 'lower_case' + - key: readability-identifier-naming.ProtectedMethodCase + value: 'lower_case' + - key: readability-identifier-naming.ProtectedMethodSuffix + value: '_' + - key: readability-identifier-naming.VirtualMethodCase + value: 'lower_case' + - key: readability-identifier-naming.VirtualMethodSuffix + value: '' diff --git a/.gitignore b/.gitignore index dd1e52450..1f342b86d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,6 @@ emsesp /interface/build node_modules /interface/.eslintcache -test.sh \ No newline at end of file +test.sh +scripts/__pycache__ +.temp diff --git a/scripts/clang-format.py b/scripts/clang-format.py new file mode 100755 index 000000000..185094457 --- /dev/null +++ b/scripts/clang-format.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +# copied from esphome +# run from Linux using ./scripts/clang-forrmat.py + +import argparse +import multiprocessing +import os +import queue +import re +import subprocess +import sys +import threading +import click + +sys.path.append(os.path.dirname(__file__)) +from helpers import get_output, src_files, filter_changed + +def run_format(args, queue, lock, failed_files): + """Takes filenames out of queue and runs clang-format on them.""" + while True: + path = queue.get() + invocation = ['clang-format'] + if args.inplace: + invocation.append('-i') + else: + invocation.extend(['--dry-run', '-Werror']) + invocation.append(path) + + proc = subprocess.run(invocation, capture_output=True, encoding='utf-8') + if proc.returncode != 0: + with lock: + print() + print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path)) + print(proc.stdout) + print(proc.stderr) + print() + failed_files.append(path) + queue.task_done() + + +def progress_bar_show(value): + return value if value is not None else '' + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-j', '--jobs', type=int, + default=multiprocessing.cpu_count(), + help='number of format instances to be run in parallel.') + parser.add_argument('files', nargs='*', default=[], + help='files to be processed (regex on path)') + parser.add_argument('-i', '--inplace', action='store_true', + help='reformat files in-place') + parser.add_argument('-c', '--changed', action='store_true', + help='only run on changed files') + args = parser.parse_args() + + try: + get_output('clang-format', '-version') + except: + print(""" + Oops. It looks like clang-format is not installed. + + Please check you can run "clang-format -version" in your terminal and install + clang-format (v11 or v12) if necessary. + + Note you can also upload your code as a pull request on GitHub and see the CI check + output to apply clang-format. + """) + return 1 + + files = [] + # all files + # for path in git_ls_files(['*.cpp', '*.h', '*.tcc']): + # files.append(os.path.relpath(path, os.getcwd())) + + # just under src + for path in src_files(['.cpp', '.h']): + files.append(os.path.relpath(path, os.getcwd())) + + if args.files: + file_name_re = re.compile('|'.join(args.files)) + files = [p for p in files if file_name_re.search(p)] + + if args.changed: + files = filter_changed(files) + + files.sort() + + failed_files = [] + try: + task_queue = queue.Queue(args.jobs) + lock = threading.Lock() + for _ in range(args.jobs): + t = threading.Thread(target=run_format, + args=(args, task_queue, lock, failed_files)) + t.daemon = True + t.start() + + # Fill the queue with files. + with click.progressbar(files, width=30, file=sys.stderr, + item_show_func=progress_bar_show) as bar: + for name in bar: + task_queue.put(name) + + # Wait for all threads to be done. + task_queue.join() + + except KeyboardInterrupt: + print() + print('Ctrl-C detected, goodbye.') + os.kill(0, 9) + + sys.exit(len(failed_files)) + + +if __name__ == '__main__': + main() diff --git a/scripts/clang-tidy.py b/scripts/clang-tidy.py new file mode 100755 index 000000000..6a86f01d3 --- /dev/null +++ b/scripts/clang-tidy.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +# copied from esphome +# run from Linux using ./scripts/clang-forrmat.py + +import argparse +import multiprocessing +import os +import queue +import shutil +import subprocess +import sys +import tempfile +import threading +import click +import pexpect + +sys.path.append(os.path.dirname(__file__)) +from helpers import shlex_quote, get_output, \ + build_all_include, temp_header_file, filter_changed, load_idedata, src_files + +def clang_options(idedata): + cmd = [ + # target 32-bit arch (this prevents size mismatch errors on a 64-bit host) + '-m32', + # disable built-in include directories from the host + '-nostdinc', + '-nostdinc++', + # allow to condition code on the presence of clang-tidy + '-DCLANG_TIDY' + ] + + # copy compiler flags, except those clang doesn't understand. + cmd.extend(flag for flag in idedata['cxx_flags'].split(' ') + if flag not in ('-free', '-fipa-pta', '-mlongcalls', '-mtext-section-literals')) + + # defines + cmd.extend(f'-D{define}' for define in idedata['defines']) + + # add include directories, using -isystem for dependencies to suppress their errors + for directory in idedata['includes']['toolchain']: + cmd.extend(['-isystem', directory]) + for directory in sorted(set(idedata['includes']['build'])): + dependency = "framework-arduino" in directory or "/libdeps/" in directory + cmd.extend(['-isystem' if dependency else '-I', directory]) + + return cmd + +def run_tidy(args, options, tmpdir, queue, lock, failed_files): + while True: + path = queue.get() + invocation = ['clang-tidy'] + + if tmpdir is not None: + invocation.append('--export-fixes') + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) + os.close(handle) + invocation.append(name) + + if args.quiet: + invocation.append('-quiet') + + invocation.append(os.path.abspath(path)) + invocation.append('--') + invocation.extend(options) + invocation_s = ' '.join(shlex_quote(x) for x in invocation) + + # Use pexpect for a pseudy-TTY with colored output + output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8', + timeout=15 * 60) + if rc != 0: + with lock: + print() + print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path)) + print(output) + print() + failed_files.append(path) + queue.task_done() + +def progress_bar_show(value): + if value is None: + return '' + +def split_list(a, n): + k, m = divmod(len(a), n) + return [a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n)] + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-j', '--jobs', type=int, + default=multiprocessing.cpu_count(), + help='number of tidy instances to be run in parallel.') + parser.add_argument('files', nargs='*', default=[], + help='files to be processed (regex on path)') + parser.add_argument('--fix', action='store_true', help='apply fix-its') + parser.add_argument('-q', '--quiet', action='store_false', + help='run clang-tidy in quiet mode') + parser.add_argument('-c', '--changed', action='store_true', + help='only run on changed files') + parser.add_argument('--split-num', type=int, help='split the files into X jobs.', + default=None) + parser.add_argument('--split-at', type=int, help='which split is this? starts at 1', + default=None) + parser.add_argument('--all-headers', action='store_true', + help='create a dummy file that checks all headers') + args = parser.parse_args() + + try: + get_output('clang-tidy', '-version') + except: + print(""" + Oops. It looks like clang-tidy is not installed. + + Please check you can run "clang-tidy -version" in your terminal and install + clang-tidy (v11 or v12) if necessary. + + Note you can also upload your code as a pull request on GitHub and see the CI check + output to apply clang-tidy. + """) + return 1 + + idedata = load_idedata("clang-tidy") + options = clang_options(idedata) + + files = [] + for path in src_files(['.cpp']): + files.append(os.path.relpath(path, os.getcwd())) + + if args.files: + # Match against files specified on command-line + file_name_re = re.compile('|'.join(args.files)) + files = [p for p in files if file_name_re.search(p)] + + if args.changed: + files = filter_changed(files) + + files.sort() + + if args.split_num: + files = split_list(files, args.split_num)[args.split_at - 1] + + if args.all_headers and args.split_at in (None, 1): + build_all_include() + files.insert(0, temp_header_file) + + tmpdir = None + if args.fix: + tmpdir = tempfile.mkdtemp() + + failed_files = [] + try: + task_queue = queue.Queue(args.jobs) + lock = threading.Lock() + for _ in range(args.jobs): + t = threading.Thread(target=run_tidy, + args=(args, options, tmpdir, task_queue, lock, failed_files)) + t.daemon = True + t.start() + + # Fill the queue with files. + with click.progressbar(files, width=30, file=sys.stderr, + item_show_func=progress_bar_show) as bar: + for name in bar: + task_queue.put(name) + + # Wait for all threads to be done. + task_queue.join() + + except KeyboardInterrupt: + print() + print('Ctrl-C detected, goodbye.') + if tmpdir: + shutil.rmtree(tmpdir) + os.kill(0, 9) + + if args.fix and failed_files: + print('Applying fixes ...') + try: + subprocess.call(['clang-apply-replacements-12', tmpdir]) + except: + print('Error applying fixes.\n', file=sys.stderr) + raise + + sys.exit(len(failed_files)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/helpers.py b/scripts/helpers.py new file mode 100755 index 000000000..efdce500d --- /dev/null +++ b/scripts/helpers.py @@ -0,0 +1,122 @@ +import os.path +import re +import subprocess +import json +from pathlib import Path + +root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", ".."))) +basepath = os.path.join(root_path, "src") +temp_folder = os.path.join(root_path, ".temp") +temp_header_file = os.path.join(temp_folder, "all-include.cpp") + + +def shlex_quote(s): + if not s: + return "''" + if re.search(r"[^\w@%+=:,./-]", s) is None: + return s + + return "'" + s.replace("'", "'\"'\"'") + "'" + +def build_all_include(): + # Build a cpp file that includes all header files in this repo. + # Otherwise header-only integrations would not be tested by clang-tidy + headers = [] + for path in walk_files(basepath): + filetypes = (".h",) + ext = os.path.splitext(path)[1] + if ext in filetypes: + path = os.path.relpath(path, root_path) + include_p = path.replace(os.path.sep, "/") + headers.append(f'#include "{include_p}"') + headers.sort() + headers.append("") + content = "\n".join(headers) + p = Path(temp_header_file) + p.parent.mkdir(exist_ok=True) + p.write_text(content) + +def src_files(filetypes=None): + file_list = [] + for path in walk_files(basepath): + ext = os.path.splitext(path)[1] + if ext in filetypes: + path = os.path.relpath(path, root_path) + file_list.append(path) + return file_list + +def walk_files(path): + for root, _, files in os.walk(path): + for name in files: + yield os.path.join(root, name) + +def get_output(*args): + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, err = proc.communicate() + return output.decode("utf-8") + +def get_err(*args): + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, err = proc.communicate() + return err.decode("utf-8") + +def splitlines_no_ends(string): + return [s.strip() for s in string.splitlines()] + +def changed_files(): + check_remotes = ["upstream", "origin"] + check_remotes.extend(splitlines_no_ends(get_output("git", "remote"))) + for remote in check_remotes: + command = ["git", "merge-base", f"refs/remotes/{remote}/dev", "HEAD"] + try: + merge_base = splitlines_no_ends(get_output(*command))[0] + break + except: + pass + else: + raise ValueError("Git not configured") + command = ["git", "diff", merge_base, "--name-only"] + changed = splitlines_no_ends(get_output(*command)) + changed = [os.path.relpath(f, os.getcwd()) for f in changed] + changed.sort() + return changed + +def filter_changed(files): + changed = changed_files() + files = [f for f in files if f in changed] + print("Changed files:") + if not files: + print(" No changed files!") + for c in files: + print(f" {c}") + return files + +def git_ls_files(patterns=None): + command = ["git", "ls-files", "-s"] + if patterns is not None: + command.extend(patterns) + proc = subprocess.Popen(command, stdout=subprocess.PIPE) + output, err = proc.communicate() + lines = [x.split() for x in output.decode("utf-8").splitlines()] + return {s[3].strip(): int(s[0]) for s in lines} + +def load_idedata(environment): + platformio_ini = Path(root_path) / "platformio.ini" + temp_idedata = Path(temp_folder) / f"idedata-{environment}.json" + if not platformio_ini.is_file() or not temp_idedata.is_file(): + changed = True + elif platformio_ini.stat().st_mtime >= temp_idedata.stat().st_mtime: + changed = True + else: + changed = False + + if not changed: + return json.loads(temp_idedata.read_text()) + + stdout = subprocess.check_output(["pio", "run", "-t", "idedata", "-e", environment]) + match = re.search(r'{\s*".*}', stdout.decode("utf-8")) + data = json.loads(match.group()) + + temp_idedata.parent.mkdir(exist_ok=True) + temp_idedata.write_text(json.dumps(data, indent=2) + "\n") + return data diff --git a/scripts/rename_fw.py b/scripts/rename_fw.py old mode 100644 new mode 100755 diff --git a/scripts/upload_fw.py b/scripts/upload_fw.py old mode 100644 new mode 100755