diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 46861500705..9d7983b1595 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -19,17 +19,24 @@ jobs: name: Scan code format and license if: github.repository_owner == 'RT-Thread' steps: - - uses: actions/checkout@main + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@main + uses: actions/setup-python@v5 with: python-version: 3.8 - - - name: Check Format and License + + - name: Install clang-format + shell: bash + run: | + python -m pip install clang-format + clang-format --version + + - name: Check clang-format shell: bash run: | - pip install click chardet PyYaml - python tools/ci/file_check.py check 'https://github.com/RT-Thread/rt-thread' 'master' + python tools/ci/clang_format_check.py --repo 'https://github.com/RT-Thread/rt-thread' --branch 'master' # # Post CI status to PR comment # post-ci-status: @@ -42,4 +49,4 @@ jobs: # pr_number: ${{ github.event.pull_request.number }} # permissions: # pull-requests: write - # issues: write \ No newline at end of file + # issues: write diff --git a/tools/ci/clang_format_check.py b/tools/ci/clang_format_check.py new file mode 100644 index 00000000000..ef72cffa650 --- /dev/null +++ b/tools/ci/clang_format_check.py @@ -0,0 +1,266 @@ +# +# Copyright (c) 2006-2026, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 +# +# Change Logs: +# Date Author Notes +# 2026-07-07 RT-Thread add clang-format CI check +# + +import argparse +import fnmatch +import logging +import re +import subprocess +import sys +from pathlib import Path + + +SOURCE_EXTENSIONS = ( + ".c", + ".h", + ".cpp", + ".hpp", + ".cc", + ".hh", + ".C", + ".H", + ".cp", + ".cxx", + ".hxx", + ".inc", + ".inl", + ".ipp", + ".tpp", + ".txx", +) + + +HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +def init_logger(): + logging.basicConfig(level=logging.INFO, format="[%(filename)s %(lineno)d %(levelname)s] %(message)s ") + + +def run_git(args, check=True): + result = subprocess.run(["git"] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if check and result.returncode != 0: + raise RuntimeError(result.stderr.decode("utf-8", errors="replace")) + return result + + +def ensure_remote(remote, repo): + result = run_git(["remote", "get-url", remote], check=False) + if result.returncode == 0: + run_git(["remote", "set-url", remote, repo]) + else: + run_git(["remote", "add", remote, repo]) + + +def split_git_paths(output): + paths = [] + for item in output.split(b"\0"): + if item: + paths.append(item.decode("utf-8", errors="surrogateescape")) + return paths + + +def get_changed_files(repo, branch, remote): + ensure_remote(remote, repo) + run_git(["fetch", "--no-tags", remote, branch]) + + merge_base = run_git(["merge-base", "FETCH_HEAD", "HEAD"]).stdout.decode("utf-8", errors="replace").strip() + if not merge_base: + raise RuntimeError("git merge-base FETCH_HEAD HEAD returned no commit") + + diff_range = [merge_base, "HEAD"] + result = run_git(["diff", "--name-only", "-z", "--diff-filter=ACMR", "--no-renames"] + diff_range + ["--"]) + return split_git_paths(result.stdout), diff_range + + +def merge_line_ranges(ranges): + if not ranges: + return [] + + merged = [] + for start, end in sorted(ranges): + if not merged or start > merged[-1][1] + 1: + merged.append([start, end]) + else: + merged[-1][1] = max(merged[-1][1], end) + return [(start, end) for start, end in merged] + + +def parse_changed_line_ranges(diff_text): + ranges = [] + for line in diff_text.splitlines(): + match = HUNK_HEADER.match(line) + if not match: + continue + + start = int(match.group(1)) + count = int(match.group(2) or "1") + if count == 0: + continue + ranges.append((start, start + count - 1)) + + return merge_line_ranges(ranges) + + +def get_changed_line_ranges(diff_range, files): + line_ranges = {} + for file_path in files: + result = run_git(["diff", "--unified=0", "--diff-filter=ACMR", "--no-renames"] + diff_range + ["--", file_path]) + line_ranges[file_path] = parse_changed_line_ranges(result.stdout.decode("utf-8", errors="replace")) + return line_ranges + + +def normalize_path(path): + return path.as_posix().strip("/") + + +def match_path(path, pattern): + return fnmatch.fnmatchcase(path, pattern) or fnmatch.fnmatchcase("/" + path, pattern) + + +def ignore_pattern_matches(pattern, relative_file): + pattern = pattern.strip() + if not pattern or pattern.startswith("#"): + return False + + pattern = pattern.lstrip("/") + dir_only = pattern.endswith("/") + pattern = pattern.rstrip("/") + if not pattern: + return False + + relative_file = relative_file.strip("/") + if dir_only: + parts = relative_file.split("/") + dirs = ["/".join(parts[:index]) for index in range(1, len(parts))] + return match_path(relative_file, pattern) or any(match_path(directory, pattern) for directory in dirs) + + if "/" not in pattern: + return any(fnmatch.fnmatchcase(part, pattern) for part in relative_file.split("/")) + + return match_path(relative_file, pattern) + + +def is_ignored_by_clang_format(root, file_path): + absolute_file = (root / file_path).resolve() + try: + relative_to_root = absolute_file.relative_to(root) + except ValueError: + return False + + current_dir = absolute_file.parent + ignore_files = [] + while True: + ignore_file = current_dir / ".clang-format-ignore" + if ignore_file.exists(): + ignore_files.append(ignore_file) + if current_dir == root: + break + current_dir = current_dir.parent + + for ignore_file in reversed(ignore_files): + try: + ignore_dir = ignore_file.parent.relative_to(root) + if str(ignore_dir) == ".": + relative_file = normalize_path(relative_to_root) + else: + relative_file = normalize_path(relative_to_root.relative_to(ignore_dir)) + except ValueError: + continue + with open(ignore_file, "r", encoding="utf-8") as ignore: + for line in ignore: + if ignore_pattern_matches(line, relative_file): + logging.info("ignore file by %s: %s", ignore_file.relative_to(root).as_posix(), file_path) + return True + return False + + +def filter_source_files(root, files): + source_files = [] + for file_path in files: + path = root / file_path + if not path.is_file(): + continue + if not file_path.endswith(SOURCE_EXTENSIONS): + continue + if is_ignored_by_clang_format(root, Path(file_path)): + continue + source_files.append(file_path) + return source_files + + +def check_clang_format(files, clang_format, line_ranges=None): + failed_files = [] + for file_path in files: + invocation = [clang_format, "--dry-run", "--Werror", "--style=file"] + if line_ranges is not None: + ranges = line_ranges.get(file_path, []) + if not ranges: + logging.info("skip clang-format, no changed lines -> %s", file_path) + continue + for start, end in ranges: + invocation.append("--lines={}:{}".format(start, end)) + + invocation.append(file_path) + logging.info("check clang-format -> %s", file_path) + result = subprocess.run( + invocation, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + if result.stdout: + sys.stdout.write(result.stdout) + if result.stderr: + sys.stderr.write(result.stderr) + if result.returncode != 0: + failed_files.append(file_path) + + if failed_files: + logging.error("clang-format check failed, please run clang-format on the files above.") + return False + return True + + +def main(): + parser = argparse.ArgumentParser(description="Check changed source files with clang-format.") + parser.add_argument("--repo", default="https://github.com/RT-Thread/rt-thread") + parser.add_argument("--branch", default="master") + parser.add_argument("--remote", default="rtt_repo") + parser.add_argument("--clang-format-executable", default="clang-format") + parser.add_argument("files", nargs="*", help="files to check; git diff is used when omitted") + args = parser.parse_args() + + init_logger() + root = Path.cwd().resolve() + if args.files: + files = args.files + line_ranges = None + else: + files, diff_range = get_changed_files(args.repo, args.branch, args.remote) + line_ranges = get_changed_line_ranges(diff_range, files) + + source_files = filter_source_files(root, files) + if line_ranges is not None: + line_ranges = {file_path: line_ranges.get(file_path, []) for file_path in source_files} + + if not source_files: + logging.warning("There are no source files to check format.") + return 0 + + if not check_clang_format(source_files, args.clang_format_executable, line_ranges): + return 1 + + logging.info("clang-format check success.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci/file_check.py b/tools/ci/file_check.py deleted file mode 100644 index 597c8c33b05..00000000000 --- a/tools/ci/file_check.py +++ /dev/null @@ -1,292 +0,0 @@ -# -# Copyright (c) 2006-2022, RT-Thread Development Team -# -# SPDX-License-Identifier: Apache-2.0 -# -# Change Logs: -# Date Author Notes -# 2021-04-01 LiuKang the first version -# - -import os -import re -import sys -import click -import yaml -import chardet -import logging -import datetime - - -def init_logger(): - log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s " - date_format = '%Y-%m-%d %H:%M:%S %a ' - logging.basicConfig(level=logging.INFO, - format=log_format, - datefmt=date_format, - ) - - -class CheckOut: - def __init__(self, rtt_repo, rtt_branch): - self.root = os.getcwd() - self.rtt_repo = rtt_repo - self.rtt_branch = rtt_branch - - def __exclude_file(self, file_path): - dir_number = file_path.split('/') - ignore_path = file_path - - # gets the file path depth. - for i in dir_number: - # current directory. - dir_name = os.path.dirname(ignore_path) - ignore_path = dir_name - # judge the ignore file exists in the current directory. - ignore_file_path = os.path.join(dir_name, ".ignore_format.yml") - if not os.path.exists(ignore_file_path): - continue - try: - with open(ignore_file_path) as f: - ignore_config = yaml.safe_load(f.read()) - file_ignore = ignore_config.get("file_path", []) - dir_ignore = ignore_config.get("dir_path", []) - except Exception as e: - logging.error(e) - continue - logging.debug("ignore file path: {}".format(ignore_file_path)) - logging.debug("file_ignore: {}".format(file_ignore)) - logging.debug("dir_ignore: {}".format(dir_ignore)) - try: - # judge file_path in the ignore file. - for file in file_ignore: - if file is not None: - file_real_path = os.path.join(dir_name, file) - if file_real_path == file_path: - logging.info("ignore file path: {}".format(file_real_path)) - return 0 - - file_dir_path = os.path.dirname(file_path) - for _dir in dir_ignore: - if _dir is not None: - dir_real_path = os.path.join(dir_name, _dir) - if file_dir_path.startswith(dir_real_path): - logging.info("ignore dir path: {}".format(dir_real_path)) - return 0 - except Exception as e: - logging.error(e) - continue - - return 1 - - def get_new_file(self): - file_list = list() - try: - os.system('git remote add rtt_repo {}'.format(self.rtt_repo)) - os.system('git fetch rtt_repo') - os.system('git merge rtt_repo/{}'.format(self.rtt_branch)) - os.system('git reset rtt_repo/{} --soft'.format(self.rtt_branch)) - os.system('git status > git.txt') - except Exception as e: - logging.error(e) - return None - try: - with open('git.txt', 'r') as f: - file_lines = f.readlines() - except Exception as e: - logging.error(e) - return None - file_path = '' - for line in file_lines: - if 'new file' in line: - file_path = line.split('new file:')[1].strip() - logging.info('new file -> {}'.format(file_path)) - elif 'deleted' in line: - logging.info('deleted file -> {}'.format(line.split('deleted:')[1].strip())) - elif 'modified' in line: - file_path = line.split('modified:')[1].strip() - logging.info('modified file -> {}'.format(file_path)) - else: - continue - - result = self.__exclude_file(file_path) - if result != 0: - file_list.append(file_path) - - return file_list - - -class FormatCheck: - def __init__(self, file_list): - self.file_list = file_list - - def __check_rt_errorcode(self, line): - pattern = re.compile(r'return\s+(RT_ERROR|RT_ETIMEOUT|RT_EFULL|RT_EEMPTY|RT_ENOMEM|RT_ENOSYS|RT_EBUSY|RT_EIO|RT_EINTR|RT_EINVAL|RT_ENOENT|RT_ENOSPC|RT_EPERM|RT_ETRAP|RT_EFAULT)') - match = pattern.search(line) - if match: - return False - else: - return True - - def __check_file(self, file_lines, file_path): - line_num = 0 - check_result = True - for line in file_lines: - line_num += 1 - # check line start - line_start = line.replace(' ', '') - # find tab - if line_start.startswith('\t'): - logging.error("{} line[{}]: please use space replace tab at the start of this line.".format(file_path, line_num)) - check_result = False - # check line end - line_end = line.split('\n')[0] - if line_end.endswith(' ') or line_end.endswith('\t'): - logging.error("{} line[{}]: please delete extra space at the end of this line.".format(file_path, line_num)) - check_result = False - if self.__check_rt_errorcode(line) == False: - logging.error("{} line[{}]: the RT-Thread error code should return negative value. e.g. return -RT_ERROR".format(file_path, line_num)) - check_result = False - return check_result - - def check(self): - logging.info("Start to check files format.") - if len(self.file_list) == 0: - logging.warning("There are no files to check format.") - return True - encoding_check_result = True - format_check_fail_files = 0 - for file_path in self.file_list: - code = '' - if file_path.endswith(".c") or file_path.endswith(".h"): - try: - with open(file_path, 'rb') as f: - file = f.read() - # get file encoding - chardet_report = chardet.detect(file) - code = chardet_report['encoding'] - confidence = chardet_report['confidence'] - except Exception as e: - logging.error(e) - else: - continue - - if code != 'utf-8' and code != 'ascii' and confidence > 0.8: - logging.error("[{0}]: encoding {1} not utf-8, please format it.".format(file_path, code)) - encoding_check_result = False - else: - logging.info('[{0}]: encoding check success.'.format(file_path)) - - with open(file_path, 'r', encoding = "utf-8") as f: - file_lines = f.readlines() - if not self.__check_file(file_lines, file_path): - format_check_fail_files += 1 - - if (not encoding_check_result) or (format_check_fail_files != 0): - logging.error("files format check fail.") - return False - - logging.info("files format check success.") - - return True - - -class LicenseCheck: - def __init__(self, file_list): - self.file_list = file_list - - def check(self): - current_year = datetime.date.today().year - logging.info("current year: {}".format(current_year)) - if len(self.file_list) == 0: - logging.warning("There are no files to check license.") - return 0 - logging.info("Start to check files license.") - check_result = True - for file_path in self.file_list: - if file_path.endswith(".c") or file_path.endswith(".h"): - try: - with open(file_path, 'r') as f: - file = f.readlines() - except Exception as e: - logging.error(e) - else: - continue - - if 'Copyright' in file[1] and 'SPDX-License-Identifier: Apache-2.0' in file[3]: - try: - license_year = re.search(r'2006-\d{4}', file[1]).group() - true_year = '2006-{}'.format(current_year) - if license_year != true_year: - logging.warning("[{0}]: license year: {} is not true: {}, please update.".format(file_path, - license_year, - true_year)) - - else: - logging.info("[{0}]: license check success.".format(file_path)) - except Exception as e: - logging.error(e) - - else: - logging.error("[{0}]: license check fail.".format(file_path)) - check_result = False - - return check_result - - -@click.group() -@click.pass_context -def cli(ctx): - pass - - -@cli.command() -@click.option( - '--license', - "check_license", - required=False, - type=click.BOOL, - flag_value=True, - help="Enable File license check.", -) -@click.argument( - 'repo', - nargs=1, - type=click.STRING, - default='https://github.com/RT-Thread/rt-thread', -) -@click.argument( - 'branch', - nargs=1, - type=click.STRING, - default='master', -) -def check(check_license, repo, branch): - """ - check files license and format. - """ - init_logger() - # get modified files list - checkout = CheckOut(repo, branch) - file_list = checkout.get_new_file() - if file_list is None: - logging.error("checkout files fail") - sys.exit(1) - - # check modified files format - format_check = FormatCheck(file_list) - format_check_result = format_check.check() - license_check_result = True - if check_license: - license_check = LicenseCheck(file_list) - license_check_result = license_check.check() - - if not format_check_result or not license_check_result: - logging.error("file format check or license check fail.") - sys.exit(1) - logging.info("check success.") - sys.exit(0) - - -if __name__ == '__main__': - cli()