diff --git a/tools/ci/ci_common.py b/tools/ci/ci_common.py new file mode 100644 index 00000000000..21c4797951b --- /dev/null +++ b/tools/ci/ci_common.py @@ -0,0 +1,187 @@ +# +# Copyright (c) 2006-2025, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 +# +# Change Logs: +# Date Author Notes +# 2026-07-08 yan hu add read-only changed-file helpers +# + +"""Shared read-only helpers for the tools/ci scripts. + +Changed-file detection must never modify repository state (HEAD, index or +working tree) or leave temporary files behind. +""" + +import os +import logging +import subprocess +from collections import namedtuple + +# Structured result of a git invocation; ``cmd`` keeps the exact argument list. +GitResult = namedtuple("GitResult", ["returncode", "stdout", "stderr", "cmd"]) + +DEFAULT_TARGET_REF = "origin/master" +TARGET_REF_ENV_VAR = "RTT_CI_TARGET_REF" +DEFAULT_DIFF_FILTER = "ACMR" + + +def run_git(args, cwd=None): + """Run a git command (no shell) and return a GitResult. + + A non-zero git exit code is reported via ``returncode``, not raised. + """ + cmd = ["git"] + list(args) + try: + completed = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=cwd, + ) + except OSError as e: + logging.error("failed to execute {}: {}".format(" ".join(cmd), e)) + return GitResult(returncode=127, stdout="", stderr=str(e), cmd=cmd) + + return GitResult( + returncode=completed.returncode, + stdout=completed.stdout or "", + stderr=completed.stderr or "", + cmd=cmd, + ) + + +def resolve_target_ref(default_ref=DEFAULT_TARGET_REF, env_var=TARGET_REF_ENV_VAR): + """Return the baseline ref to diff against. + + Uses ``RTT_CI_TARGET_REF`` when set (for fork/CI setups), otherwise the + default. Read-only. + """ + ref = os.environ.get(env_var) + if ref: + ref = ref.strip() + if ref: + return ref + return default_ref + + +def normalize_changed_files(output): + """Split output into lines, drop empties and normalise separators to ``/``.""" + files = [] + for line in output.splitlines(): + line = line.strip() + if not line: + continue + files.append(line.replace("\\", "/")) + return files + + +def get_merge_base(target_ref=DEFAULT_TARGET_REF, head_ref="HEAD", cwd=None): + """Return the merge base of the two refs, or ``None`` on failure. Read-only.""" + result = run_git(["merge-base", target_ref, head_ref], cwd=cwd) + if result.returncode != 0: + logging.error( + "git merge-base {} {} failed: {}".format( + target_ref, head_ref, result.stderr.strip() + ) + ) + return None + base = result.stdout.strip() + if not base: + logging.error( + "git merge-base {} {} returned no commit".format(target_ref, head_ref) + ) + return None + return base + + +def get_changed_files_between(base_ref, head_ref="HEAD", + diff_filter=DEFAULT_DIFF_FILTER, cwd=None): + """Files changed between two refs. ``None`` on failure, ``[]`` on empty diff.""" + args = [ + "diff", + "--name-only", + "--diff-filter={}".format(diff_filter), + "--no-renames", + "--full-index", + base_ref, + head_ref, + ] + result = run_git(args, cwd=cwd) + if result.returncode != 0: + logging.error( + "git diff {}..{} failed: {}".format( + base_ref, head_ref, result.stderr.strip() + ) + ) + return None + return normalize_changed_files(result.stdout) + + +def get_changed_files(target_ref=DEFAULT_TARGET_REF, head_ref="HEAD", + diff_filter=DEFAULT_DIFF_FILTER, cwd=None): + """Files changed on ``head_ref`` vs the merge base with ``target_ref``. + + Read-only. ``None`` on git failure, ``[]`` on empty diff. + """ + base = get_merge_base(target_ref, head_ref, cwd=cwd) + if base is None: + return None + return get_changed_files_between(base, head_ref, diff_filter=diff_filter, cwd=cwd) + + +def maybe_fetch_remote(remote_name, remote_url=None, cwd=None): + """Ensure ``remote_name`` points at ``remote_url`` and fetch its refs. + + Adds the remote when missing, or updates its URL when it already exists but + differs. Touches ``.git/config`` and ``FETCH_HEAD`` only, never HEAD, the + index or the working tree. Returns ``True`` on success, ``False`` on any + failure so callers stop instead of diffing against a stale baseline. + """ + existing = run_git(["remote"], cwd=cwd) + if existing.returncode != 0: + logging.error("git remote failed: {}".format(existing.stderr.strip())) + return False + + remotes = normalize_changed_files(existing.stdout) + if remote_name not in remotes: + if not remote_url: + logging.error( + "remote '{}' does not exist and no url was provided".format(remote_name) + ) + return False + added = run_git(["remote", "add", remote_name, remote_url], cwd=cwd) + if added.returncode != 0: + logging.error( + "git remote add {} failed: {}".format(remote_name, added.stderr.strip()) + ) + return False + elif remote_url: + current = run_git(["remote", "get-url", remote_name], cwd=cwd) + if current.returncode != 0: + logging.error( + "git remote get-url {} failed: {}".format( + remote_name, current.stderr.strip() + ) + ) + return False + if current.stdout.strip() != remote_url: + updated = run_git(["remote", "set-url", remote_name, remote_url], cwd=cwd) + if updated.returncode != 0: + logging.error( + "git remote set-url {} failed: {}".format( + remote_name, updated.stderr.strip() + ) + ) + return False + + fetched = run_git(["fetch", remote_name], cwd=cwd) + if fetched.returncode != 0: + logging.error( + "git fetch {} failed: {}".format(remote_name, fetched.stderr.strip()) + ) + return False + + return True diff --git a/tools/ci/compile_bsp_with_drivers.py b/tools/ci/compile_bsp_with_drivers.py index 0ddc7c58e67..a58adfe5c88 100644 --- a/tools/ci/compile_bsp_with_drivers.py +++ b/tools/ci/compile_bsp_with_drivers.py @@ -13,6 +13,8 @@ import os import sys +import ci_common + CONFIG_BSP_USING_X = ["CONFIG_BSP_USING_UART", "CONFIG_BSP_USING_I2C", "CONFIG_BSP_USING_SPI", "CONFIG_BSP_USING_ADC", "CONFIG_BSP_USING_DAC"] def init_logger(): @@ -24,8 +26,13 @@ def init_logger(): ) def diff(): - result = subprocess.run(['git', 'diff', '--name-only', 'HEAD', 'origin/master', '--diff-filter=ACMR', '--no-renames', '--full-index'], stdout = subprocess.PIPE) - file_list = result.stdout.decode().strip().split('\n') + # Read-only diff against the configurable baseline. Returns None on git + # failure (so the caller can fail instead of trusting an unreliable empty + # result); an empty diff returns an empty set. + file_list = ci_common.get_changed_files(ci_common.resolve_target_ref()) + if file_list is None: + logging.error("failed to determine changed files") + return None logging.info(file_list) bsp_paths = set() for file in file_list: @@ -107,6 +114,9 @@ def recompile_bsp(dir): if __name__ == '__main__': init_logger() recompile_bsp_dirs = diff() + if recompile_bsp_dirs is None: + logging.error("failed to determine changed files, aborting") + sys.exit(1) failed = 0 for dir in recompile_bsp_dirs: dot_config_path = dir + "/" + ".config" diff --git a/tools/ci/file_check.py b/tools/ci/file_check.py index 597c8c33b05..b5869ec10b9 100644 --- a/tools/ci/file_check.py +++ b/tools/ci/file_check.py @@ -17,6 +17,8 @@ import logging import datetime +import ci_common + def init_logger(): log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s " @@ -80,37 +82,28 @@ def __exclude_file(self, file_path): return 1 def get_new_file(self): - file_list = list() - try: - os.system('git remote add rtt_repo {}'.format(self.rtt_repo)) - os.system('git fetch rtt_repo') - os.system('git merge rtt_repo/{}'.format(self.rtt_branch)) - os.system('git reset rtt_repo/{} --soft'.format(self.rtt_branch)) - os.system('git status > git.txt') - except Exception as e: - logging.error(e) - return None - try: - with open('git.txt', 'r') as f: - file_lines = f.readlines() - except Exception as e: - logging.error(e) + # Read-only changed-file detection. When a repo/branch pair is given we + # fetch it into a dedicated remote (which only touches .git/config and + # FETCH_HEAD) and diff against it; otherwise we fall back to the + # configurable default baseline. HEAD, the index and the working tree + # are never modified and no temporary files are created. + if self.rtt_repo: + if not ci_common.maybe_fetch_remote("rtt_repo", self.rtt_repo): + logging.error("failed to fetch baseline repo, aborting file check") + return None + target_ref = "rtt_repo/{}".format(self.rtt_branch) + else: + target_ref = ci_common.resolve_target_ref() + + changed_files = ci_common.get_changed_files(target_ref) + if changed_files is None: + logging.error("failed to determine changed files") return None - file_path = '' - for line in file_lines: - if 'new file' in line: - file_path = line.split('new file:')[1].strip() - logging.info('new file -> {}'.format(file_path)) - elif 'deleted' in line: - logging.info('deleted file -> {}'.format(line.split('deleted:')[1].strip())) - elif 'modified' in line: - file_path = line.split('modified:')[1].strip() - logging.info('modified file -> {}'.format(file_path)) - else: - continue - result = self.__exclude_file(file_path) - if result != 0: + file_list = list() + for file_path in changed_files: + logging.info("changed file -> {}".format(file_path)) + if self.__exclude_file(file_path) != 0: file_list.append(file_path) return file_list diff --git a/tools/ci/format_ignore.py b/tools/ci/format_ignore.py index a7f9be2cbfc..e0290812870 100644 --- a/tools/ci/format_ignore.py +++ b/tools/ci/format_ignore.py @@ -11,7 +11,8 @@ import yaml import logging import os -import subprocess + +import ci_common def init_logger(): log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s " @@ -72,13 +73,17 @@ def __exclude_file(self, file_path): return 1 def get_new_file(self): - result = subprocess.run(['git', 'diff', '--name-only', 'HEAD', 'origin/master', '--diff-filter=ACMR', '--no-renames', '--full-index'], stdout = subprocess.PIPE) - file_list = result.stdout.decode().strip().split('\n') + # Read-only diff against the configurable baseline (merge base of the + # target ref and HEAD). Returns None on git failure, [] on empty diff. + changed_files = ci_common.get_changed_files(ci_common.resolve_target_ref()) + if changed_files is None: + logging.error("failed to determine changed files") + return None + new_files = [] - for line in file_list: + for line in changed_files: logging.info("modified file -> {}".format(line)) - result = self.__exclude_file(line) - if result != 0: + if self.__exclude_file(line) != 0: new_files.append(line) - + return new_files \ No newline at end of file