diff --git a/src/borg/archive.py b/src/borg/archive.py index 3299d07399..33b00c6b3b 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -32,7 +32,7 @@ from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError from .helpers import HardLinkManager from .helpers import ChunkIteratorFileWrapper, open_item -from .helpers import Error, IntegrityError, set_ec +from .helpers import Error, IntegrityError, set_ec, sig_int from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns from .helpers import parse_timestamp, archive_ts_now, CompressionSpec from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize @@ -1924,12 +1924,28 @@ def check( rebuild_manifest = True if rebuild_manifest: self.manifest = self.rebuild_manifest() - if find_lost_archives: - self.rebuild_archives_directory() - self.rebuild_archives( - match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, newer=newer, newest=newest - ) + # On Ctrl-C, skip the remaining scans. + if not sig_int: + if find_lost_archives: + self.rebuild_archives_directory() + self.rebuild_archives( + match=match, + first=first, + last=last, + sort_by=sort_by, + older=older, + oldest=oldest, + newer=newer, + newest=newest, + ) + # finish() writes the manifest and a consistent chunk index; run it on Ctrl-C too (#9850). self.finish() + if sig_int: + if self.error_found: + logger.error("Archive consistency check interrupted, problems found so far.") + else: + logger.info("Archive consistency check interrupted, no problems found so far.") + raise Error("Got Ctrl-C / SIGINT.") if self.error_found: logger.error("Archive consistency check complete, problems found.") else: @@ -1977,12 +1993,16 @@ def verify_data(self): logger.info("Starting cryptographic data integrity verification...") chunks_count = len(self.chunks) errors = 0 + verified = 0 # chunks actually verified defect_chunks = [] pi = ProgressIndicatorPercent( total=chunks_count, msg="Verifying data %6.2f%%", step=0.01, msgid="check.verify_data" ) for chunk_id, _ in self.chunks.iteritems(): + if sig_int: + break pi.show() + verified += 1 try: encrypted_data = self.repository.get(chunk_id) except (Repository.ObjectNotFound, IntegrityErrorBase) as err: @@ -2038,11 +2058,20 @@ def verify_data(self): for defect_chunk in defect_chunks: logger.debug("chunk %s is defect.", bin_to_hex(defect_chunk)) log = logger.error if errors else logger.info - log( - "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.", - chunks_count, - errors, - ) + if sig_int: + log( + "Interrupted cryptographic data integrity verification, " + "verified %d of %d chunks with %d integrity errors.", + verified, + chunks_count, + errors, + ) + else: + log( + "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.", + verified, + errors, + ) def rebuild_manifest(self): """Rebuild the manifest object.""" @@ -2078,6 +2107,8 @@ def valid_archive(obj): msgid="check.rebuild_archives_directory", ) for chunk_id, _ in self.chunks.iteritems(): + if sig_int: + break pi.show() cdata = self.repository.get(chunk_id, read_data=False) # only get metadata try: @@ -2123,7 +2154,10 @@ def valid_archive(obj): logger.warning(f"Would create archives directory entry for {name} {archive_id_hex}.") pi.finish() - logger.info("Rebuilding missing archives directory entries completed.") + if sig_int: + logger.info("Rebuilding missing archives directory entries interrupted.") + else: + logger.info("Rebuilding missing archives directory entries completed.") def rebuild_archives( self, first=0, last=0, sort_by="", match=None, older=None, newer=None, oldest=None, newest=None @@ -2271,6 +2305,9 @@ def valid_item(obj): total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_archives" ) for i, info in enumerate(archive_infos): + if sig_int: + # Break only between archives, as --repair rewrites each archive as a whole. + break pi.show(i) archive_id, archive_id_hex = info.id, bin_to_hex(info.id) try: diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 682f3a9279..6a3092e3f2 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -3,8 +3,8 @@ from ._common import with_repository, Highlander from ..archive import ArchiveChecker from ..constants import * # NOQA -from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError -from ..helpers import yes, ArchiveFormatter +from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, Error, IntegrityError +from ..helpers import yes, ArchiveFormatter, sig_int from ..helpers.argparsing import ArgumentParser from ..logger import create_logger @@ -66,6 +66,8 @@ def do_check(self, args, repository): if not args.archives_only: if not repository.check(repair=args.repair, max_duration=args.max_duration): set_ec(EXIT_WARNING) + if sig_int: # repository check interrupted; skip the archive check + raise Error("Got Ctrl-C / SIGINT.") if not args.repo_only and not archive_checker.check( repository, verify_data=args.verify_data, @@ -160,6 +162,19 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser): formatted by giving a custom format using ``--format`` (see the ``borg repo-list`` description for more details about the format string). + If the ``borg check`` process receives a SIGINT signal (Ctrl-C), it stops at the + next safe boundary (a pack boundary during the repository check, an archive boundary + during the archive check), leaving the repository and its chunk index in a consistent + state. A partial repository check (``--max-duration``) saves its progress so a later + partial check resumes where it stopped; a full check restarts from the beginning. + With ``--repair``, an interrupted archive check may leave some archives already + repaired and others not yet processed, so run ``borg check --repair`` again to finish. + + During a ``--repair`` run, the archive check first rebuilds the chunk index from the + packs, and, if the key must be recovered, scans chunks for it. These phases do not yet + respond to SIGINT, so on a large repository a Ctrl-C during them may appear to have no + effect until they finish. + About repair mode +++++++++++++++++ diff --git a/src/borg/repository.py b/src/borg/repository.py index 254c094589..7b13e770cc 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -877,6 +877,10 @@ def store_list(namespace): pack_infos = store_list("packs") pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs") for info in pack_infos: + if sig_int: # on Ctrl-C, persist checked packs, then stop + logger.info(f"Interrupted repository check, {len(tracker)} packs checked so far.") + tracker.save() + break self._lock_refresh() pack_pi.show(increase=1) # advance for skipped packs too, so the bar tracks packs/, not work done pack_id = hex_to_bin(info.name) @@ -913,11 +917,15 @@ def store_list(namespace): f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)." ) if objs_errors == 0: - logger.info(f"Finished {mode} repository check, no problems found.") + if sig_int: + logger.info(f"Interrupted {mode} repository check, no problems found so far.") + else: + logger.info(f"Finished {mode} repository check, no problems found.") elif repair: logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).") else: logger.error(f"Finished {mode} repository check, errors found.") + # True means the checked objects were clean; on Ctrl-C that covers only the packs seen so far. return objs_errors == 0 or repair def list(self, limit=None, marker=None): diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index ef42ad776b..434079bfe1 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -6,11 +6,11 @@ import pytest -from ...archive import ChunkBuffer +from ...archive import ArchiveChecker, ChunkBuffer from ...constants import * # NOQA -from ...helpers import bin_to_hex, msgpack, CommandError, IntegrityError -from ...manifest import Manifest -from ...repository import Repository +from ...helpers import bin_to_hex, msgpack, CommandError, Error, IntegrityError, sig_int +from ...manifest import Archives, Manifest +from ...repository import PackTracker, Repository from ..repository_test import fchunk, corrupt_chunk_on_disk from . import ( cmd, @@ -74,6 +74,96 @@ def test_check_usage(archivers, request): assert "archive2" in output +def test_check_soft_interrupt(archivers, request, monkeypatch): + """A mid-run Ctrl-C stops both check phases at a safe boundary (#7893): the repository check persists + its checked packs for a later partial check to resume, and the archive check runs finish() and then + raises. The check is read-only, so a normal check still passes afterwards.""" + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) # produces many packs + + # repository check: interrupt after the first pack. + with Repository(archiver.repository_path, exclusive=True) as repository: + orig_hash = repository.store.hash + pack_checks = [] + + def hash_then_interrupt(key): + result = orig_hash(key) + if key.startswith("packs/"): # count pack checks, not the index files hashed first + pack_checks.append(key) + if len(pack_checks) == 1: # one Ctrl-C after the first pack is checked + sig_int._sig_int_triggered = True + return result + + monkeypatch.setattr(repository.store, "hash", hash_then_interrupt) + try: + repository.check() + finally: + sig_int._sig_int_triggered = False + assert len(PackTracker.load(repository.store)) == 1 # the pack checked before the break persisted + + # a partial check resumes the saved cycle. + output = cmd(archiver, "check", "-v", "--repository-only", "--max-duration=600", exit_code=0) + assert "Continuing check cycle" in output + + # archive check: interrupt verify_data after 3 chunks. + with Repository(archiver.repository_path, exclusive=True) as repository: + orig_get = repository.get + get_calls = 0 + interrupted_after = None + + def get_then_interrupt(*args, **kwargs): + nonlocal get_calls, interrupted_after + get_calls += 1 + if get_calls == 3: # trip mid-loop, after 3 chunks + sig_int._sig_int_triggered = True + interrupted_after = get_calls + return orig_get(*args, **kwargs) + + monkeypatch.setattr(repository, "get", get_then_interrupt) + try: + with pytest.raises(Error, match="Got Ctrl-C"): + ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}") + finally: + sig_int._sig_int_triggered = False + assert interrupted_after == 3 # the loop stopped after verifying 3 chunks + + # nothing changed, so a normal check passes. + cmd(archiver, "check", exit_code=0) + + +def test_check_repair_soft_interrupt(archivers, request, monkeypatch): + """A Ctrl-C after the first archive of a --repair archive check stops at the archive boundary, runs + finish() (dropping the chunk index, writing the manifest), then raises. No archive is lost, and a + second --repair finishes the job so a following check reports the repository consistent.""" + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) # two archives + + orig_create = Archives.create + + def create_then_interrupt(self, *args, **kwargs): + orig_create(self, *args, **kwargs) + sig_int._sig_int_triggered = True # one Ctrl-C after the first archive was rebuilt + + monkeypatch.setattr(Archives, "create", create_then_interrupt) + try: + with Repository(archiver.repository_path, exclusive=True) as repository: + with pytest.raises(Error, match="Got Ctrl-C"): + ArchiveChecker().check(repository, repair=True, sort_by="ts", format="{archive} {time} {id}") + finally: + sig_int._sig_int_triggered = False # reset the global flag for the following tests + # restore the real method; monkeypatch.undo() would also drop the autouse env (BORG_TESTONLY_WEAKEN_KDF). + monkeypatch.setattr(Archives, "create", orig_create) + + # both archives survive the interrupt between archives. + output = cmd(archiver, "repo-list", exit_code=0) + assert "archive1" in output + assert "archive2" in output + + # a second --repair finishes the job; a plain check then finds no problems. + cmd(archiver, "check", "--repair", exit_code=0) + cmd(archiver, "check", exit_code=0) + + def test_date_matching(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver)