Stable 24.8.14 - Mark known sanitizer-timeout tests as broken and bump job timeout

docker/test/util/Dockerfile

@@ -20,7 +20,7 @@ RUN apt-get update \
         lsb-release \
         wget \
         --yes --no-install-recommends --verbose-versions \
-    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
+    && export LLVM_PUBKEY_HASH="5ffc7c9a9299ce774f81cada703e23ebba5bdfb0345b6c3b667b3ead7aa21c75ef62ccd74f7a8f2aa0cbe158d3068bbe" \
     && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
     && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
     && apt-key add /tmp/llvm-snapshot.gpg.key \

docker/test/util/process_functional_tests_result.py

@@ -21,7 +21,27 @@
 RETRIES_SIGN = "Some tests were restarted"
 
 
+def _is_known_failing_test(
+    test_name, known_failing_tests, check_name, full_log_content
+):
+    if test_name not in known_failing_tests:
+        return False
+
+    entry = known_failing_tests[test_name]
+    check_types = entry.get("check_types", [])
+
+    if check_types and all(ct not in check_name for ct in check_types):
+        return False
+
+    message = entry.get("message")
+    if not message:
+        return True
+
+    return message in full_log_content
+
+
 def process_test_log(log_path, broken_tests, known_failing_tests):
+    check_name = os.getenv("CHECK_NAME", "")
     total = 0
     skipped = 0
     unknown = 0
@@ -34,7 +54,8 @@ def process_test_log(log_path, broken_tests, known_failing_tests):
     test_results = []
     test_end = True
     with open(log_path, "r") as test_file:
-        for line in test_file:
+        full_log_content = test_file.read()
+        for line in full_log_content.splitlines(keepends=True):
             original_line = line
             line = line.strip()
 
@@ -72,7 +93,9 @@ def process_test_log(log_path, broken_tests, known_failing_tests):
                         failed += 1
                         test_results.append((test_name, "Timeout", test_time, []))
                 elif FAIL_SIGN in line:
-                    if test_name in broken_tests or test_name in known_failing_tests:
+                    if test_name in broken_tests or _is_known_failing_test(
+                        test_name, known_failing_tests, check_name, full_log_content
+                    ):
                         success += 1
                         test_results.append((test_name, "BROKEN", test_time, []))
                     else:
@@ -232,12 +255,12 @@ def write_results(results_file, status_file, results, status):
         with open(args.broken_tests) as f:
             broken_tests = f.read().splitlines()
 
-    known_failing_tests = list()
+    known_failing_tests = {}
     if os.path.exists(args.broken_tests_json):
         logging.info(f"File {args.broken_tests_json} with broken tests found")
 
         with open(args.broken_tests_json) as f:
-            known_failing_tests = list(json.load(f).keys())
+            known_failing_tests = json.load(f)
 
     if broken_tests:
         logging.info(f"Broken tests in the list: {len(broken_tests)}")

tests/broken_tests.json

@@ -25,5 +25,29 @@
     },
     "02490_benchmark_max_consecutive_errors": {
         "reason": "INVESTIGATE: Unstable on msan"
+    },
+    "01782_field_oom": {
+        "reason": "KNOWN: timeout with sanitizer",
+        "check_types": ["tsan", "asan", "msan", "ubsan"]
+    },
+    "02389_analyzer_nested_lambda": {
+        "reason": "KNOWN: timeout with sanitizer",
+        "check_types": ["tsan", "asan", "msan", "ubsan"]
+    },
+    "00312_position_case_insensitive_utf8": {
+        "reason": "KNOWN: timeout with sanitizer",
+        "check_types": ["tsan", "asan", "msan", "ubsan"]
+    },
+    "01037_polygon_dicts_correctness_fast": {
+        "reason": "KNOWN: timeout with sanitizer",
+        "check_types": ["tsan", "asan", "msan", "ubsan"]
+    },
+    "01655_plan_optimizations_optimize_read_in_window_order_long": {
+        "reason": "KNOWN: timeout with sanitizer",
+        "check_types": ["tsan", "asan", "msan", "ubsan"]
+    },
+    "02098_hashed_array_dictionary_simple_key": {
+        "reason": "KNOWN: timeout with sanitizer",
+        "check_types": ["tsan", "asan", "msan", "ubsan"]
     }
 }

tests/ci/ci_config.py

@@ -321,19 +321,29 @@ class CI:
             timeout=3600,
         ),
         JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
+            required_builds=[BuildNames.PACKAGE_ASAN],
+            num_batches=2,
+            timeout=3 * 3600,  # the job timed out with default value (7200)
         ),
         JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=4
+            required_builds=[BuildNames.PACKAGE_TSAN],
+            num_batches=4,
+            timeout=3 * 3600,  # the job timed out with default value (7200)
         ),
         JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=4
+            required_builds=[BuildNames.PACKAGE_MSAN],
+            num_batches=4,
+            timeout=3 * 3600,  # the job timed out with default value (7200)
         ),
         JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2
+            required_builds=[BuildNames.PACKAGE_UBSAN],
+            num_batches=2,
+            timeout=3 * 3600,  # the job timed out with default value (7200)
         ),
         JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
+            required_builds=[BuildNames.PACKAGE_DEBUG], 
+            num_batches=2,
+            timeout=3 * 3600,  # the job timed out with default value (7200)
         ),
         JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],

tests/ci/functional_test_check.py

@@ -108,6 +108,7 @@ def get_run_command(
     envs = [
         # a static link, don't use S3_URL or S3_DOWNLOAD
         '-e S3_URL="https://s3.amazonaws.com/clickhouse-datasets"',
+        f'-e CHECK_NAME="{check_name}"',
     ]
 
     if flaky_check:

tests/ci/integration_tests_runner.py

@@ -17,7 +17,7 @@
 import zlib  # for crc32
 from collections import defaultdict
 from itertools import chain
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from ci_utils import kill_ci_runner
 from env_helper import IS_CI
@@ -521,6 +521,83 @@ def _update_counters(main_counters, current_counters):
             for test in current_counters[state]:
                 main_counters[state].append(test)
 
+    def _handle_broken_tests(
+        self,
+        counters: Dict[str, List[str]],
+        known_broken_tests: Dict[str, Dict[str, str]],
+        log_paths: Union[Dict[str, List[str]], List[str]],
+    ) -> None:
+
+        def get_log_paths(test_name):
+            """Could be a list of logs for all tests or a dict with test name as a key"""
+            if isinstance(log_paths, dict):
+                return log_paths[test_name]
+            return log_paths
+
+        check_name = os.getenv("CHECK_NAME", "")
+
+        broken_tests_log = os.path.join(self.result_path, "broken_tests_handler.log")
+
+        with open(broken_tests_log, "a") as log_file:
+            log_file.write(f"{len(known_broken_tests)} Known broken tests\n")
+            for status, tests in counters.items():
+                log_file.write(f"Total tests in {status} state: {len(tests)}\n")
+
+            for fail_status in ("ERROR", "FAILED"):
+                for failed_test in counters[fail_status].copy():
+                    log_file.write(
+                        f"Checking test {failed_test} (status: {fail_status})\n"
+                    )
+                    if failed_test not in known_broken_tests.keys():
+                        log_file.write(
+                            f"Test {failed_test} is not in known broken tests\n"
+                        )
+                    else:
+                        check_types = known_broken_tests[failed_test].get(
+                            "check_types", []
+                        )
+
+                        if check_types and all(
+                            check_type not in check_name for check_type in check_types
+                        ):
+                            log_file.write(
+                                f"Test {failed_test} is not known to fail in check {check_name}\n"
+                            )
+                            continue
+
+                        fail_message = known_broken_tests[failed_test].get("message")
+
+                        if not fail_message:
+                            log_file.write(
+                                "No fail message specified, marking as broken\n"
+                            )
+                            mark_as_broken = True
+                        else:
+                            log_file.write(
+                                f"Looking for fail message: {fail_message}\n"
+                            )
+                            mark_as_broken = False
+                            for log_path in get_log_paths(failed_test):
+                                if log_path.endswith(".log"):
+                                    log_file.write(f"Checking log file: {log_path}\n")
+                                    with open(log_path) as test_log:
+                                        if fail_message in test_log.read():
+                                            log_file.write(
+                                                "Found fail message in logs\n"
+                                            )
+                                            mark_as_broken = True
+                                            break
+
+                        if mark_as_broken:
+                            log_file.write(f"Moving test to BROKEN state\n")
+                            counters[fail_status].remove(failed_test)
+                            counters["BROKEN"].append(failed_test)
+                        else:
+                            log_file.write("Test not marked as broken\n")
+
+            for status, tests in counters.items():
+                log_file.write(f"Total tests in {status} state: {len(tests)}\n")
+
     def _get_runner_image_cmd(self, repo_path):
         image_cmd = ""
         if self._can_run_with(
@@ -960,24 +1037,7 @@ def run_impl(self, repo_path, build_path):
                 repo_path, group, tests, MAX_RETRY, NUM_WORKERS, 0
             )
 
-            for fail_status in ("ERROR", "FAILED"):
-                for failed_test in group_counters[fail_status]:
-                    if failed_test in known_broken_tests.keys():
-                        fail_message = known_broken_tests[failed_test].get("message")
-                        if not fail_message:
-                            mark_as_broken = True
-                        else:
-                            mark_as_broken = False
-                            for log_path in log_paths:
-                                if log_path.endswith(".log"):
-                                    with open(log_path) as log_file:
-                                        if fail_message in log_file.read():
-                                            mark_as_broken = True
-                                            break
-
-                        if mark_as_broken:
-                            group_counters[fail_status].remove(failed_test)
-                            group_counters["BROKEN"].append(failed_test)
+            self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
 
             total_tests = 0
             for counter, value in group_counters.items():