diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 1c0bf41ff789..795666a9907a 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -78,6 +78,7 @@ if (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME "-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}" "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" "-DARCH_AMD64=${HOST_ARCH_AMD64}" "-DARCH_AARCH64=${HOST_ARCH_AARCH64}" "-DARCH_PPC64LE=${HOST_ARCH_PPC64LE}" @@ -127,6 +128,7 @@ if (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME "-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}" "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" "-DABSL_ROOT_DIR=${abseil_source_dir}" "-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${zlib_config_source_dir}" "-DgRPC_INSTALL=0" diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt index a9e4a3df698b..f3388606f005 100644 --- a/contrib/openssl-cmake/CMakeLists.txt +++ b/contrib/openssl-cmake/CMakeLists.txt @@ -1,3 +1,5 @@ +cmake_minimum_required(VERSION 3.5) + # Actually, so many 3rd party libraries + unit tests need SSL that we cannot disable it # without breaking the build ... set(ENABLE_SSL 1 CACHE INTERNAL "") diff --git a/docker/packager/binary-builder/build.sh b/docker/packager/binary-builder/build.sh index ba9311c6880a..c09f3eec4680 100755 --- a/docker/packager/binary-builder/build.sh +++ b/docker/packager/binary-builder/build.sh @@ -68,6 +68,13 @@ fi read -ra CMAKE_FLAGS_ARRAY <<< "${CMAKE_FLAGS:-}" env +# Newer toolchain environments may have rustup metadata/toolchain discovery issues. +# Provide an explicit rustc path to CMake's FindRust and avoid rustup toolchain probing, +# which can silently disable Rust features (e.g. PRQL) when detection fails. +if [ -x /rust/cargo/bin/rustc ]; then + CMAKE_FLAGS_ARRAY+=(-DRust_COMPILER=/rust/cargo/bin/rustc -DRust_RESOLVE_RUSTUP_TOOLCHAINS=OFF) +fi + if [ "$BUILD_MUSL_KEEPER" == "1" ] then # build keeper with musl separately diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index c4fae94d4abe..3d8285694a9b 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -51,11 +51,12 @@ RUN mkdir /tmp/ccache \ -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \ -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \ && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \ - && gpg --verify ccache-4.6.1.tar.xz.asc \ + && gpg --verify ccache-$CCACHE_VERSION.tar.xz.asc \ && tar xf ccache-$CCACHE_VERSION.tar.xz \ && cd /tmp/ccache/ccache-$CCACHE_VERSION \ && cmake -DCMAKE_INSTALL_PREFIX=/usr \ -DCMAKE_BUILD_TYPE=None \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DZSTD_FROM_INTERNET=ON \ -DREDIS_STORAGE_BACKEND=OFF \ -Wno-dev \ diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 6601acc708fe..81e140be8ffe 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -20,7 +20,7 @@ RUN apt-get update \ lsb-release \ wget \ --yes --no-install-recommends --verbose-versions \ - && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && export LLVM_PUBKEY_HASH="5ffc7c9a9299ce774f81cada703e23ebba5bdfb0345b6c3b667b3ead7aa21c75ef62ccd74f7a8f2aa0cbe158d3068bbe" \ && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ && apt-key add /tmp/llvm-snapshot.gpg.key \ diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index 995d11cf818e..42ac087eb36a 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -21,7 +21,27 @@ RETRIES_SIGN = "Some tests were restarted" +def _is_known_failing_test( + test_name, known_failing_tests, check_name, full_log_content +): + if test_name not in known_failing_tests: + return False + + entry = known_failing_tests[test_name] + check_types = entry.get("check_types", []) + + if check_types and all(ct not in check_name for ct in check_types): + return False + + message = entry.get("message") + if not message: + return True + + return message in full_log_content + + def process_test_log(log_path, broken_tests, known_failing_tests): + check_name = os.getenv("CHECK_NAME", "") total = 0 skipped = 0 unknown = 0 @@ -34,7 +54,8 @@ def process_test_log(log_path, broken_tests, known_failing_tests): test_results = [] test_end = True with open(log_path, "r") as test_file: - for line in test_file: + full_log_content = test_file.read() + for line in full_log_content.splitlines(keepends=True): original_line = line line = line.strip() @@ -72,7 +93,9 @@ def process_test_log(log_path, broken_tests, known_failing_tests): failed += 1 test_results.append((test_name, "Timeout", test_time, [])) elif FAIL_SIGN in line: - if test_name in broken_tests or test_name in known_failing_tests: + if test_name in broken_tests or _is_known_failing_test( + test_name, known_failing_tests, check_name, full_log_content + ): success += 1 test_results.append((test_name, "BROKEN", test_time, [])) else: @@ -232,12 +255,12 @@ def write_results(results_file, status_file, results, status): with open(args.broken_tests) as f: broken_tests = f.read().splitlines() - known_failing_tests = list() + known_failing_tests = {} if os.path.exists(args.broken_tests_json): logging.info(f"File {args.broken_tests_json} with broken tests found") with open(args.broken_tests_json) as f: - known_failing_tests = list(json.load(f).keys()) + known_failing_tests = json.load(f) if broken_tests: logging.info(f"Broken tests in the list: {len(broken_tests)}") diff --git a/tests/broken_tests.json b/tests/broken_tests.json index 358e0967f14a..4093b27853da 100644 --- a/tests/broken_tests.json +++ b/tests/broken_tests.json @@ -25,5 +25,29 @@ }, "02490_benchmark_max_consecutive_errors": { "reason": "INVESTIGATE: Unstable on msan" + }, + "01782_field_oom": { + "reason": "KNOWN: timeout with sanitizer", + "check_types": ["tsan", "asan", "msan", "ubsan"] + }, + "02389_analyzer_nested_lambda": { + "reason": "KNOWN: timeout with sanitizer", + "check_types": ["tsan", "asan", "msan", "ubsan"] + }, + "00312_position_case_insensitive_utf8": { + "reason": "KNOWN: timeout with sanitizer", + "check_types": ["tsan", "asan", "msan", "ubsan"] + }, + "01037_polygon_dicts_correctness_fast": { + "reason": "KNOWN: timeout with sanitizer", + "check_types": ["tsan", "asan", "msan", "ubsan"] + }, + "01655_plan_optimizations_optimize_read_in_window_order_long": { + "reason": "KNOWN: timeout with sanitizer", + "check_types": ["tsan", "asan", "msan", "ubsan"] + }, + "02098_hashed_array_dictionary_simple_key": { + "reason": "KNOWN: timeout with sanitizer", + "check_types": ["tsan", "asan", "msan", "ubsan"] } } \ No newline at end of file diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 483c008a51ba..30e19982e46b 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -321,19 +321,29 @@ class CI: timeout=3600, ), JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2 + required_builds=[BuildNames.PACKAGE_ASAN], + num_batches=2, + timeout=3 * 3600, # the job timed out with default value (7200) ), JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_TSAN], num_batches=4 + required_builds=[BuildNames.PACKAGE_TSAN], + num_batches=4, + timeout=3 * 3600, # the job timed out with default value (7200) ), JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_MSAN], num_batches=4 + required_builds=[BuildNames.PACKAGE_MSAN], + num_batches=4, + timeout=3 * 3600, # the job timed out with default value (7200) ), JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2 + required_builds=[BuildNames.PACKAGE_UBSAN], + num_batches=2, + timeout=3 * 3600, # the job timed out with default value (7200) ), JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2 + required_builds=[BuildNames.PACKAGE_DEBUG], + num_batches=2, + timeout=3 * 3600, # the job timed out with default value (7200) ), JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index f2c9ab46e697..653b2374eecb 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -108,6 +108,7 @@ def get_run_command( envs = [ # a static link, don't use S3_URL or S3_DOWNLOAD '-e S3_URL="https://s3.amazonaws.com/clickhouse-datasets"', + f'-e CHECK_NAME="{check_name}"', ] if flaky_check: diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index 4abf966ce52e..bb41cee1a909 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -17,7 +17,7 @@ import zlib # for crc32 from collections import defaultdict from itertools import chain -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional, Union from ci_utils import kill_ci_runner from env_helper import IS_CI @@ -521,6 +521,83 @@ def _update_counters(main_counters, current_counters): for test in current_counters[state]: main_counters[state].append(test) + def _handle_broken_tests( + self, + counters: Dict[str, List[str]], + known_broken_tests: Dict[str, Dict[str, str]], + log_paths: Union[Dict[str, List[str]], List[str]], + ) -> None: + + def get_log_paths(test_name): + """Could be a list of logs for all tests or a dict with test name as a key""" + if isinstance(log_paths, dict): + return log_paths[test_name] + return log_paths + + check_name = os.getenv("CHECK_NAME", "") + + broken_tests_log = os.path.join(self.result_path, "broken_tests_handler.log") + + with open(broken_tests_log, "a") as log_file: + log_file.write(f"{len(known_broken_tests)} Known broken tests\n") + for status, tests in counters.items(): + log_file.write(f"Total tests in {status} state: {len(tests)}\n") + + for fail_status in ("ERROR", "FAILED"): + for failed_test in counters[fail_status].copy(): + log_file.write( + f"Checking test {failed_test} (status: {fail_status})\n" + ) + if failed_test not in known_broken_tests.keys(): + log_file.write( + f"Test {failed_test} is not in known broken tests\n" + ) + else: + check_types = known_broken_tests[failed_test].get( + "check_types", [] + ) + + if check_types and all( + check_type not in check_name for check_type in check_types + ): + log_file.write( + f"Test {failed_test} is not known to fail in check {check_name}\n" + ) + continue + + fail_message = known_broken_tests[failed_test].get("message") + + if not fail_message: + log_file.write( + "No fail message specified, marking as broken\n" + ) + mark_as_broken = True + else: + log_file.write( + f"Looking for fail message: {fail_message}\n" + ) + mark_as_broken = False + for log_path in get_log_paths(failed_test): + if log_path.endswith(".log"): + log_file.write(f"Checking log file: {log_path}\n") + with open(log_path) as test_log: + if fail_message in test_log.read(): + log_file.write( + "Found fail message in logs\n" + ) + mark_as_broken = True + break + + if mark_as_broken: + log_file.write(f"Moving test to BROKEN state\n") + counters[fail_status].remove(failed_test) + counters["BROKEN"].append(failed_test) + else: + log_file.write("Test not marked as broken\n") + + for status, tests in counters.items(): + log_file.write(f"Total tests in {status} state: {len(tests)}\n") + def _get_runner_image_cmd(self, repo_path): image_cmd = "" if self._can_run_with( @@ -960,24 +1037,7 @@ def run_impl(self, repo_path, build_path): repo_path, group, tests, MAX_RETRY, NUM_WORKERS, 0 ) - for fail_status in ("ERROR", "FAILED"): - for failed_test in group_counters[fail_status]: - if failed_test in known_broken_tests.keys(): - fail_message = known_broken_tests[failed_test].get("message") - if not fail_message: - mark_as_broken = True - else: - mark_as_broken = False - for log_path in log_paths: - if log_path.endswith(".log"): - with open(log_path) as log_file: - if fail_message in log_file.read(): - mark_as_broken = True - break - - if mark_as_broken: - group_counters[fail_status].remove(failed_test) - group_counters["BROKEN"].append(failed_test) + self._handle_broken_tests(group_counters, known_broken_tests, log_paths) total_tests = 0 for counter, value in group_counters.items():