From 8a5c6532996d6be6940a59fc886ece2ec9042941 Mon Sep 17 00:00:00 2001 From: daidai Date: Thu, 6 Aug 2026 18:07:00 +0800 Subject: [PATCH 1/2] [fix](build) Keep Arrow 17 and 24 in shared thirdparty ### What problem does this PR solve? Issue Number: None Related PR: #66221 Problem Summary: master and branch-4.1 consume the same prebuilt thirdparty prefix. Upgrading the unversioned Arrow/Paimon stack to Arrow 24 makes unchanged branch-4.1 compile against incompatible headers and libraries. Keep Arrow 17/Paimon 17 in the legacy unversioned prefix, install Arrow 24/Paimon 24 in a versioned prefix selected by master, build and validate both source closures independently, and recover only stale stacks. ### Release note None ### Check List (For Author) - Test: - Manual test: built Arrow/Paimon 17 and Arrow/Paimon 24 thirdparty stacks - Manual test: ran thirdparty/test/arrow-paimon-lifecycle-test.sh - Manual test: ran build-support/check-format.sh - Partial build check: configured ASAN BE and compiled 13,378 of 14,456 targets without errors before stopping at the user's request - Behavior changed: Yes. The shared thirdparty prefix keeps Arrow/Paimon 17 at the legacy root and installs master's Arrow/Paimon 24 under a versioned directory. - Does this need documentation: No --- .github/workflows/be-ut-mac.yml | 13 +- be/CMakeLists.txt | 35 +- be/cmake/thirdparty.cmake | 59 ++-- build.sh | 12 +- thirdparty/arrow-paimon-vars.sh | 317 +++++++++++++++++- thirdparty/build-thirdparty.sh | 121 +++++-- thirdparty/download-thirdparty.sh | 67 +++- thirdparty/paimon-cpp-cache.cmake | 27 +- ...-17.0.0-force-write-int96-timestamps.patch | 98 ++++++ .../patches/apache-arrow-17.0.0-lzo.patch | 84 +++++ .../patches/apache-arrow-17.0.0-paimon.patch | 224 +++++++++++++ .../test/arrow-paimon-lifecycle-test.sh | 168 +++++++++- thirdparty/vars.sh | 3 + 13 files changed, 1112 insertions(+), 116 deletions(-) create mode 100644 thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch create mode 100644 thirdparty/patches/apache-arrow-17.0.0-lzo.patch create mode 100644 thirdparty/patches/apache-arrow-17.0.0-paimon.patch diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 47145326a92343..ba15a5e5f809c7 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -129,24 +129,27 @@ jobs: fi tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz - # Rebuild the Arrow/Paimon stack when its inputs change or the shared - # prebuilt predates the selected Arrow version/component closure. + # Rebuild either Arrow/Paimon stack when its inputs change or the + # shared prebuilt predates the dual-version component closure. # The artifact check also covers scheduled and later BE-only builds, # where Paths Filter is skipped or reports no Arrow/Paimon changes. # shellcheck source=thirdparty/arrow-paimon-vars.sh . ./arrow-paimon-vars.sh + select_arrow_paimon_rebuild_packages installed arrow_paimon_prebuilt_is_valid=false - if arrow_paimon_prebuilt_valid installed; then + if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]]; then arrow_paimon_prebuilt_is_valid=true fi if [[ "${{ steps.filter.outputs.arrow_paimon_changes }}" == "true" || "${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then + if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]]; then + ARROW_PAIMON_REBUILD_PACKAGES=("${ARROW_PAIMON_SHARED_BUILD_PACKAGES[@]}") + fi curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ -o doris-thirdparty-source.tgz tar -zxvf doris-thirdparty-source.tgz - ./download-thirdparty.sh arrow paimon_cpp xsimd brotli export MACOSX_DEPLOYMENT_TARGET=12.0 - ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp + ./build-thirdparty.sh -j "$(nproc)" "${ARROW_PAIMON_REBUILD_PACKAGES[@]}" fi popd diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index aed51c2fb2e644..a012dba11843e2 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -156,7 +156,8 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) -set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") +set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH "Selected Arrow install prefix") +set(PAIMON_HOME "" CACHE PATH "Selected Paimon install prefix") option(ENABLE_TDE "Enable TDE feature module" OFF) set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src") option(ENABLE_TLS "Enable TLS feature module" OFF) @@ -178,8 +179,24 @@ endif() if (DEFINED ENV{ENABLE_PAIMON_CPP}) set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE) endif() -if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) +if (DEFINED ENV{ARROW_HOME}) + set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "" FORCE) +endif() +if (DEFINED ENV{PAIMON_HOME}) set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) +elseif (NOT PAIMON_HOME) + set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "" FORCE) +endif() + +if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h") + message(FATAL_ERROR "Selected Arrow installation is incomplete: ${ARROW_HOME}") +endif() +message(STATUS "ARROW_HOME is ${ARROW_HOME}") +if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon") + message(FATAL_ERROR "Selected Paimon installation is incomplete: ${PAIMON_HOME}") +endif() +if (ENABLE_PAIMON_CPP) + message(STATUS "PAIMON_HOME is ${PAIMON_HOME}") endif() set(CMAKE_SKIP_RPATH TRUE) @@ -520,6 +537,12 @@ include_directories( ${SRC_DIR}/ ) +include_directories( + BEFORE SYSTEM + ${ARROW_HOME}/include + ${PAIMON_HOME}/include +) + include_directories( SYSTEM ${COMMON_SRC_DIR} @@ -665,10 +688,10 @@ endif() set(PAIMON_FACTORY_REGISTRY_LIBS) if (ENABLE_PAIMON_CPP) - # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via - # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). - # No dual-stack selection needed — single Arrow for everything. + # The Arrow selected for this BE build includes COMPUTE/DATASET/ACERO/FILESYSTEM, + # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY + # via thirdparty.cmake. Paimon uses this same selected Arrow (no paimon_deps); + # the legacy Arrow 17 stack remains installed only for branch-4.1 consumers. # paimon_parquet_file_format depends on Arrow Dataset symbols. # Force-link it with --whole-archive so its factory registration runs. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index f3bd9867ee0fdd..1003dda92890df 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -25,10 +25,11 @@ set(COMMON_THIRDPARTY) # if arg exist noadd, not append to COMMON_THIRDPARTY variable # if arg exist libname, use libname to find library # if arg exist wholelibpath, use wholelibpath to find library +# if arg exist root, resolve the library below that install prefix function(add_thirdparty) cmake_parse_arguments(DORIS_THIRDPARTY "NOTADD;LIB64" - "LIBNAME;WHOLELIBPATH" + "LIBNAME;WHOLELIBPATH;ROOT" "" ${ARGN}) @@ -39,14 +40,20 @@ function(add_thirdparty) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME} PARENT_SCOPE) endif() + if (DORIS_THIRDPARTY_ROOT) + set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT}) + else() + set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR}) + endif() + if (DORIS_THIRDPARTY_LIB64) - set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a) + set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a) elseif (DORIS_THIRDPARTY_LIBNAME) - set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME}) + set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME}) elseif (DORIS_THIRDPARTY_WHOLELIBPATH) set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH}) else() - set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a) + set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a) endif() set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH}) endfunction() @@ -100,18 +107,18 @@ add_thirdparty(cares LIB64) add_thirdparty(address_sorting LIB64) add_thirdparty(z LIB64) -add_thirdparty(brotlicommon LIB64) -add_thirdparty(brotlidec LIB64) -add_thirdparty(brotlienc LIB64) +add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME}) add_thirdparty(zstd LIB64) -add_thirdparty(arrow LIB64) -add_thirdparty(arrow_compute LIB64) -add_thirdparty(arrow_flight LIB64) -add_thirdparty(arrow_flight_sql LIB64) -add_thirdparty(arrow_dataset LIB64) -add_thirdparty(arrow_acero LIB64) +add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME}) add_thirdparty(adbc_driver_manager LIB64) -add_thirdparty(parquet LIB64) +add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME}) # liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it # so the final linker resolves C math symbols from the system library first. add_thirdparty(lance_c LIB64 NOTADD) @@ -191,16 +198,16 @@ add_thirdparty(icudata LIB64) add_thirdparty(pugixml LIB64) if (ENABLE_PAIMON_CPP) - add_thirdparty(paimon LIB64) - add_thirdparty(paimon_parquet_file_format LIB64) - add_thirdparty(paimon_orc_file_format LIB64) - add_thirdparty(paimon_blob_file_format LIB64) - add_thirdparty(paimon_local_file_system LIB64) - add_thirdparty(paimon_file_index LIB64) - add_thirdparty(paimon_global_index LIB64) - - add_thirdparty(roaring_bitmap_paimon LIB64) - add_thirdparty(xxhash_paimon LIB64) - add_thirdparty(fmt_paimon LIB64) - add_thirdparty(tbb_paimon LIB64) + add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME}) + + add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME}) endif() diff --git a/build.sh b/build.sh index ceaa09a0779593..fdd74054241efb 100755 --- a/build.sh +++ b/build.sh @@ -518,7 +518,7 @@ rebuild_thirdparty_libraries() { build_args+=(--clean) fi bash "${build_script}" "${build_args[@]}" "$@" - if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then + if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2 exit 1 fi @@ -527,10 +527,12 @@ rebuild_thirdparty_libraries() { if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then echo "Thirdparty libraries need to be build ..." rebuild_thirdparty_libraries true -elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] && - ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." - rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}" +elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then + select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed" + if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then + echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." + rebuild_thirdparty_libraries false "${ARROW_PAIMON_REBUILD_PACKAGES[@]}" + fi fi update_submodule() { diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh index 1091c4fc84aa56..828dcc70da8f08 100644 --- a/thirdparty/arrow-paimon-vars.sh +++ b/thirdparty/arrow-paimon-vars.sh @@ -21,12 +21,23 @@ # Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can # distinguish this stack from unrelated thirdparty changes. -# arrow +# Arrow 24 is installed in a versioned prefix. The unversioned install prefix is +# deliberately reserved for Arrow 17 so the shared thirdparty package remains +# consumable by branch-4.1. ARROW_VERSION="24.0.0" ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz" ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz" ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}" ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" +ARROW_INSTALL_SUBDIR="arrow-${ARROW_VERSION}" + +# Arrow 17 compatibility stack for branch-4.1. Keep these variables separate +# from ARROW_* so master can build both versions from one source bundle. +ARROW_17_VERSION="17.0.0" +ARROW_17_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_17_VERSION}.tar.gz" +ARROW_17_NAME="apache-arrow-${ARROW_17_VERSION}.tar.gz" +ARROW_17_SOURCE="arrow-apache-arrow-${ARROW_17_VERSION}" +ARROW_17_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0" # Arrow bundled dependencies BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" @@ -39,16 +50,36 @@ XSIMD_NAME="14.0.0.tar.gz" XSIMD_SOURCE=xsimd-14.0.0 XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" +XSIMD_17_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz" +XSIMD_17_NAME="13.0.0.tar.gz" +XSIMD_17_SOURCE=xsimd-13.0.0 +XSIMD_17_MD5SUM="c661deb91836e82d3070f81032014fe6" + # paimon-cpp PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" +# Both Paimon variants use the same archive, but they need independent source +# trees because only the Arrow 24 tree receives the API compatibility patches. +PAIMON_CPP_17_DOWNLOAD="${PAIMON_CPP_DOWNLOAD}" +PAIMON_CPP_17_NAME="${PAIMON_CPP_NAME}" +PAIMON_CPP_17_ARCHIVE_SOURCE="${PAIMON_CPP_SOURCE}" +PAIMON_CPP_17_SOURCE="${PAIMON_CPP_SOURCE}-arrow-17" +PAIMON_CPP_17_MD5SUM="${PAIMON_CPP_MD5SUM}" + # Arrow consumes xsimd and Brotli as bundled source archives, but neither is a # build target in the focused Arrow/Paimon recovery path. +ARROW_PAIMON_17_BUILD_PACKAGES=(arrow_17 paimon_cpp_17) ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp) +ARROW_PAIMON_SHARED_BUILD_PACKAGES=( + "${ARROW_PAIMON_17_BUILD_PACKAGES[@]}" + "${ARROW_PAIMON_BUILD_PACKAGES[@]}" +) +ARROW_PAIMON_REBUILD_PACKAGES=() ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli) +ARROW_17_BUNDLED_SOURCE_PACKAGES=(xsimd_17 brotli) ARROW_PAIMON_DOWNLOAD_PACKAGES=() prepare_arrow_paimon_download_packages() { @@ -57,18 +88,25 @@ prepare_arrow_paimon_download_packages() { local package local source_package local arrow_requested=false + local arrow_17_requested=false local source_requested for package in "$@"; do if [[ "${package}" == "arrow" ]]; then arrow_requested=true - break + elif [[ "${package}" == "arrow_17" ]]; then + arrow_17_requested=true fi done - if [[ "${arrow_requested}" != "true" ]]; then - return + + local bundled_source_packages=() + if [[ "${arrow_requested}" == "true" ]]; then + bundled_source_packages+=("${ARROW_BUNDLED_SOURCE_PACKAGES[@]}") + fi + if [[ "${arrow_17_requested}" == "true" ]]; then + bundled_source_packages+=("${ARROW_17_BUNDLED_SOURCE_PACKAGES[@]}") fi - for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do + for source_package in "${bundled_source_packages[@]}"; do source_requested=false for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do if [[ "${package}" == "${source_package}" ]]; then @@ -82,6 +120,10 @@ prepare_arrow_paimon_download_packages() { done } +arrow_install_dir() { + printf '%s/%s\n' "$1" "${ARROW_INSTALL_SUBDIR}" +} + # Identify the checked-in source, patch, and build inputs selected for Arrow. # Arrow and Paimon publish separate installed markers so a package-only build # cannot certify a component that it did not rebuild. @@ -116,7 +158,9 @@ paimon_build_fingerprint() { download-thirdparty.sh \ build-thirdparty.sh \ paimon-cpp-cache.cmake \ - patches/paimon-cpp-*.patch + patches/paimon-cpp-buildutils-static-deps.patch \ + patches/paimon-cpp-arrow-24-compatibility.patch \ + patches/paimon-cpp-arrow-24-compute.patch } | git hash-object --stdin ) } @@ -130,6 +174,49 @@ arrow_paimon_build_fingerprint() { } | git hash-object --stdin } +arrow_17_build_fingerprint() { + local vars_dir + vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ( + cd "${vars_dir}" || return 1 + LC_ALL=C + git hash-object \ + ../env.sh \ + arrow-paimon-vars.sh \ + vars.sh \ + download-thirdparty.sh \ + build-thirdparty.sh \ + patches/apache-arrow-"${ARROW_17_VERSION}"-*.patch | + git hash-object --stdin + ) +} + +paimon_17_build_fingerprint() { + local vars_dir + vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ( + cd "${vars_dir}" || return 1 + LC_ALL=C + { + arrow_17_build_fingerprint + git hash-object \ + arrow-paimon-vars.sh \ + vars.sh \ + download-thirdparty.sh \ + build-thirdparty.sh \ + paimon-cpp-cache.cmake \ + patches/paimon-cpp-buildutils-static-deps.patch + } | git hash-object --stdin + ) +} + +arrow_paimon_17_build_fingerprint() { + { + arrow_17_build_fingerprint + paimon_17_build_fingerprint + } | git hash-object --stdin +} + ARROW_REQUIRED_LIBRARIES=( libbrotlicommon.a libbrotlidec.a @@ -144,6 +231,19 @@ ARROW_REQUIRED_LIBRARIES=( libparquet.a ) +ARROW_17_REQUIRED_LIBRARIES=( + libbrotlicommon.a + libbrotlidec.a + libbrotlienc.a + libarrow.a + libarrow_flight.a + libarrow_flight_sql.a + libarrow_dataset.a + libarrow_acero.a + libarrow_bundled_dependencies.a + libparquet.a +) + PAIMON_REQUIRED_LIBRARIES=( libpaimon.a libpaimon_parquet_file_format.a @@ -158,13 +258,64 @@ PAIMON_REQUIRED_LIBRARIES=( libtbb_paimon.a ) +# Remove only artifacts owned by the selected Arrow/Paimon stack before an +# install. This matters for the legacy prefix: installing Arrow 17 over an +# existing Arrow 24 prefix must not leave Arrow 24-only headers or libraries +# behind and turn it into a mixed, internally inconsistent SDK. +clean_arrow_artifacts_in() { + local install_dir="$1" + : "${install_dir:?Arrow install directory must be set}" + + rm -rf -- \ + "${install_dir}/include/arrow" \ + "${install_dir}/include/parquet" \ + "${install_dir}/share/arrow" \ + "${install_dir}/share/doc/arrow" + + ( + shopt -s nullglob + local generated_artifacts=( + "${install_dir}/lib64"/libarrow* + "${install_dir}/lib64"/libparquet* + "${install_dir}/lib64/cmake"/Arrow* + "${install_dir}/lib64/cmake"/Parquet + "${install_dir}/lib64/pkgconfig"/arrow*.pc + "${install_dir}/lib64/pkgconfig"/parquet.pc + ) + rm -rf -- "${generated_artifacts[@]}" + ) +} + +clean_paimon_artifacts_in() { + local install_dir="$1" + : "${install_dir:?Paimon install directory must be set}" + + rm -rf -- \ + "${install_dir}/include/paimon" \ + "${install_dir}/lib64/cmake/Paimon" \ + "${install_dir}/paimon-cpp" + + ( + shopt -s nullglob + local generated_artifacts=( + "${install_dir}/lib64"/libpaimon* + "${install_dir}/lib64"/libroaring_bitmap_paimon.* + "${install_dir}/lib64"/libxxhash_paimon.* + "${install_dir}/lib64"/libfmt_paimon.* + "${install_dir}/lib64"/libtbb_paimon.* + ) + rm -rf -- "${generated_artifacts[@]}" + ) +} + ARROW_PAIMON_REQUIRED_LIBRARIES=( "${ARROW_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}" ) arrow_artifacts_valid() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" local installed_arrow_version local library @@ -191,7 +342,35 @@ arrow_artifacts_valid() { return 0 } -paimon_artifacts_valid() { +arrow_17_artifacts_valid() { + local install_dir="$1" + local installed_arrow_version + local library + + if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then + echo "Missing installed Arrow 17 version header" >&2 + return 1 + fi + installed_arrow_version="$( + awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" { + gsub(/"/, "", $3); print $3; exit + }' "${install_dir}/include/arrow/util/config.h" + )" + if [[ "${installed_arrow_version}" != "${ARROW_17_VERSION}" ]]; then + echo "Installed legacy Arrow version ${installed_arrow_version} does not match ${ARROW_17_VERSION}" >&2 + return 1 + fi + + for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"; do + if [[ ! -f "${install_dir}/lib64/${library}" ]]; then + echo "Missing Arrow 17 library: ${library}" >&2 + return 1 + fi + done + return 0 +} + +paimon_artifacts_valid_in() { local install_dir="$1" local library @@ -204,8 +383,13 @@ paimon_artifacts_valid() { return 0 } +paimon_artifacts_valid() { + paimon_artifacts_valid_in "$(arrow_install_dir "$1")" +} + arrow_prebuilt_valid() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt" local expected_fingerprint local installed_fingerprint @@ -220,11 +404,12 @@ arrow_prebuilt_valid() { echo "Arrow build fingerprint does not match selected inputs" >&2 return 1 fi - arrow_artifacts_valid "${install_dir}" + arrow_artifacts_valid "$1" } paimon_prebuilt_valid() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt" local expected_fingerprint local installed_fingerprint @@ -239,7 +424,7 @@ paimon_prebuilt_valid() { echo "Paimon build fingerprint does not match selected inputs" >&2 return 1 fi - paimon_artifacts_valid "${install_dir}" + paimon_artifacts_valid "$1" } arrow_paimon_prebuilt_valid() { @@ -248,26 +433,32 @@ arrow_paimon_prebuilt_valid() { } invalidate_arrow_prebuilt_marker() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" + mkdir -p "${install_dir}" rm -f "${install_dir}/arrow-build-fingerprint.txt" \ "${install_dir}/arrow-paimon-build-fingerprint.txt" } publish_arrow_prebuilt_marker() { - local install_dir="$1" - arrow_artifacts_valid "${install_dir}" + local install_dir + install_dir="$(arrow_install_dir "$1")" + arrow_artifacts_valid "$1" arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt" } invalidate_paimon_prebuilt_marker() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" + mkdir -p "${install_dir}" rm -f "${install_dir}/paimon-build-fingerprint.txt" \ "${install_dir}/arrow-paimon-build-fingerprint.txt" } publish_paimon_prebuilt_marker() { - local install_dir="$1" - paimon_artifacts_valid "${install_dir}" + local install_dir + install_dir="$(arrow_install_dir "$1")" + paimon_artifacts_valid "$1" paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt" } @@ -278,3 +469,93 @@ require_arrow_prebuilt_for_paimon() { return 1 fi } + +invalidate_arrow_17_prebuilt_marker() { + local install_dir="$1" + rm -f "${install_dir}/arrow-17-build-fingerprint.txt" \ + "${install_dir}/arrow-paimon-17-build-fingerprint.txt" +} + +publish_arrow_17_prebuilt_marker() { + local install_dir="$1" + arrow_17_artifacts_valid "${install_dir}" + arrow_17_build_fingerprint >"${install_dir}/arrow-17-build-fingerprint.txt" +} + +arrow_17_prebuilt_valid() { + local install_dir="$1" + local fingerprint_mark="${install_dir}/arrow-17-build-fingerprint.txt" + local expected_fingerprint + + if [[ ! -f "${fingerprint_mark}" ]]; then + echo "Missing Arrow 17 build fingerprint: ${fingerprint_mark}" >&2 + return 1 + fi + expected_fingerprint="$(arrow_17_build_fingerprint)" + if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then + echo "Arrow 17 build fingerprint does not match selected inputs" >&2 + return 1 + fi + arrow_17_artifacts_valid "${install_dir}" +} + +require_arrow_17_prebuilt_for_paimon() { + local install_dir="$1" + if ! arrow_17_prebuilt_valid "${install_dir}"; then + echo "Paimon for branch-4.1 requires Arrow 17 to be built first" >&2 + return 1 + fi +} + +invalidate_paimon_17_prebuilt_marker() { + local install_dir="$1" + rm -f "${install_dir}/paimon-arrow-17-build-fingerprint.txt" \ + "${install_dir}/arrow-paimon-17-build-fingerprint.txt" +} + +publish_paimon_17_prebuilt_marker() { + local install_dir="$1" + paimon_artifacts_valid_in "${install_dir}" + paimon_17_build_fingerprint >"${install_dir}/paimon-arrow-17-build-fingerprint.txt" +} + +paimon_17_prebuilt_valid() { + local install_dir="$1" + local fingerprint_mark="${install_dir}/paimon-arrow-17-build-fingerprint.txt" + local expected_fingerprint + + if [[ ! -f "${fingerprint_mark}" ]]; then + echo "Missing Paimon Arrow 17 build fingerprint: ${fingerprint_mark}" >&2 + return 1 + fi + expected_fingerprint="$(paimon_17_build_fingerprint)" + if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then + echo "Paimon Arrow 17 build fingerprint does not match selected inputs" >&2 + return 1 + fi + paimon_artifacts_valid_in "${install_dir}" +} + +arrow_paimon_17_prebuilt_valid() { + local install_dir="$1" + arrow_17_prebuilt_valid "${install_dir}" && + paimon_17_prebuilt_valid "${install_dir}" +} + +shared_arrow_paimon_prebuilt_valid() { + local install_dir="$1" + arrow_paimon_17_prebuilt_valid "${install_dir}" && + arrow_paimon_prebuilt_valid "${install_dir}" +} + +select_arrow_paimon_rebuild_packages() { + local install_dir="$1" + ARROW_PAIMON_REBUILD_PACKAGES=() + + if ! arrow_paimon_17_prebuilt_valid "${install_dir}"; then + ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_17_BUILD_PACKAGES[@]}") + fi + if ! arrow_paimon_prebuilt_valid "${install_dir}"; then + ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_BUILD_PACKAGES[@]}") + fi +} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index a6544a57ac2fcd..178ef8f204f63b 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -313,20 +313,26 @@ else echo "Do not strip thirdparty libraries" fi -strip_lib() { +strip_lib_at() { + local install_dir="$1" + local library="$2" if [[ "${STRIP_TP_LIB}" = "ON" ]]; then - if [[ -z $1 ]]; then + if [[ -z "${library}" ]]; then echo "Must specify the library to be stripped." exit 1 fi - if [[ ! -f "${TP_LIB_DIR}/$1" ]]; then - echo "Library to be stripped (${TP_LIB_DIR}/$1) does not exist." + if [[ ! -f "${install_dir}/lib/${library}" ]]; then + echo "Library to be stripped (${install_dir}/lib/${library}) does not exist." exit 1 fi - strip --strip-debug --strip-unneeded "${TP_LIB_DIR}/$1" + strip --strip-debug --strip-unneeded "${install_dir}/lib/${library}" fi } +strip_lib() { + strip_lib_at "${TP_INSTALL_DIR}" "$1" +} + #libbacktrace build_libbacktrace() { check_if_source_exist "${LIBBACKTRACE_SOURCE}" @@ -1062,11 +1068,18 @@ build_grpc() { # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake } -# arrow -build_arrow() { - check_if_source_exist "${ARROW_SOURCE}" - invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp" +# Arrow 17 is installed in the legacy unversioned prefix for branch-4.1, while +# Arrow 24 is installed in a versioned prefix selected by master. +build_arrow_stack() { + local arrow_source="$1" + local xsimd_archive="$2" + local install_dir="$3" + local has_separate_compute_archive="$4" + + check_if_source_exist "${arrow_source}" + mkdir -p "${install_dir}/lib64" + ln -sfn lib64 "${install_dir}/lib" + cd "${TP_SOURCE_DIR}/${arrow_source}/cpp" mkdir -p release cd release @@ -1079,7 +1092,7 @@ build_arrow() { export ARROW_Thrift_URL="${TP_SOURCE_DIR}/${THRIFT_NAME}" export ARROW_SNAPPY_URL="${TP_SOURCE_DIR}/${SNAPPY_NAME}" export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}" - export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}" + export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${xsimd_archive}" export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}" export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}" export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}" @@ -1101,7 +1114,7 @@ build_arrow() { -DARROW_FILESYSTEM=ON \ -DARROW_DATASET=ON \ -DARROW_ACERO=ON \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DCMAKE_INSTALL_LIBDIR=lib64 \ -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_WITH_GRPC=ON \ @@ -1144,15 +1157,31 @@ build_arrow() { "${BUILD_SYSTEM}" install #copy dep libs - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a "${TP_INSTALL_DIR}/lib64/libbrotlienc.a" - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${TP_INSTALL_DIR}/lib64/libbrotlidec.a" - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${TP_INSTALL_DIR}/lib64/libbrotlicommon.a" - strip_lib libarrow.a - strip_lib libarrow_compute.a - strip_lib libparquet.a - strip_lib libarrow_dataset.a - strip_lib libarrow_acero.a + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a "${install_dir}/lib64/libbrotlienc.a" + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${install_dir}/lib64/libbrotlidec.a" + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${install_dir}/lib64/libbrotlicommon.a" + strip_lib_at "${install_dir}" libarrow.a + if [[ "${has_separate_compute_archive}" == "true" ]]; then + strip_lib_at "${install_dir}" libarrow_compute.a + fi + strip_lib_at "${install_dir}" libparquet.a + strip_lib_at "${install_dir}" libarrow_dataset.a + strip_lib_at "${install_dir}" libarrow_acero.a +} +build_arrow_17() { + invalidate_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}" + clean_arrow_artifacts_in "${TP_INSTALL_DIR}" + build_arrow_stack "${ARROW_17_SOURCE}" "${XSIMD_17_NAME}" "${TP_INSTALL_DIR}" false + publish_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}" +} + +build_arrow() { + local install_dir + install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")" + invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" + clean_arrow_artifacts_in "${install_dir}" + build_arrow_stack "${ARROW_SOURCE}" "${XSIMD_NAME}" "${install_dir}" true publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}" } @@ -2131,12 +2160,18 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } -# paimon-cpp -build_paimon_cpp() { - check_if_source_exist "${PAIMON_CPP_SOURCE}" - require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" +# Build each Paimon variant against the matching Arrow prefix and install it +# beside that Arrow version. Arrow types cross Paimon's public C++ boundary, so +# mixing the two versions is not ABI-safe. +build_paimon_cpp_stack() { + local paimon_source="$1" + local arrow_install_dir="$2" + local install_dir="$3" + + check_if_source_exist "${paimon_source}" + mkdir -p "${install_dir}/lib64" + ln -sfn lib64 "${install_dir}/lib" + cd "${TP_SOURCE_DIR}/${paimon_source}" rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" @@ -2148,12 +2183,13 @@ build_paimon_cpp() { paimon_linker_flags="${paimon_linker_flags} -lunwind" fi + PAIMON_ARROW_INSTALL_DIR="${arrow_install_dir}" \ CXXFLAGS="-Wno-nontrivial-memcall" \ "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \ -G "${GENERATOR}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DPAIMON_BUILD_SHARED=OFF \ -DPAIMON_BUILD_STATIC=ON \ -DPAIMON_BUILD_TESTS=OFF \ @@ -2177,7 +2213,7 @@ build_paimon_cpp() { # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps # directory is not needed. When building its own Arrow (legacy), copy # arrow artefacts into an isolated directory to avoid clashing with Doris. - local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps" + local paimon_deps_dir="${install_dir}/paimon-cpp/lib64/paimon_deps" if [ -d "arrow_ep-install/lib" ]; then mkdir -p "${paimon_deps_dir}" for paimon_arrow_dep in \ @@ -2197,25 +2233,42 @@ build_paimon_cpp() { # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap if [ -f "release/libroaring_bitmap.a" ]; then - cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a" + cp -v "release/libroaring_bitmap.a" "${install_dir}/lib64/libroaring_bitmap_paimon.a" fi # Install xxhash, renamed to avoid conflict with Doris's xxhash if [ -f "release/libxxhash.a" ]; then - cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a" + cp -v "release/libxxhash.a" "${install_dir}/lib64/libxxhash_paimon.a" fi # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7) if [ -f "fmt_ep-install/lib/libfmt.a" ]; then - cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a" + cp -v "fmt_ep-install/lib/libfmt.a" "${install_dir}/lib64/libfmt_paimon.a" fi # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb) if [ -f "tbb_ep-install/lib/libtbb.a" ]; then - cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a" + cp -v "tbb_ep-install/lib/libtbb.a" "${install_dir}/lib64/libtbb_paimon.a" fi echo "Paimon-cpp internal dependencies installed successfully" +} + +build_paimon_cpp_17() { + require_arrow_17_prebuilt_for_paimon "${TP_INSTALL_DIR}" + invalidate_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}" + clean_paimon_artifacts_in "${TP_INSTALL_DIR}" + build_paimon_cpp_stack "${PAIMON_CPP_17_SOURCE}" "${TP_INSTALL_DIR}" "${TP_INSTALL_DIR}" + publish_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}" +} + +build_paimon_cpp() { + local install_dir + install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")" + require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" + invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" + clean_paimon_artifacts_in "${install_dir}" + build_paimon_cpp_stack "${PAIMON_CPP_SOURCE}" "${install_dir}" "${install_dir}" publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}" } @@ -2320,6 +2373,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then orc cares grpc # after cares, protobuf + arrow_17 arrow arrow_adbc lance_c @@ -2357,6 +2411,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then brotli icu pugixml + paimon_cpp_17 paimon_cpp ) if [[ "$(uname -s)" == 'Darwin' ]]; then @@ -2412,6 +2467,7 @@ cleanup_package_source() { cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;; librdkafka) src_var="LIBRDKAFKA_SOURCE" ;; flatbuffers) src_var="FLATBUFFERS_SOURCE" ;; + arrow_17) src_var="ARROW_17_SOURCE" ;; arrow) src_var="ARROW_SOURCE" ;; arrow_adbc) # arrow_adbc also unpacks the prebuilt flightsql driver, clean both @@ -2465,6 +2521,7 @@ cleanup_package_source() { jindofs) src_var="JINDOFS_SOURCE" ;; juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; + paimon_cpp_17) src_var="PAIMON_CPP_17_SOURCE" ;; paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index ea9436be2dec72..bf5fd05c496581 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -238,13 +238,24 @@ for TP_ARCH in "${TP_ARCHIVES[@]}"; do fi NAME="${TP_ARCH}_NAME" SOURCE="${TP_ARCH}_SOURCE" + ARCHIVE_SOURCE_VAR="${TP_ARCH}_ARCHIVE_SOURCE" + ARCHIVE_SOURCE="${!ARCHIVE_SOURCE_VAR}" if [[ -z "${!SOURCE}" ]]; then continue fi if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then - if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then + if [[ -n "${ARCHIVE_SOURCE}" && "${ARCHIVE_SOURCE}" != "${!SOURCE}" ]]; then + alias_unpack_dir="$(mktemp -d "${TP_SOURCE_DIR}/.unpack-${TP_ARCH}.XXXXXX")" + if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${alias_unpack_dir}"; then + echo "Failed to untar ${!NAME} for ${!SOURCE}" + rm -rf "${alias_unpack_dir}" + exit 1 + fi + mv "${alias_unpack_dir}/${ARCHIVE_SOURCE}" "${TP_SOURCE_DIR}/${!SOURCE}" + rm -rf "${alias_unpack_dir}" + elif [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then echo "${TP_SOURCE_DIR}/${!NAME}" echo "${TP_SOURCE_DIR}/${!SOURCE}" if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then @@ -336,17 +347,31 @@ echo "===== Patching thirdparty archives..." PATCHED_MARK="patched_mark" ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint" ARROW_PAIMON_BUILD_FINGERPRINT="" +ARROW_PAIMON_17_BUILD_FINGERPRINT="" if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " || " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)" fi +if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " || + " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then + ARROW_PAIMON_17_BUILD_FINGERPRINT="$(arrow_paimon_17_build_fingerprint)" +fi reset_arrow_paimon_source() { local archive_name="$1" local source_name="$2" + local archive_source="${3:-${source_name}}" echo "Resetting ${source_name} because its patch state is incomplete or stale" rm -rf "${TP_SOURCE_DIR:?}/${source_name}" - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" + if [[ "${archive_source}" == "${source_name}" ]]; then + "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" + else + local alias_unpack_dir + alias_unpack_dir="$(mktemp -d "${TP_SOURCE_DIR}/.reset-${source_name}.XXXXXX")" + "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${alias_unpack_dir}" + mv "${alias_unpack_dir}/${archive_source}" "${TP_SOURCE_DIR}/${source_name}" + rm -rf "${alias_unpack_dir}" + fi } # glog patch @@ -442,6 +467,26 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ROCKSDB " ]]; then echo "Finished patching ${ROCKSDB_SOURCE}" fi +# Keep the Arrow 17 source used by branch-4.1 independently patched from the +# Arrow 24 source selected by master. +if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ]]; then + arrow_17_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${PATCHED_MARK}" && + -f "${arrow_17_fingerprint_mark}" ]] || + [[ "$(<"${arrow_17_fingerprint_mark}")" != "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then + reset_arrow_paimon_source "${ARROW_17_NAME}" "${ARROW_17_SOURCE}" + cd "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch" + touch "${PATCHED_MARK}" + printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \ + >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + cd - + fi + echo "Finished patching ${ARROW_17_SOURCE}" +fi + # arrow patch is used to get the raw orc reader for filter prune. if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then @@ -761,6 +806,24 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi +# Keep the Arrow 17 Paimon source free of the Arrow 24 API and Compute patches. +if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then + paimon_17_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${PATCHED_MARK}" && + -f "${paimon_17_fingerprint_mark}" ]] || + [[ "$(<"${paimon_17_fingerprint_mark}")" != "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then + reset_arrow_paimon_source "${PAIMON_CPP_17_NAME}" "${PAIMON_CPP_17_SOURCE}" \ + "${PAIMON_CPP_17_ARCHIVE_SOURCE}" + cd "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}" + patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" + touch "${PATCHED_MARK}" + printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \ + >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + cd - + fi + echo "Finished patching ${PAIMON_CPP_17_SOURCE}" +fi + # patch paimon-cpp if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake index 6ab6049c59b0d5..a12540e5d3ed4e 100644 --- a/thirdparty/paimon-cpp-cache.cmake +++ b/thirdparty/paimon-cpp-cache.cmake @@ -21,12 +21,17 @@ # Get the Doris thirdparty installation directory from environment set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory") +set(DORIS_ARROW_DIR "$ENV{PAIMON_ARROW_INSTALL_DIR}" CACHE PATH "Selected Doris Arrow install directory") if(NOT DORIS_THIRDPARTY_DIR) message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set") endif() +if(NOT DORIS_ARROW_DIR) + message(FATAL_ERROR "PAIMON_ARROW_INSTALL_DIR environment variable must be set") +endif() message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}") +message(STATUS "Using selected Arrow libraries from: ${DORIS_ARROW_DIR}") # Set CMAKE_PREFIX_PATH to help find_package locate our libraries set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package") @@ -64,21 +69,21 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") # ============================================================================ # ============================================================================ -# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp -# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). +# Arrow - Reuse the version selected by the caller. Both installed Arrow stacks +# include DATASET/ACERO/FILESYSTEM; Arrow 24 also has a separate Compute archive. # ============================================================================ set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") -set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory") +set(DORIS_ARROW_LIB64_DIR "${DORIS_ARROW_DIR}/lib64" CACHE PATH "Selected Arrow lib64 directory") +set(DORIS_ARROW_INCLUDE_DIR "${DORIS_ARROW_DIR}/include" CACHE PATH "Selected Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") -set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") -set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") -set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") -set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") -set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") +set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_ARROW_INCLUDE_DIR}" CACHE PATH "Arrow include directory") +set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") +set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") +set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") +set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") +set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_ARROW_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") +set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") # Protobuf, Thrift - still built separately by paimon-cpp diff --git a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch new file mode 100644 index 00000000000000..5a75424756671d --- /dev/null +++ b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch @@ -0,0 +1,98 @@ +diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc +--- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:23:23.651831424 +0800 ++++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:28:36.855281965 +0800 +@@ -178,7 +178,8 @@ + + // The user is explicitly asking for Impala int96 encoding, there is no + // logical type. +- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { ++ if (arrow_properties.force_write_int96_timestamps() || ++ (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO)) { + *physical_type = ParquetType::INT96; + return Status::OK(); + } +diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h +--- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h 2026-03-27 01:23:23.643831362 +0800 ++++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27 01:27:47.717897537 +0800 +@@ -980,6 +980,7 @@ + public: + Builder() + : write_timestamps_as_int96_(false), ++ force_write_int96_timestamps_(false), + coerce_timestamps_enabled_(false), + coerce_timestamps_unit_(::arrow::TimeUnit::SECOND), + truncated_timestamps_allowed_(false), +@@ -1005,6 +1006,21 @@ + return this; + } + ++ /// \brief Force writing legacy int96 timestamps. ++ /// ++ /// This bypasses unit-based guards and writes INT96 whenever timestamp ++ /// metadata is resolved. ++ Builder* enable_force_write_int96_timestamps() { ++ force_write_int96_timestamps_ = true; ++ return this; ++ } ++ ++ /// \brief Disable forcing legacy int96 timestamps (default). ++ Builder* disable_force_write_int96_timestamps() { ++ force_write_int96_timestamps_ = false; ++ return this; ++ } ++ + /// \brief Coerce all timestamps to the specified time unit. + /// \param unit time unit to truncate to. + /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds. +@@ -1085,7 +1101,8 @@ + /// Create the final properties. + std::shared_ptr build() { + return std::shared_ptr(new ArrowWriterProperties( +- write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_, ++ write_timestamps_as_int96_, force_write_int96_timestamps_, ++ coerce_timestamps_enabled_, coerce_timestamps_unit_, + truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, + engine_version_, use_threads_, executor_)); + } +@@ -1093,6 +1110,8 @@ + private: + bool write_timestamps_as_int96_; + ++ bool force_write_int96_timestamps_; ++ + bool coerce_timestamps_enabled_; + ::arrow::TimeUnit::type coerce_timestamps_unit_; + bool truncated_timestamps_allowed_; +@@ -1107,6 +1126,8 @@ + + bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; } + ++ bool force_write_int96_timestamps() const { return force_write_int96_timestamps_; } ++ + bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; } + ::arrow::TimeUnit::type coerce_timestamps_unit() const { + return coerce_timestamps_unit_; +@@ -1138,6 +1159,7 @@ + + private: + explicit ArrowWriterProperties(bool write_nanos_as_int96, ++ bool force_write_int96_timestamps, + bool coerce_timestamps_enabled, + ::arrow::TimeUnit::type coerce_timestamps_unit, + bool truncated_timestamps_allowed, bool store_schema, +@@ -1145,6 +1167,7 @@ + EngineVersion engine_version, bool use_threads, + ::arrow::internal::Executor* executor) + : write_timestamps_as_int96_(write_nanos_as_int96), ++ force_write_int96_timestamps_(force_write_int96_timestamps), + coerce_timestamps_enabled_(coerce_timestamps_enabled), + coerce_timestamps_unit_(coerce_timestamps_unit), + truncated_timestamps_allowed_(truncated_timestamps_allowed), +@@ -1155,6 +1178,7 @@ + executor_(executor) {} + + const bool write_timestamps_as_int96_; ++ const bool force_write_int96_timestamps_; + const bool coerce_timestamps_enabled_; + const ::arrow::TimeUnit::type coerce_timestamps_unit_; + const bool truncated_timestamps_allowed_; diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch new file mode 100644 index 00000000000000..a983818413a01c --- /dev/null +++ b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch @@ -0,0 +1,84 @@ +--- a/cpp/src/parquet/column_reader.cc ++++ b/cpp/src/parquet/column_reader.cc +@@ -30,0 +31,2 @@ ++ ++#include +@@ -268,0 +269 @@ ++ compression_codec_(codec), +@@ -279 +282,7 @@ +- decompressor_ = GetCodec(codec); ++ if (compression_codec_ == Compression::LZO) { ++ if (lzo_init() != LZO_E_OK) { ++ throw ParquetException("Failed to initialize LZO codec"); ++ } ++ } else { ++ decompressor_ = GetCodec(codec); ++ } +@@ -315,0 +325 @@ ++ Compression::type compression_codec_; +@@ -585 +595 @@ +- if (decompressor_ == nullptr) { ++ if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) { +@@ -601,0 +612,61 @@ ++ if (compression_codec_ == Compression::LZO) { ++ const uint8_t* input = page_buffer->data() + levels_byte_len; ++ const uint8_t* const input_end = page_buffer->data() + compressed_len; ++ uint8_t* output = decompression_buffer_->mutable_data() + levels_byte_len; ++ uint8_t* const output_end = decompression_buffer_->mutable_data() + uncompressed_len; ++ ++ auto load_big_endian_u32 = [](const uint8_t* data) { ++ return (static_cast(data[0]) << 24) | ++ (static_cast(data[1]) << 16) | ++ (static_cast(data[2]) << 8) | static_cast(data[3]); ++ }; ++ ++ while (input < input_end) { ++ if (input_end - input < 4) { ++ throw ParquetException("LZO page decompression failed: truncated large block length"); ++ } ++ ++ uint32_t large_block_uncompressed_len = load_big_endian_u32(input); ++ input += 4; ++ if (static_cast(output_end - output) < large_block_uncompressed_len) { ++ throw ParquetException("LZO page decompression failed: output buffer too small"); ++ } ++ ++ while (large_block_uncompressed_len > 0) { ++ if (input_end - input < 4) { ++ throw ParquetException("LZO page decompression failed: truncated small block length"); ++ } ++ ++ uint32_t small_block_compressed_len = load_big_endian_u32(input); ++ input += 4; ++ if (static_cast(input_end - input) < small_block_compressed_len) { ++ throw ParquetException("LZO page decompression failed: truncated small block data"); ++ } ++ ++ auto small_block_uncompressed_len = ++ static_cast(large_block_uncompressed_len); ++ const int result = ++ lzo1x_decompress_safe(input, static_cast(small_block_compressed_len), ++ output, &small_block_uncompressed_len, nullptr); ++ if (result != LZO_E_OK) { ++ throw ParquetException("LZO page decompression failed, error: " + ++ std::to_string(result)); ++ } ++ if (small_block_uncompressed_len > large_block_uncompressed_len) { ++ throw ParquetException("LZO page decompression failed: invalid small block size"); ++ } ++ ++ input += small_block_compressed_len; ++ output += small_block_uncompressed_len; ++ large_block_uncompressed_len -= small_block_uncompressed_len; ++ } ++ } ++ if (output != output_end) { ++ throw ParquetException("Page didn't decompress to expected size, expected: " + ++ std::to_string(uncompressed_len - levels_byte_len) + ", but got:" + ++ std::to_string(output - (decompression_buffer_->mutable_data() + ++ levels_byte_len))); ++ } ++ ++ return decompression_buffer_; ++ } ++ diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch new file mode 100644 index 00000000000000..4e53117b79b65b --- /dev/null +++ b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch @@ -0,0 +1,224 @@ +diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc +index ec3890a41f..943f69bb6c 100644 +--- a/cpp/src/parquet/arrow/schema.cc ++++ b/cpp/src/parquet/arrow/schema.cc +@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, + + // The user is explicitly asking for Impala int96 encoding, there is no + // logical type. +- if (arrow_properties.support_deprecated_int96_timestamps()) { ++ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { + *physical_type = ParquetType::INT96; + return Status::OK(); + } + +diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc +index 285e2a5973..aa6f92f077 100644 +--- a/cpp/src/parquet/arrow/reader.cc ++++ b/cpp/src/parquet/arrow/reader.cc +@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector& row_groups, + return Status::OK(); + } + +- int64_t num_rows = 0; ++ std::vector num_rows; + for (int row_group : row_groups) { +- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); ++ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); + } + + using ::arrow::RecordBatchIterator; ++ int row_group_idx = 0; + + // NB: This lambda will be invoked outside the scope of this call to + // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. + // `this` is a non-owning pointer so we are relying on the parent FileReader outliving + // this RecordBatchReader. + ::arrow::Iterator batches = ::arrow::MakeFunctionIterator( +- [readers, batch_schema, num_rows, ++ [readers, batch_schema, num_rows, row_group_idx, + this]() mutable -> ::arrow::Result { + ::arrow::ChunkedArrayVector columns(readers.size()); + +- // don't reserve more rows than necessary +- int64_t batch_size = std::min(properties().batch_size(), num_rows); +- num_rows -= batch_size; ++ int64_t batch_size = 0; ++ if (!num_rows.empty()) { ++ // don't reserve more rows than necessary ++ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); ++ num_rows[row_group_idx] -= batch_size; ++ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) != row_group_idx) { ++ row_group_idx++; ++ } ++ } + + RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( + reader_properties_.use_threads(), static_cast(readers.size()), +diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc +index 4fd7ef1b47..87326a54f1 100644 +--- a/cpp/src/parquet/arrow/writer.cc ++++ b/cpp/src/parquet/arrow/writer.cc +@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter { + return Status::OK(); + } + ++ int64_t GetBufferedSize() override { ++ if (row_group_writer_ == nullptr) { ++ return 0; ++ } ++ return row_group_writer_->total_compressed_bytes() + ++ row_group_writer_->total_compressed_bytes_written(); ++ } ++ + Status Close() override { + if (!closed_) { + // Make idempotent +@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter { + + // Max number of rows allowed in a row group. + const int64_t max_row_group_length = this->properties().max_row_group_length(); ++ const int64_t max_row_group_size = this->properties().max_row_group_size(); + + // Initialize a new buffered row group writer if necessary. + if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || +- row_group_writer_->num_rows() >= max_row_group_length) { ++ row_group_writer_->num_rows() >= max_row_group_length || ++ (row_group_writer_->total_compressed_bytes_written() + ++ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) { + RETURN_NOT_OK(NewBufferedRowGroup()); + } + +diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h +index 4a1a033a7b..0f13d05e44 100644 +--- a/cpp/src/parquet/arrow/writer.h ++++ b/cpp/src/parquet/arrow/writer.h +@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter { + /// option in this case. + virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; + ++ /// \brief Return the buffered size in bytes. ++ virtual int64_t GetBufferedSize() = 0; ++ + /// \brief Write the footer and close the file. + virtual ::arrow::Status Close() = 0; + virtual ~FileWriter(); +diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h +index 4d3acb491e..3906ff3c59 100644 +--- a/cpp/src/parquet/properties.h ++++ b/cpp/src/parquet/properties.h +@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true; + static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; + static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; + static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; ++static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024; + static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; + static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; + static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; +@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties { + dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), + write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), + max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), ++ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), + pagesize_(kDefaultDataPageSize), + version_(ParquetVersion::PARQUET_2_6), + data_page_version_(ParquetDataPageVersion::V1), +@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties { + dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), + write_batch_size_(properties.write_batch_size()), + max_row_group_length_(properties.max_row_group_length()), ++ max_row_group_size_(properties.max_row_group_size()), + pagesize_(properties.data_pagesize()), + version_(properties.version()), + data_page_version_(properties.data_page_version()), +@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties { + return this; + } + ++ /// Specify the max bytes size to put in a single row group. ++ /// Default 128 M. ++ Builder* max_row_group_size(int64_t max_row_group_size) { ++ max_row_group_size_ = max_row_group_size; ++ return this; ++ } ++ + /// Specify the data page size. + /// Default 1MB. + Builder* data_pagesize(int64_t pg_size) { +@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties { + + return std::shared_ptr(new WriterProperties( + pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, +- pagesize_, version_, created_by_, page_checksum_enabled_, ++ max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_, + std::move(file_encryption_properties_), default_column_properties_, + column_properties, data_page_version_, store_decimal_as_integer_, + std::move(sorting_columns_))); +@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties { + int64_t dictionary_pagesize_limit_; + int64_t write_batch_size_; + int64_t max_row_group_length_; ++ int64_t max_row_group_size_; + int64_t pagesize_; + ParquetVersion::type version_; + ParquetDataPageVersion data_page_version_; +@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties { + + inline int64_t max_row_group_length() const { return max_row_group_length_; } + ++ inline int64_t max_row_group_size() const { return max_row_group_size_; } ++ + inline int64_t data_pagesize() const { return pagesize_; } + + inline ParquetDataPageVersion data_page_version() const { +@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties { + private: + explicit WriterProperties( + MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, +- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version, ++ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version, + const std::string& created_by, bool page_write_checksum_enabled, + std::shared_ptr file_encryption_properties, + const ColumnProperties& default_column_properties, +@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties { + dictionary_pagesize_limit_(dictionary_pagesize_limit), + write_batch_size_(write_batch_size), + max_row_group_length_(max_row_group_length), ++ max_row_group_size_(max_row_group_size), + pagesize_(pagesize), + parquet_data_page_version_(data_page_version), + parquet_version_(version), +@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties { + int64_t dictionary_pagesize_limit_; + int64_t write_batch_size_; + int64_t max_row_group_length_; ++ int64_t max_row_group_size_; + int64_t pagesize_; + ParquetDataPageVersion parquet_data_page_version_; + ParquetVersion::type parquet_version_; +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +index 9df922afa2..5c8b3d4d07 100644 +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) + REQUIRED_VERSION + 0.11.0) + +- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) ++ if(NOT Thrift_VERSION) ++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) ++ set(Thrift_VERSION "${thrift_PC_VERSION}") ++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) ++ set(Thrift_VERSION "${ThriftAlt_VERSION}") ++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) ++ set(Thrift_VERSION "${THRIFT_VERSION}") ++ endif() ++ endif() ++ if(NOT Thrift_VERSION) ++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") ++ endif() ++ ++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") + list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) + list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) + list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh index ab48d3bcd2278b..ba65efb7a2d1da 100755 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh @@ -60,10 +60,14 @@ create_archive() { arrow_source="arrow-apache-arrow-24.0.0" arrow_archive="apache-arrow-24.0.0.tar.gz" +arrow_17_source="arrow-apache-arrow-17.0.0" +arrow_17_archive="apache-arrow-17.0.0.tar.gz" paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2" +paimon_17_source="${paimon_source}-arrow-17" paimon_archive="paimon-cpp-0a4f4e2.tar.gz" create_archive "${arrow_source}" "${arrow_archive}" arrow +create_archive "${arrow_17_source}" "${arrow_17_archive}" arrow17 create_archive "${paimon_source}" "${paimon_archive}" paimon arrow_patches=( @@ -71,6 +75,11 @@ arrow_patches=( apache-arrow-24.0.0-force-write-int96-timestamps.patch apache-arrow-24.0.0-lzo.patch ) +arrow_17_patches=( + apache-arrow-17.0.0-paimon.patch + apache-arrow-17.0.0-force-write-int96-timestamps.patch + apache-arrow-17.0.0-lzo.patch +) paimon_patches=( paimon-cpp-buildutils-static-deps.patch paimon-cpp-arrow-24-compatibility.patch @@ -79,16 +88,22 @@ paimon_patches=( for index in 0 1 2; do create_patch "${harness}/patches/${arrow_patches[${index}]}" "arrow-$((index + 1)).txt" + create_patch "${harness}/patches/${arrow_17_patches[${index}]}" "arrow17-$((index + 1)).txt" create_patch "${harness}/patches/${paimon_patches[${index}]}" "paimon-$((index + 1)).txt" done arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')" +arrow_17_md5="$(md5sum "${harness}/src/${arrow_17_archive}" | awk '{print $1}')" paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" { printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src" printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed" printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches" - printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)' + printf '%s\n' 'TP_ARCHIVES=(ARROW_17 ARROW PAIMON_CPP_17 PAIMON_CPP)' + printf 'ARROW_17_NAME="%s"\n' "${arrow_17_archive}" + printf 'ARROW_17_SOURCE="%s"\n' "${arrow_17_source}" + printf 'ARROW_17_MD5SUM="%s"\n' "${arrow_17_md5}" + printf '%s\n' 'ARROW_17_DOWNLOAD="unused"' printf 'ARROW_NAME="%s"\n' "${arrow_archive}" printf 'ARROW_SOURCE="%s"\n' "${arrow_source}" printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}" @@ -97,7 +112,13 @@ paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}" printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}" printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"' + printf 'PAIMON_CPP_17_NAME="%s"\n' "${paimon_archive}" + printf 'PAIMON_CPP_17_ARCHIVE_SOURCE="%s"\n' "${paimon_source}" + printf 'PAIMON_CPP_17_SOURCE="%s"\n' "${paimon_17_source}" + printf 'PAIMON_CPP_17_MD5SUM="%s"\n' "${paimon_md5}" + printf '%s\n' 'PAIMON_CPP_17_DOWNLOAD="unused"' printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n" test-fingerprint; }' + printf '%s\n' 'arrow_paimon_17_build_fingerprint() { printf "%s\n" test-17-fingerprint; }' } >"${harness}/vars.sh" exercise_interrupted_patch_set() { @@ -143,6 +164,34 @@ exercise_interrupted_patch_set ARROW "${arrow_source}" "${arrow_archive}" arrow exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}" "${paimon_archive}" paimon \ "${paimon_patches[@]}" +exercise_legacy_source_isolation() { + local index + + touch "${harness}/src/${arrow_source}/current-arrow-sentinel" + TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ + bash "${harness}/download-thirdparty.sh" ARROW_17 >/dev/null + for index in 1 2 3; do + [[ "$(<"${harness}/src/${arrow_17_source}/arrow17-${index}.txt")" == "patched" ]] || + fail "Arrow 17 patch ${index} was not applied" + done + [[ -f "${harness}/src/${arrow_source}/current-arrow-sentinel" ]] || + fail "extracting Arrow 17 modified the Arrow 24 source" + + touch "${harness}/src/${paimon_source}/current-paimon-sentinel" + TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ + bash "${harness}/download-thirdparty.sh" PAIMON_CPP_17 >/dev/null + [[ "$(<"${harness}/src/${paimon_17_source}/paimon-1.txt")" == "patched" ]] || + fail "Paimon Arrow 17 static dependency patch was not applied" + for index in 2 3; do + [[ "$(<"${harness}/src/${paimon_17_source}/paimon-${index}.txt")" == "original" ]] || + fail "Paimon Arrow 17 received an Arrow 24-only patch" + done + [[ -f "${harness}/src/${paimon_source}/current-paimon-sentinel" ]] || + fail "extracting Paimon for Arrow 17 modified the Arrow 24 source" +} + +exercise_legacy_source_isolation + exercise_generic_recovery_dispatch() { local generic="${tmpdir}/generic-recovery" local thirdparty_dir="${generic}/thirdparty" @@ -159,6 +208,8 @@ exercise_generic_recovery_dispatch() { local clean local package1 local package2 + local package3 + local package4 local extra mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ @@ -230,9 +281,10 @@ exercise_generic_recovery_dispatch() { status=$? fi [[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking its builder" - read -r flag parallel package1 package2 extra <"${args_file}" + read -r flag parallel package1 package2 package3 package4 extra <"${args_file}" [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || + "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && + "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || fail "generic recovery dispatched the wrong build package set" if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ @@ -242,9 +294,10 @@ exercise_generic_recovery_dispatch() { status=$? fi [[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before invoking its builder" - read -r flag parallel clean package1 package2 extra <"${args_file}" + read -r flag parallel clean package1 package2 package3 package4 extra <"${args_file}" [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" == "--clean" && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || + "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && + "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || fail "generic clean recovery dispatched the wrong build package set" if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ @@ -284,18 +337,24 @@ exercise_generic_recovery_dispatch # stale Arrow installation pass the shared prebuilt validation. . "${ROOT}/arrow-paimon-vars.sh" prebuilt="${tmpdir}/prebuilt" -mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" +selected_prebuilt="$(arrow_install_dir "${prebuilt}")" +mkdir -p "${selected_prebuilt}/include/arrow/util" "${selected_prebuilt}/lib64" printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \ - >"${prebuilt}/include/arrow/util/config.h" + >"${selected_prebuilt}/include/arrow/util/config.h" for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${prebuilt}/lib64/${library}" + touch "${selected_prebuilt}/lib64/${library}" done prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}" [[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || fail "focused recovery dispatches a bundled source package as a build target" +[[ "${ARROW_PAIMON_SHARED_BUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 arrow paimon_cpp" ]] || + fail "shared recovery does not cover both installed Arrow/Paimon stacks" [[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli" ]] || fail "focused recovery does not download the complete Arrow source closure" +prepare_arrow_paimon_download_packages arrow_17 paimon_cpp_17 +[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 xsimd_17 brotli" ]] || + fail "Arrow 17 build does not download its independent source closure" # A legacy prebuilt may have the old combined marker but no component markers. # Generic build.sh consumers must reject it before importing Arrow Compute. @@ -306,13 +365,13 @@ fi publish_arrow_prebuilt_marker "${prebuilt}" publish_paimon_prebuilt_marker "${prebuilt}" -rm "${prebuilt}/lib64/libarrow_compute.a" +rm "${selected_prebuilt}/lib64/libarrow_compute.a" if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fail "prebuilt validation accepted a missing Arrow Compute archive" fi -touch "${prebuilt}/lib64/libarrow_compute.a" +touch "${selected_prebuilt}/lib64/libarrow_compute.a" -printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt" +printf '%s\n' stale-arrow >"${selected_prebuilt}/arrow-build-fingerprint.txt" if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fail "Paimon-only marker update certified a stale Arrow build" fi @@ -338,5 +397,92 @@ fi publish_arrow_prebuilt_marker "${prebuilt}" arrow_paimon_prebuilt_valid "${prebuilt}" || fail "republished component markers were rejected" +select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 +[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] || + fail "recovery did not select only the missing Arrow 17 stack" + +# The legacy stack remains at the unversioned prefix and is validated +# independently. Rebuilding or invalidating either stack must not affect the +# other branch's artifacts or fingerprints. +mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" +printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_17_VERSION}" \ + >"${prebuilt}/include/arrow/util/config.h" +for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}"; do + touch "${prebuilt}/lib64/${library}" +done +publish_arrow_17_prebuilt_marker "${prebuilt}" +publish_paimon_17_prebuilt_marker "${prebuilt}" +arrow_paimon_17_prebuilt_valid "${prebuilt}" || + fail "matching Arrow 17 and Paimon artifacts were rejected" +arrow_paimon_prebuilt_valid "${prebuilt}" || + fail "publishing the Arrow 17 stack invalidated Arrow 24" +shared_arrow_paimon_prebuilt_valid "${prebuilt}" || + fail "matching shared Arrow/Paimon stacks were rejected" +select_arrow_paimon_rebuild_packages "${prebuilt}" +[[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]] || + fail "recovery rebuilt an already valid shared stack" + +invalidate_arrow_prebuilt_marker "${prebuilt}" +arrow_paimon_17_prebuilt_valid "${prebuilt}" || + fail "invalidating Arrow 24 affected the Arrow 17 stack" +select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 +[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || + fail "recovery did not isolate an invalid Arrow 24 stack" +publish_arrow_prebuilt_marker "${prebuilt}" + +invalidate_arrow_17_prebuilt_marker "${prebuilt}" +arrow_paimon_prebuilt_valid "${prebuilt}" || + fail "invalidating Arrow 17 affected the Arrow 24 stack" +select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 +[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] || + fail "recovery did not isolate an invalid Arrow 17 stack" +publish_arrow_17_prebuilt_marker "${prebuilt}" +arrow_paimon_17_prebuilt_valid "${prebuilt}" || + fail "republished Arrow 17 component markers were rejected" + +# Reinstalling one stack cleans only files owned by that stack. In particular, +# a downgrade of the legacy prefix must remove Arrow 24-only artifacts without +# deleting Paimon or unrelated thirdparty files. +cleanup_prefix="${tmpdir}/cleanup-prefix" +mkdir -p \ + "${cleanup_prefix}/include/arrow" \ + "${cleanup_prefix}/include/parquet" \ + "${cleanup_prefix}/include/paimon" \ + "${cleanup_prefix}/include/unrelated" \ + "${cleanup_prefix}/lib64/cmake/ArrowCompute" \ + "${cleanup_prefix}/lib64/cmake/Paimon" \ + "${cleanup_prefix}/lib64/pkgconfig" \ + "${cleanup_prefix}/share/arrow" \ + "${cleanup_prefix}/share/doc/arrow" +touch \ + "${cleanup_prefix}/lib64/libarrow_compute.a" \ + "${cleanup_prefix}/lib64/libparquet.a" \ + "${cleanup_prefix}/lib64/libpaimon.a" \ + "${cleanup_prefix}/lib64/libfmt_paimon.a" \ + "${cleanup_prefix}/lib64/libunrelated.a" \ + "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" \ + "${cleanup_prefix}/include/unrelated/sentinel" + +clean_arrow_artifacts_in "${cleanup_prefix}" +[[ ! -e "${cleanup_prefix}/include/arrow" && + ! -e "${cleanup_prefix}/include/parquet" && + ! -e "${cleanup_prefix}/lib64/libarrow_compute.a" && + ! -e "${cleanup_prefix}/lib64/cmake/ArrowCompute" && + ! -e "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" ]] || + fail "Arrow cleanup left stale artifacts in the selected prefix" +[[ -e "${cleanup_prefix}/include/paimon" && + -e "${cleanup_prefix}/lib64/libpaimon.a" && + -e "${cleanup_prefix}/include/unrelated/sentinel" ]] || + fail "Arrow cleanup removed another package's artifacts" + +clean_paimon_artifacts_in "${cleanup_prefix}" +[[ ! -e "${cleanup_prefix}/include/paimon" && + ! -e "${cleanup_prefix}/lib64/libpaimon.a" && + ! -e "${cleanup_prefix}/lib64/libfmt_paimon.a" && + ! -e "${cleanup_prefix}/lib64/cmake/Paimon" ]] || + fail "Paimon cleanup left stale artifacts in the selected prefix" +[[ -e "${cleanup_prefix}/lib64/libunrelated.a" && + -e "${cleanup_prefix}/include/unrelated/sentinel" ]] || + fail "Paimon cleanup removed another package's artifacts" echo "PASS" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index f8eedabe0fcbff..a41c9f39d1a63c 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -619,6 +619,7 @@ export TP_ARCHIVES=( 'CYRUS_SASL' 'LIBRDKAFKA' 'FLATBUFFERS' + 'ARROW_17' 'ARROW' 'ARROW_ADBC' 'BROTLI' @@ -651,6 +652,7 @@ export TP_ARCHIVES=( 'PDQSORT' 'TIMSORT' 'BENCHMARK' + 'XSIMD_17' 'XSIMD' 'SIMDJSON' 'NLOHMANN_JSON' @@ -674,6 +676,7 @@ export TP_ARCHIVES=( 'JINDOFS' 'JUICEFS' 'PUGIXML' + 'PAIMON_CPP_17' 'PAIMON_CPP' 'LANCE_C' ) From 4f867efda45e7815d4156db7fcef487ea64ef247 Mon Sep 17 00:00:00 2001 From: daidai Date: Sat, 8 Aug 2026 22:16:52 +0800 Subject: [PATCH 2/2] [fix](regression) Load packaged ADBC JNI library in external tests ### What problem does this PR solve? Issue Number: None Related PR: #66221 Problem Summary: External regression replaces the standard FE configuration with its own fe.conf. That override omitted arrow.adbc.driver.jni.library.path, so the ADBC connector extracted the JNI library bundled in the Maven JAR instead of loading the compatible library built and packaged by Doris thirdparty. On older TeamCity hosts, every Flight SQL catalog case then failed while initializing JniLoader because the bundled library requires newer GLIBC and GLIBCXX versions. Keep the external regression JVM options aligned with the standard FE configuration so it loads output/fe/lib/libadbc_driver_jni.so. ### Release note None ### Check List (For Author) - Test: - Manual test: verified the JDK 17 option expands arrow.adbc.driver.jni.library.path to the configured DORIS_HOME lib directory - Manual test: verified the packaged thirdparty JNI library is present - Manual test: git diff --cached --check - Behavior changed: No. This only corrects the External Regression FE startup configuration. - Does this need documentation: No --- regression-test/pipeline/external/conf/fe.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index fc818a862a1a15..c5c55f7fb807b6 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -28,7 +28,7 @@ DATE = `date +%Y%m%d-%H%M%S` JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # For jdk 17+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false -Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" ## ## the lowercase properties are read by main program.