diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 47145326a92343..ba15a5e5f809c7 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -129,24 +129,27 @@ jobs: fi tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz - # Rebuild the Arrow/Paimon stack when its inputs change or the shared - # prebuilt predates the selected Arrow version/component closure. + # Rebuild either Arrow/Paimon stack when its inputs change or the + # shared prebuilt predates the dual-version component closure. # The artifact check also covers scheduled and later BE-only builds, # where Paths Filter is skipped or reports no Arrow/Paimon changes. # shellcheck source=thirdparty/arrow-paimon-vars.sh . ./arrow-paimon-vars.sh + select_arrow_paimon_rebuild_packages installed arrow_paimon_prebuilt_is_valid=false - if arrow_paimon_prebuilt_valid installed; then + if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]]; then arrow_paimon_prebuilt_is_valid=true fi if [[ "${{ steps.filter.outputs.arrow_paimon_changes }}" == "true" || "${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then + if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]]; then + ARROW_PAIMON_REBUILD_PACKAGES=("${ARROW_PAIMON_SHARED_BUILD_PACKAGES[@]}") + fi curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ -o doris-thirdparty-source.tgz tar -zxvf doris-thirdparty-source.tgz - ./download-thirdparty.sh arrow paimon_cpp xsimd brotli export MACOSX_DEPLOYMENT_TARGET=12.0 - ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp + ./build-thirdparty.sh -j "$(nproc)" "${ARROW_PAIMON_REBUILD_PACKAGES[@]}" fi popd diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index aed51c2fb2e644..a012dba11843e2 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -156,7 +156,8 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) -set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") +set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH "Selected Arrow install prefix") +set(PAIMON_HOME "" CACHE PATH "Selected Paimon install prefix") option(ENABLE_TDE "Enable TDE feature module" OFF) set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src") option(ENABLE_TLS "Enable TLS feature module" OFF) @@ -178,8 +179,24 @@ endif() if (DEFINED ENV{ENABLE_PAIMON_CPP}) set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE) endif() -if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) +if (DEFINED ENV{ARROW_HOME}) + set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "" FORCE) +endif() +if (DEFINED ENV{PAIMON_HOME}) set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) +elseif (NOT PAIMON_HOME) + set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "" FORCE) +endif() + +if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h") + message(FATAL_ERROR "Selected Arrow installation is incomplete: ${ARROW_HOME}") +endif() +message(STATUS "ARROW_HOME is ${ARROW_HOME}") +if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon") + message(FATAL_ERROR "Selected Paimon installation is incomplete: ${PAIMON_HOME}") +endif() +if (ENABLE_PAIMON_CPP) + message(STATUS "PAIMON_HOME is ${PAIMON_HOME}") endif() set(CMAKE_SKIP_RPATH TRUE) @@ -520,6 +537,12 @@ include_directories( ${SRC_DIR}/ ) +include_directories( + BEFORE SYSTEM + ${ARROW_HOME}/include + ${PAIMON_HOME}/include +) + include_directories( SYSTEM ${COMMON_SRC_DIR} @@ -665,10 +688,10 @@ endif() set(PAIMON_FACTORY_REGISTRY_LIBS) if (ENABLE_PAIMON_CPP) - # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via - # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). - # No dual-stack selection needed — single Arrow for everything. + # The Arrow selected for this BE build includes COMPUTE/DATASET/ACERO/FILESYSTEM, + # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY + # via thirdparty.cmake. Paimon uses this same selected Arrow (no paimon_deps); + # the legacy Arrow 17 stack remains installed only for branch-4.1 consumers. # paimon_parquet_file_format depends on Arrow Dataset symbols. # Force-link it with --whole-archive so its factory registration runs. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index f3bd9867ee0fdd..1003dda92890df 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -25,10 +25,11 @@ set(COMMON_THIRDPARTY) # if arg exist noadd, not append to COMMON_THIRDPARTY variable # if arg exist libname, use libname to find library # if arg exist wholelibpath, use wholelibpath to find library +# if arg exist root, resolve the library below that install prefix function(add_thirdparty) cmake_parse_arguments(DORIS_THIRDPARTY "NOTADD;LIB64" - "LIBNAME;WHOLELIBPATH" + "LIBNAME;WHOLELIBPATH;ROOT" "" ${ARGN}) @@ -39,14 +40,20 @@ function(add_thirdparty) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME} PARENT_SCOPE) endif() + if (DORIS_THIRDPARTY_ROOT) + set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT}) + else() + set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR}) + endif() + if (DORIS_THIRDPARTY_LIB64) - set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a) + set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a) elseif (DORIS_THIRDPARTY_LIBNAME) - set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME}) + set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME}) elseif (DORIS_THIRDPARTY_WHOLELIBPATH) set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH}) else() - set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a) + set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a) endif() set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH}) endfunction() @@ -100,18 +107,18 @@ add_thirdparty(cares LIB64) add_thirdparty(address_sorting LIB64) add_thirdparty(z LIB64) -add_thirdparty(brotlicommon LIB64) -add_thirdparty(brotlidec LIB64) -add_thirdparty(brotlienc LIB64) +add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME}) add_thirdparty(zstd LIB64) -add_thirdparty(arrow LIB64) -add_thirdparty(arrow_compute LIB64) -add_thirdparty(arrow_flight LIB64) -add_thirdparty(arrow_flight_sql LIB64) -add_thirdparty(arrow_dataset LIB64) -add_thirdparty(arrow_acero LIB64) +add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME}) add_thirdparty(adbc_driver_manager LIB64) -add_thirdparty(parquet LIB64) +add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME}) # liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it # so the final linker resolves C math symbols from the system library first. add_thirdparty(lance_c LIB64 NOTADD) @@ -191,16 +198,16 @@ add_thirdparty(icudata LIB64) add_thirdparty(pugixml LIB64) if (ENABLE_PAIMON_CPP) - add_thirdparty(paimon LIB64) - add_thirdparty(paimon_parquet_file_format LIB64) - add_thirdparty(paimon_orc_file_format LIB64) - add_thirdparty(paimon_blob_file_format LIB64) - add_thirdparty(paimon_local_file_system LIB64) - add_thirdparty(paimon_file_index LIB64) - add_thirdparty(paimon_global_index LIB64) - - add_thirdparty(roaring_bitmap_paimon LIB64) - add_thirdparty(xxhash_paimon LIB64) - add_thirdparty(fmt_paimon LIB64) - add_thirdparty(tbb_paimon LIB64) + add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME}) + + add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME}) endif() diff --git a/build.sh b/build.sh index ceaa09a0779593..fdd74054241efb 100755 --- a/build.sh +++ b/build.sh @@ -518,7 +518,7 @@ rebuild_thirdparty_libraries() { build_args+=(--clean) fi bash "${build_script}" "${build_args[@]}" "$@" - if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then + if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2 exit 1 fi @@ -527,10 +527,12 @@ rebuild_thirdparty_libraries() { if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then echo "Thirdparty libraries need to be build ..." rebuild_thirdparty_libraries true -elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] && - ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." - rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}" +elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then + select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed" + if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then + echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." + rebuild_thirdparty_libraries false "${ARROW_PAIMON_REBUILD_PACKAGES[@]}" + fi fi update_submodule() { diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index fc818a862a1a15..c5c55f7fb807b6 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -28,7 +28,7 @@ DATE = `date +%Y%m%d-%H%M%S` JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # For jdk 17+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false -Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" ## ## the lowercase properties are read by main program. diff --git a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy index 9d27447cb7067d..c1f47c4a399eee 100644 --- a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy +++ b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy @@ -227,7 +227,12 @@ suite("test_adbc_type_mapping", "p0,external") { // ---- floating point ---- qt_desc_float """DESC ${catalogName}.${dbName}.t_float""" - qt_select_float """SELECT id, c_float, c_double FROM ${catalogName}.${dbName}.t_float ORDER BY id""" + // Render the double with the historical 16-digit baseline so DBL_MAX is not reparsed by the + // test client. The bit-for-bit comparison below still validates the DOUBLE value inside Doris. + qt_select_float """ + SELECT id, c_float, format('{:.16g}', c_double) AS c_double + FROM ${catalogName}.${dbName}.t_float ORDER BY id + """ // Compared INSIDE Doris rather than through sameAsSource, and not because ADBC needs the // help: the fixture's DBL_MAX cannot make the trip to the test client at all. Doris renders a diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh index 1091c4fc84aa56..828dcc70da8f08 100644 --- a/thirdparty/arrow-paimon-vars.sh +++ b/thirdparty/arrow-paimon-vars.sh @@ -21,12 +21,23 @@ # Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can # distinguish this stack from unrelated thirdparty changes. -# arrow +# Arrow 24 is installed in a versioned prefix. The unversioned install prefix is +# deliberately reserved for Arrow 17 so the shared thirdparty package remains +# consumable by branch-4.1. ARROW_VERSION="24.0.0" ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz" ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz" ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}" ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" +ARROW_INSTALL_SUBDIR="arrow-${ARROW_VERSION}" + +# Arrow 17 compatibility stack for branch-4.1. Keep these variables separate +# from ARROW_* so master can build both versions from one source bundle. +ARROW_17_VERSION="17.0.0" +ARROW_17_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_17_VERSION}.tar.gz" +ARROW_17_NAME="apache-arrow-${ARROW_17_VERSION}.tar.gz" +ARROW_17_SOURCE="arrow-apache-arrow-${ARROW_17_VERSION}" +ARROW_17_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0" # Arrow bundled dependencies BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" @@ -39,16 +50,36 @@ XSIMD_NAME="14.0.0.tar.gz" XSIMD_SOURCE=xsimd-14.0.0 XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" +XSIMD_17_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz" +XSIMD_17_NAME="13.0.0.tar.gz" +XSIMD_17_SOURCE=xsimd-13.0.0 +XSIMD_17_MD5SUM="c661deb91836e82d3070f81032014fe6" + # paimon-cpp PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" +# Both Paimon variants use the same archive, but they need independent source +# trees because only the Arrow 24 tree receives the API compatibility patches. +PAIMON_CPP_17_DOWNLOAD="${PAIMON_CPP_DOWNLOAD}" +PAIMON_CPP_17_NAME="${PAIMON_CPP_NAME}" +PAIMON_CPP_17_ARCHIVE_SOURCE="${PAIMON_CPP_SOURCE}" +PAIMON_CPP_17_SOURCE="${PAIMON_CPP_SOURCE}-arrow-17" +PAIMON_CPP_17_MD5SUM="${PAIMON_CPP_MD5SUM}" + # Arrow consumes xsimd and Brotli as bundled source archives, but neither is a # build target in the focused Arrow/Paimon recovery path. +ARROW_PAIMON_17_BUILD_PACKAGES=(arrow_17 paimon_cpp_17) ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp) +ARROW_PAIMON_SHARED_BUILD_PACKAGES=( + "${ARROW_PAIMON_17_BUILD_PACKAGES[@]}" + "${ARROW_PAIMON_BUILD_PACKAGES[@]}" +) +ARROW_PAIMON_REBUILD_PACKAGES=() ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli) +ARROW_17_BUNDLED_SOURCE_PACKAGES=(xsimd_17 brotli) ARROW_PAIMON_DOWNLOAD_PACKAGES=() prepare_arrow_paimon_download_packages() { @@ -57,18 +88,25 @@ prepare_arrow_paimon_download_packages() { local package local source_package local arrow_requested=false + local arrow_17_requested=false local source_requested for package in "$@"; do if [[ "${package}" == "arrow" ]]; then arrow_requested=true - break + elif [[ "${package}" == "arrow_17" ]]; then + arrow_17_requested=true fi done - if [[ "${arrow_requested}" != "true" ]]; then - return + + local bundled_source_packages=() + if [[ "${arrow_requested}" == "true" ]]; then + bundled_source_packages+=("${ARROW_BUNDLED_SOURCE_PACKAGES[@]}") + fi + if [[ "${arrow_17_requested}" == "true" ]]; then + bundled_source_packages+=("${ARROW_17_BUNDLED_SOURCE_PACKAGES[@]}") fi - for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do + for source_package in "${bundled_source_packages[@]}"; do source_requested=false for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do if [[ "${package}" == "${source_package}" ]]; then @@ -82,6 +120,10 @@ prepare_arrow_paimon_download_packages() { done } +arrow_install_dir() { + printf '%s/%s\n' "$1" "${ARROW_INSTALL_SUBDIR}" +} + # Identify the checked-in source, patch, and build inputs selected for Arrow. # Arrow and Paimon publish separate installed markers so a package-only build # cannot certify a component that it did not rebuild. @@ -116,7 +158,9 @@ paimon_build_fingerprint() { download-thirdparty.sh \ build-thirdparty.sh \ paimon-cpp-cache.cmake \ - patches/paimon-cpp-*.patch + patches/paimon-cpp-buildutils-static-deps.patch \ + patches/paimon-cpp-arrow-24-compatibility.patch \ + patches/paimon-cpp-arrow-24-compute.patch } | git hash-object --stdin ) } @@ -130,6 +174,49 @@ arrow_paimon_build_fingerprint() { } | git hash-object --stdin } +arrow_17_build_fingerprint() { + local vars_dir + vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ( + cd "${vars_dir}" || return 1 + LC_ALL=C + git hash-object \ + ../env.sh \ + arrow-paimon-vars.sh \ + vars.sh \ + download-thirdparty.sh \ + build-thirdparty.sh \ + patches/apache-arrow-"${ARROW_17_VERSION}"-*.patch | + git hash-object --stdin + ) +} + +paimon_17_build_fingerprint() { + local vars_dir + vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ( + cd "${vars_dir}" || return 1 + LC_ALL=C + { + arrow_17_build_fingerprint + git hash-object \ + arrow-paimon-vars.sh \ + vars.sh \ + download-thirdparty.sh \ + build-thirdparty.sh \ + paimon-cpp-cache.cmake \ + patches/paimon-cpp-buildutils-static-deps.patch + } | git hash-object --stdin + ) +} + +arrow_paimon_17_build_fingerprint() { + { + arrow_17_build_fingerprint + paimon_17_build_fingerprint + } | git hash-object --stdin +} + ARROW_REQUIRED_LIBRARIES=( libbrotlicommon.a libbrotlidec.a @@ -144,6 +231,19 @@ ARROW_REQUIRED_LIBRARIES=( libparquet.a ) +ARROW_17_REQUIRED_LIBRARIES=( + libbrotlicommon.a + libbrotlidec.a + libbrotlienc.a + libarrow.a + libarrow_flight.a + libarrow_flight_sql.a + libarrow_dataset.a + libarrow_acero.a + libarrow_bundled_dependencies.a + libparquet.a +) + PAIMON_REQUIRED_LIBRARIES=( libpaimon.a libpaimon_parquet_file_format.a @@ -158,13 +258,64 @@ PAIMON_REQUIRED_LIBRARIES=( libtbb_paimon.a ) +# Remove only artifacts owned by the selected Arrow/Paimon stack before an +# install. This matters for the legacy prefix: installing Arrow 17 over an +# existing Arrow 24 prefix must not leave Arrow 24-only headers or libraries +# behind and turn it into a mixed, internally inconsistent SDK. +clean_arrow_artifacts_in() { + local install_dir="$1" + : "${install_dir:?Arrow install directory must be set}" + + rm -rf -- \ + "${install_dir}/include/arrow" \ + "${install_dir}/include/parquet" \ + "${install_dir}/share/arrow" \ + "${install_dir}/share/doc/arrow" + + ( + shopt -s nullglob + local generated_artifacts=( + "${install_dir}/lib64"/libarrow* + "${install_dir}/lib64"/libparquet* + "${install_dir}/lib64/cmake"/Arrow* + "${install_dir}/lib64/cmake"/Parquet + "${install_dir}/lib64/pkgconfig"/arrow*.pc + "${install_dir}/lib64/pkgconfig"/parquet.pc + ) + rm -rf -- "${generated_artifacts[@]}" + ) +} + +clean_paimon_artifacts_in() { + local install_dir="$1" + : "${install_dir:?Paimon install directory must be set}" + + rm -rf -- \ + "${install_dir}/include/paimon" \ + "${install_dir}/lib64/cmake/Paimon" \ + "${install_dir}/paimon-cpp" + + ( + shopt -s nullglob + local generated_artifacts=( + "${install_dir}/lib64"/libpaimon* + "${install_dir}/lib64"/libroaring_bitmap_paimon.* + "${install_dir}/lib64"/libxxhash_paimon.* + "${install_dir}/lib64"/libfmt_paimon.* + "${install_dir}/lib64"/libtbb_paimon.* + ) + rm -rf -- "${generated_artifacts[@]}" + ) +} + ARROW_PAIMON_REQUIRED_LIBRARIES=( "${ARROW_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}" ) arrow_artifacts_valid() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" local installed_arrow_version local library @@ -191,7 +342,35 @@ arrow_artifacts_valid() { return 0 } -paimon_artifacts_valid() { +arrow_17_artifacts_valid() { + local install_dir="$1" + local installed_arrow_version + local library + + if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then + echo "Missing installed Arrow 17 version header" >&2 + return 1 + fi + installed_arrow_version="$( + awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" { + gsub(/"/, "", $3); print $3; exit + }' "${install_dir}/include/arrow/util/config.h" + )" + if [[ "${installed_arrow_version}" != "${ARROW_17_VERSION}" ]]; then + echo "Installed legacy Arrow version ${installed_arrow_version} does not match ${ARROW_17_VERSION}" >&2 + return 1 + fi + + for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"; do + if [[ ! -f "${install_dir}/lib64/${library}" ]]; then + echo "Missing Arrow 17 library: ${library}" >&2 + return 1 + fi + done + return 0 +} + +paimon_artifacts_valid_in() { local install_dir="$1" local library @@ -204,8 +383,13 @@ paimon_artifacts_valid() { return 0 } +paimon_artifacts_valid() { + paimon_artifacts_valid_in "$(arrow_install_dir "$1")" +} + arrow_prebuilt_valid() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt" local expected_fingerprint local installed_fingerprint @@ -220,11 +404,12 @@ arrow_prebuilt_valid() { echo "Arrow build fingerprint does not match selected inputs" >&2 return 1 fi - arrow_artifacts_valid "${install_dir}" + arrow_artifacts_valid "$1" } paimon_prebuilt_valid() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt" local expected_fingerprint local installed_fingerprint @@ -239,7 +424,7 @@ paimon_prebuilt_valid() { echo "Paimon build fingerprint does not match selected inputs" >&2 return 1 fi - paimon_artifacts_valid "${install_dir}" + paimon_artifacts_valid "$1" } arrow_paimon_prebuilt_valid() { @@ -248,26 +433,32 @@ arrow_paimon_prebuilt_valid() { } invalidate_arrow_prebuilt_marker() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" + mkdir -p "${install_dir}" rm -f "${install_dir}/arrow-build-fingerprint.txt" \ "${install_dir}/arrow-paimon-build-fingerprint.txt" } publish_arrow_prebuilt_marker() { - local install_dir="$1" - arrow_artifacts_valid "${install_dir}" + local install_dir + install_dir="$(arrow_install_dir "$1")" + arrow_artifacts_valid "$1" arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt" } invalidate_paimon_prebuilt_marker() { - local install_dir="$1" + local install_dir + install_dir="$(arrow_install_dir "$1")" + mkdir -p "${install_dir}" rm -f "${install_dir}/paimon-build-fingerprint.txt" \ "${install_dir}/arrow-paimon-build-fingerprint.txt" } publish_paimon_prebuilt_marker() { - local install_dir="$1" - paimon_artifacts_valid "${install_dir}" + local install_dir + install_dir="$(arrow_install_dir "$1")" + paimon_artifacts_valid "$1" paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt" } @@ -278,3 +469,93 @@ require_arrow_prebuilt_for_paimon() { return 1 fi } + +invalidate_arrow_17_prebuilt_marker() { + local install_dir="$1" + rm -f "${install_dir}/arrow-17-build-fingerprint.txt" \ + "${install_dir}/arrow-paimon-17-build-fingerprint.txt" +} + +publish_arrow_17_prebuilt_marker() { + local install_dir="$1" + arrow_17_artifacts_valid "${install_dir}" + arrow_17_build_fingerprint >"${install_dir}/arrow-17-build-fingerprint.txt" +} + +arrow_17_prebuilt_valid() { + local install_dir="$1" + local fingerprint_mark="${install_dir}/arrow-17-build-fingerprint.txt" + local expected_fingerprint + + if [[ ! -f "${fingerprint_mark}" ]]; then + echo "Missing Arrow 17 build fingerprint: ${fingerprint_mark}" >&2 + return 1 + fi + expected_fingerprint="$(arrow_17_build_fingerprint)" + if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then + echo "Arrow 17 build fingerprint does not match selected inputs" >&2 + return 1 + fi + arrow_17_artifacts_valid "${install_dir}" +} + +require_arrow_17_prebuilt_for_paimon() { + local install_dir="$1" + if ! arrow_17_prebuilt_valid "${install_dir}"; then + echo "Paimon for branch-4.1 requires Arrow 17 to be built first" >&2 + return 1 + fi +} + +invalidate_paimon_17_prebuilt_marker() { + local install_dir="$1" + rm -f "${install_dir}/paimon-arrow-17-build-fingerprint.txt" \ + "${install_dir}/arrow-paimon-17-build-fingerprint.txt" +} + +publish_paimon_17_prebuilt_marker() { + local install_dir="$1" + paimon_artifacts_valid_in "${install_dir}" + paimon_17_build_fingerprint >"${install_dir}/paimon-arrow-17-build-fingerprint.txt" +} + +paimon_17_prebuilt_valid() { + local install_dir="$1" + local fingerprint_mark="${install_dir}/paimon-arrow-17-build-fingerprint.txt" + local expected_fingerprint + + if [[ ! -f "${fingerprint_mark}" ]]; then + echo "Missing Paimon Arrow 17 build fingerprint: ${fingerprint_mark}" >&2 + return 1 + fi + expected_fingerprint="$(paimon_17_build_fingerprint)" + if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then + echo "Paimon Arrow 17 build fingerprint does not match selected inputs" >&2 + return 1 + fi + paimon_artifacts_valid_in "${install_dir}" +} + +arrow_paimon_17_prebuilt_valid() { + local install_dir="$1" + arrow_17_prebuilt_valid "${install_dir}" && + paimon_17_prebuilt_valid "${install_dir}" +} + +shared_arrow_paimon_prebuilt_valid() { + local install_dir="$1" + arrow_paimon_17_prebuilt_valid "${install_dir}" && + arrow_paimon_prebuilt_valid "${install_dir}" +} + +select_arrow_paimon_rebuild_packages() { + local install_dir="$1" + ARROW_PAIMON_REBUILD_PACKAGES=() + + if ! arrow_paimon_17_prebuilt_valid "${install_dir}"; then + ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_17_BUILD_PACKAGES[@]}") + fi + if ! arrow_paimon_prebuilt_valid "${install_dir}"; then + ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_BUILD_PACKAGES[@]}") + fi +} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index a6544a57ac2fcd..178ef8f204f63b 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -313,20 +313,26 @@ else echo "Do not strip thirdparty libraries" fi -strip_lib() { +strip_lib_at() { + local install_dir="$1" + local library="$2" if [[ "${STRIP_TP_LIB}" = "ON" ]]; then - if [[ -z $1 ]]; then + if [[ -z "${library}" ]]; then echo "Must specify the library to be stripped." exit 1 fi - if [[ ! -f "${TP_LIB_DIR}/$1" ]]; then - echo "Library to be stripped (${TP_LIB_DIR}/$1) does not exist." + if [[ ! -f "${install_dir}/lib/${library}" ]]; then + echo "Library to be stripped (${install_dir}/lib/${library}) does not exist." exit 1 fi - strip --strip-debug --strip-unneeded "${TP_LIB_DIR}/$1" + strip --strip-debug --strip-unneeded "${install_dir}/lib/${library}" fi } +strip_lib() { + strip_lib_at "${TP_INSTALL_DIR}" "$1" +} + #libbacktrace build_libbacktrace() { check_if_source_exist "${LIBBACKTRACE_SOURCE}" @@ -1062,11 +1068,18 @@ build_grpc() { # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake } -# arrow -build_arrow() { - check_if_source_exist "${ARROW_SOURCE}" - invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp" +# Arrow 17 is installed in the legacy unversioned prefix for branch-4.1, while +# Arrow 24 is installed in a versioned prefix selected by master. +build_arrow_stack() { + local arrow_source="$1" + local xsimd_archive="$2" + local install_dir="$3" + local has_separate_compute_archive="$4" + + check_if_source_exist "${arrow_source}" + mkdir -p "${install_dir}/lib64" + ln -sfn lib64 "${install_dir}/lib" + cd "${TP_SOURCE_DIR}/${arrow_source}/cpp" mkdir -p release cd release @@ -1079,7 +1092,7 @@ build_arrow() { export ARROW_Thrift_URL="${TP_SOURCE_DIR}/${THRIFT_NAME}" export ARROW_SNAPPY_URL="${TP_SOURCE_DIR}/${SNAPPY_NAME}" export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}" - export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}" + export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${xsimd_archive}" export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}" export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}" export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}" @@ -1101,7 +1114,7 @@ build_arrow() { -DARROW_FILESYSTEM=ON \ -DARROW_DATASET=ON \ -DARROW_ACERO=ON \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DCMAKE_INSTALL_LIBDIR=lib64 \ -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_WITH_GRPC=ON \ @@ -1144,15 +1157,31 @@ build_arrow() { "${BUILD_SYSTEM}" install #copy dep libs - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a "${TP_INSTALL_DIR}/lib64/libbrotlienc.a" - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${TP_INSTALL_DIR}/lib64/libbrotlidec.a" - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${TP_INSTALL_DIR}/lib64/libbrotlicommon.a" - strip_lib libarrow.a - strip_lib libarrow_compute.a - strip_lib libparquet.a - strip_lib libarrow_dataset.a - strip_lib libarrow_acero.a + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a "${install_dir}/lib64/libbrotlienc.a" + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${install_dir}/lib64/libbrotlidec.a" + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${install_dir}/lib64/libbrotlicommon.a" + strip_lib_at "${install_dir}" libarrow.a + if [[ "${has_separate_compute_archive}" == "true" ]]; then + strip_lib_at "${install_dir}" libarrow_compute.a + fi + strip_lib_at "${install_dir}" libparquet.a + strip_lib_at "${install_dir}" libarrow_dataset.a + strip_lib_at "${install_dir}" libarrow_acero.a +} +build_arrow_17() { + invalidate_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}" + clean_arrow_artifacts_in "${TP_INSTALL_DIR}" + build_arrow_stack "${ARROW_17_SOURCE}" "${XSIMD_17_NAME}" "${TP_INSTALL_DIR}" false + publish_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}" +} + +build_arrow() { + local install_dir + install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")" + invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" + clean_arrow_artifacts_in "${install_dir}" + build_arrow_stack "${ARROW_SOURCE}" "${XSIMD_NAME}" "${install_dir}" true publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}" } @@ -2131,12 +2160,18 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } -# paimon-cpp -build_paimon_cpp() { - check_if_source_exist "${PAIMON_CPP_SOURCE}" - require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" +# Build each Paimon variant against the matching Arrow prefix and install it +# beside that Arrow version. Arrow types cross Paimon's public C++ boundary, so +# mixing the two versions is not ABI-safe. +build_paimon_cpp_stack() { + local paimon_source="$1" + local arrow_install_dir="$2" + local install_dir="$3" + + check_if_source_exist "${paimon_source}" + mkdir -p "${install_dir}/lib64" + ln -sfn lib64 "${install_dir}/lib" + cd "${TP_SOURCE_DIR}/${paimon_source}" rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" @@ -2148,12 +2183,13 @@ build_paimon_cpp() { paimon_linker_flags="${paimon_linker_flags} -lunwind" fi + PAIMON_ARROW_INSTALL_DIR="${arrow_install_dir}" \ CXXFLAGS="-Wno-nontrivial-memcall" \ "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \ -G "${GENERATOR}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DPAIMON_BUILD_SHARED=OFF \ -DPAIMON_BUILD_STATIC=ON \ -DPAIMON_BUILD_TESTS=OFF \ @@ -2177,7 +2213,7 @@ build_paimon_cpp() { # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps # directory is not needed. When building its own Arrow (legacy), copy # arrow artefacts into an isolated directory to avoid clashing with Doris. - local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps" + local paimon_deps_dir="${install_dir}/paimon-cpp/lib64/paimon_deps" if [ -d "arrow_ep-install/lib" ]; then mkdir -p "${paimon_deps_dir}" for paimon_arrow_dep in \ @@ -2197,25 +2233,42 @@ build_paimon_cpp() { # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap if [ -f "release/libroaring_bitmap.a" ]; then - cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a" + cp -v "release/libroaring_bitmap.a" "${install_dir}/lib64/libroaring_bitmap_paimon.a" fi # Install xxhash, renamed to avoid conflict with Doris's xxhash if [ -f "release/libxxhash.a" ]; then - cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a" + cp -v "release/libxxhash.a" "${install_dir}/lib64/libxxhash_paimon.a" fi # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7) if [ -f "fmt_ep-install/lib/libfmt.a" ]; then - cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a" + cp -v "fmt_ep-install/lib/libfmt.a" "${install_dir}/lib64/libfmt_paimon.a" fi # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb) if [ -f "tbb_ep-install/lib/libtbb.a" ]; then - cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a" + cp -v "tbb_ep-install/lib/libtbb.a" "${install_dir}/lib64/libtbb_paimon.a" fi echo "Paimon-cpp internal dependencies installed successfully" +} + +build_paimon_cpp_17() { + require_arrow_17_prebuilt_for_paimon "${TP_INSTALL_DIR}" + invalidate_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}" + clean_paimon_artifacts_in "${TP_INSTALL_DIR}" + build_paimon_cpp_stack "${PAIMON_CPP_17_SOURCE}" "${TP_INSTALL_DIR}" "${TP_INSTALL_DIR}" + publish_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}" +} + +build_paimon_cpp() { + local install_dir + install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")" + require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" + invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" + clean_paimon_artifacts_in "${install_dir}" + build_paimon_cpp_stack "${PAIMON_CPP_SOURCE}" "${install_dir}" "${install_dir}" publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}" } @@ -2320,6 +2373,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then orc cares grpc # after cares, protobuf + arrow_17 arrow arrow_adbc lance_c @@ -2357,6 +2411,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then brotli icu pugixml + paimon_cpp_17 paimon_cpp ) if [[ "$(uname -s)" == 'Darwin' ]]; then @@ -2412,6 +2467,7 @@ cleanup_package_source() { cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;; librdkafka) src_var="LIBRDKAFKA_SOURCE" ;; flatbuffers) src_var="FLATBUFFERS_SOURCE" ;; + arrow_17) src_var="ARROW_17_SOURCE" ;; arrow) src_var="ARROW_SOURCE" ;; arrow_adbc) # arrow_adbc also unpacks the prebuilt flightsql driver, clean both @@ -2465,6 +2521,7 @@ cleanup_package_source() { jindofs) src_var="JINDOFS_SOURCE" ;; juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; + paimon_cpp_17) src_var="PAIMON_CPP_17_SOURCE" ;; paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index ea9436be2dec72..bf5fd05c496581 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -238,13 +238,24 @@ for TP_ARCH in "${TP_ARCHIVES[@]}"; do fi NAME="${TP_ARCH}_NAME" SOURCE="${TP_ARCH}_SOURCE" + ARCHIVE_SOURCE_VAR="${TP_ARCH}_ARCHIVE_SOURCE" + ARCHIVE_SOURCE="${!ARCHIVE_SOURCE_VAR}" if [[ -z "${!SOURCE}" ]]; then continue fi if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then - if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then + if [[ -n "${ARCHIVE_SOURCE}" && "${ARCHIVE_SOURCE}" != "${!SOURCE}" ]]; then + alias_unpack_dir="$(mktemp -d "${TP_SOURCE_DIR}/.unpack-${TP_ARCH}.XXXXXX")" + if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${alias_unpack_dir}"; then + echo "Failed to untar ${!NAME} for ${!SOURCE}" + rm -rf "${alias_unpack_dir}" + exit 1 + fi + mv "${alias_unpack_dir}/${ARCHIVE_SOURCE}" "${TP_SOURCE_DIR}/${!SOURCE}" + rm -rf "${alias_unpack_dir}" + elif [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then echo "${TP_SOURCE_DIR}/${!NAME}" echo "${TP_SOURCE_DIR}/${!SOURCE}" if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then @@ -336,17 +347,31 @@ echo "===== Patching thirdparty archives..." PATCHED_MARK="patched_mark" ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint" ARROW_PAIMON_BUILD_FINGERPRINT="" +ARROW_PAIMON_17_BUILD_FINGERPRINT="" if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " || " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)" fi +if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " || + " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then + ARROW_PAIMON_17_BUILD_FINGERPRINT="$(arrow_paimon_17_build_fingerprint)" +fi reset_arrow_paimon_source() { local archive_name="$1" local source_name="$2" + local archive_source="${3:-${source_name}}" echo "Resetting ${source_name} because its patch state is incomplete or stale" rm -rf "${TP_SOURCE_DIR:?}/${source_name}" - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" + if [[ "${archive_source}" == "${source_name}" ]]; then + "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" + else + local alias_unpack_dir + alias_unpack_dir="$(mktemp -d "${TP_SOURCE_DIR}/.reset-${source_name}.XXXXXX")" + "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${alias_unpack_dir}" + mv "${alias_unpack_dir}/${archive_source}" "${TP_SOURCE_DIR}/${source_name}" + rm -rf "${alias_unpack_dir}" + fi } # glog patch @@ -442,6 +467,26 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ROCKSDB " ]]; then echo "Finished patching ${ROCKSDB_SOURCE}" fi +# Keep the Arrow 17 source used by branch-4.1 independently patched from the +# Arrow 24 source selected by master. +if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ]]; then + arrow_17_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${PATCHED_MARK}" && + -f "${arrow_17_fingerprint_mark}" ]] || + [[ "$(<"${arrow_17_fingerprint_mark}")" != "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then + reset_arrow_paimon_source "${ARROW_17_NAME}" "${ARROW_17_SOURCE}" + cd "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch" + touch "${PATCHED_MARK}" + printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \ + >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + cd - + fi + echo "Finished patching ${ARROW_17_SOURCE}" +fi + # arrow patch is used to get the raw orc reader for filter prune. if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then @@ -761,6 +806,24 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi +# Keep the Arrow 17 Paimon source free of the Arrow 24 API and Compute patches. +if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then + paimon_17_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${PATCHED_MARK}" && + -f "${paimon_17_fingerprint_mark}" ]] || + [[ "$(<"${paimon_17_fingerprint_mark}")" != "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then + reset_arrow_paimon_source "${PAIMON_CPP_17_NAME}" "${PAIMON_CPP_17_SOURCE}" \ + "${PAIMON_CPP_17_ARCHIVE_SOURCE}" + cd "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}" + patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" + touch "${PATCHED_MARK}" + printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \ + >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" + cd - + fi + echo "Finished patching ${PAIMON_CPP_17_SOURCE}" +fi + # patch paimon-cpp if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake index 6ab6049c59b0d5..a12540e5d3ed4e 100644 --- a/thirdparty/paimon-cpp-cache.cmake +++ b/thirdparty/paimon-cpp-cache.cmake @@ -21,12 +21,17 @@ # Get the Doris thirdparty installation directory from environment set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory") +set(DORIS_ARROW_DIR "$ENV{PAIMON_ARROW_INSTALL_DIR}" CACHE PATH "Selected Doris Arrow install directory") if(NOT DORIS_THIRDPARTY_DIR) message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set") endif() +if(NOT DORIS_ARROW_DIR) + message(FATAL_ERROR "PAIMON_ARROW_INSTALL_DIR environment variable must be set") +endif() message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}") +message(STATUS "Using selected Arrow libraries from: ${DORIS_ARROW_DIR}") # Set CMAKE_PREFIX_PATH to help find_package locate our libraries set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package") @@ -64,21 +69,21 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") # ============================================================================ # ============================================================================ -# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp -# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). +# Arrow - Reuse the version selected by the caller. Both installed Arrow stacks +# include DATASET/ACERO/FILESYSTEM; Arrow 24 also has a separate Compute archive. # ============================================================================ set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") -set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory") +set(DORIS_ARROW_LIB64_DIR "${DORIS_ARROW_DIR}/lib64" CACHE PATH "Selected Arrow lib64 directory") +set(DORIS_ARROW_INCLUDE_DIR "${DORIS_ARROW_DIR}/include" CACHE PATH "Selected Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") -set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") -set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") -set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") -set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") -set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") +set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_ARROW_INCLUDE_DIR}" CACHE PATH "Arrow include directory") +set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") +set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") +set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") +set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") +set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_ARROW_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") +set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") # Protobuf, Thrift - still built separately by paimon-cpp diff --git a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch new file mode 100644 index 00000000000000..5a75424756671d --- /dev/null +++ b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch @@ -0,0 +1,98 @@ +diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc +--- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:23:23.651831424 +0800 ++++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:28:36.855281965 +0800 +@@ -178,7 +178,8 @@ + + // The user is explicitly asking for Impala int96 encoding, there is no + // logical type. +- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { ++ if (arrow_properties.force_write_int96_timestamps() || ++ (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO)) { + *physical_type = ParquetType::INT96; + return Status::OK(); + } +diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h +--- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h 2026-03-27 01:23:23.643831362 +0800 ++++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27 01:27:47.717897537 +0800 +@@ -980,6 +980,7 @@ + public: + Builder() + : write_timestamps_as_int96_(false), ++ force_write_int96_timestamps_(false), + coerce_timestamps_enabled_(false), + coerce_timestamps_unit_(::arrow::TimeUnit::SECOND), + truncated_timestamps_allowed_(false), +@@ -1005,6 +1006,21 @@ + return this; + } + ++ /// \brief Force writing legacy int96 timestamps. ++ /// ++ /// This bypasses unit-based guards and writes INT96 whenever timestamp ++ /// metadata is resolved. ++ Builder* enable_force_write_int96_timestamps() { ++ force_write_int96_timestamps_ = true; ++ return this; ++ } ++ ++ /// \brief Disable forcing legacy int96 timestamps (default). ++ Builder* disable_force_write_int96_timestamps() { ++ force_write_int96_timestamps_ = false; ++ return this; ++ } ++ + /// \brief Coerce all timestamps to the specified time unit. + /// \param unit time unit to truncate to. + /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds. +@@ -1085,7 +1101,8 @@ + /// Create the final properties. + std::shared_ptr build() { + return std::shared_ptr(new ArrowWriterProperties( +- write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_, ++ write_timestamps_as_int96_, force_write_int96_timestamps_, ++ coerce_timestamps_enabled_, coerce_timestamps_unit_, + truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, + engine_version_, use_threads_, executor_)); + } +@@ -1093,6 +1110,8 @@ + private: + bool write_timestamps_as_int96_; + ++ bool force_write_int96_timestamps_; ++ + bool coerce_timestamps_enabled_; + ::arrow::TimeUnit::type coerce_timestamps_unit_; + bool truncated_timestamps_allowed_; +@@ -1107,6 +1126,8 @@ + + bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; } + ++ bool force_write_int96_timestamps() const { return force_write_int96_timestamps_; } ++ + bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; } + ::arrow::TimeUnit::type coerce_timestamps_unit() const { + return coerce_timestamps_unit_; +@@ -1138,6 +1159,7 @@ + + private: + explicit ArrowWriterProperties(bool write_nanos_as_int96, ++ bool force_write_int96_timestamps, + bool coerce_timestamps_enabled, + ::arrow::TimeUnit::type coerce_timestamps_unit, + bool truncated_timestamps_allowed, bool store_schema, +@@ -1145,6 +1167,7 @@ + EngineVersion engine_version, bool use_threads, + ::arrow::internal::Executor* executor) + : write_timestamps_as_int96_(write_nanos_as_int96), ++ force_write_int96_timestamps_(force_write_int96_timestamps), + coerce_timestamps_enabled_(coerce_timestamps_enabled), + coerce_timestamps_unit_(coerce_timestamps_unit), + truncated_timestamps_allowed_(truncated_timestamps_allowed), +@@ -1155,6 +1178,7 @@ + executor_(executor) {} + + const bool write_timestamps_as_int96_; ++ const bool force_write_int96_timestamps_; + const bool coerce_timestamps_enabled_; + const ::arrow::TimeUnit::type coerce_timestamps_unit_; + const bool truncated_timestamps_allowed_; diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch new file mode 100644 index 00000000000000..a983818413a01c --- /dev/null +++ b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch @@ -0,0 +1,84 @@ +--- a/cpp/src/parquet/column_reader.cc ++++ b/cpp/src/parquet/column_reader.cc +@@ -30,0 +31,2 @@ ++ ++#include +@@ -268,0 +269 @@ ++ compression_codec_(codec), +@@ -279 +282,7 @@ +- decompressor_ = GetCodec(codec); ++ if (compression_codec_ == Compression::LZO) { ++ if (lzo_init() != LZO_E_OK) { ++ throw ParquetException("Failed to initialize LZO codec"); ++ } ++ } else { ++ decompressor_ = GetCodec(codec); ++ } +@@ -315,0 +325 @@ ++ Compression::type compression_codec_; +@@ -585 +595 @@ +- if (decompressor_ == nullptr) { ++ if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) { +@@ -601,0 +612,61 @@ ++ if (compression_codec_ == Compression::LZO) { ++ const uint8_t* input = page_buffer->data() + levels_byte_len; ++ const uint8_t* const input_end = page_buffer->data() + compressed_len; ++ uint8_t* output = decompression_buffer_->mutable_data() + levels_byte_len; ++ uint8_t* const output_end = decompression_buffer_->mutable_data() + uncompressed_len; ++ ++ auto load_big_endian_u32 = [](const uint8_t* data) { ++ return (static_cast(data[0]) << 24) | ++ (static_cast(data[1]) << 16) | ++ (static_cast(data[2]) << 8) | static_cast(data[3]); ++ }; ++ ++ while (input < input_end) { ++ if (input_end - input < 4) { ++ throw ParquetException("LZO page decompression failed: truncated large block length"); ++ } ++ ++ uint32_t large_block_uncompressed_len = load_big_endian_u32(input); ++ input += 4; ++ if (static_cast(output_end - output) < large_block_uncompressed_len) { ++ throw ParquetException("LZO page decompression failed: output buffer too small"); ++ } ++ ++ while (large_block_uncompressed_len > 0) { ++ if (input_end - input < 4) { ++ throw ParquetException("LZO page decompression failed: truncated small block length"); ++ } ++ ++ uint32_t small_block_compressed_len = load_big_endian_u32(input); ++ input += 4; ++ if (static_cast(input_end - input) < small_block_compressed_len) { ++ throw ParquetException("LZO page decompression failed: truncated small block data"); ++ } ++ ++ auto small_block_uncompressed_len = ++ static_cast(large_block_uncompressed_len); ++ const int result = ++ lzo1x_decompress_safe(input, static_cast(small_block_compressed_len), ++ output, &small_block_uncompressed_len, nullptr); ++ if (result != LZO_E_OK) { ++ throw ParquetException("LZO page decompression failed, error: " + ++ std::to_string(result)); ++ } ++ if (small_block_uncompressed_len > large_block_uncompressed_len) { ++ throw ParquetException("LZO page decompression failed: invalid small block size"); ++ } ++ ++ input += small_block_compressed_len; ++ output += small_block_uncompressed_len; ++ large_block_uncompressed_len -= small_block_uncompressed_len; ++ } ++ } ++ if (output != output_end) { ++ throw ParquetException("Page didn't decompress to expected size, expected: " + ++ std::to_string(uncompressed_len - levels_byte_len) + ", but got:" + ++ std::to_string(output - (decompression_buffer_->mutable_data() + ++ levels_byte_len))); ++ } ++ ++ return decompression_buffer_; ++ } ++ diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch new file mode 100644 index 00000000000000..4e53117b79b65b --- /dev/null +++ b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch @@ -0,0 +1,224 @@ +diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc +index ec3890a41f..943f69bb6c 100644 +--- a/cpp/src/parquet/arrow/schema.cc ++++ b/cpp/src/parquet/arrow/schema.cc +@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, + + // The user is explicitly asking for Impala int96 encoding, there is no + // logical type. +- if (arrow_properties.support_deprecated_int96_timestamps()) { ++ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { + *physical_type = ParquetType::INT96; + return Status::OK(); + } + +diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc +index 285e2a5973..aa6f92f077 100644 +--- a/cpp/src/parquet/arrow/reader.cc ++++ b/cpp/src/parquet/arrow/reader.cc +@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector& row_groups, + return Status::OK(); + } + +- int64_t num_rows = 0; ++ std::vector num_rows; + for (int row_group : row_groups) { +- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); ++ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); + } + + using ::arrow::RecordBatchIterator; ++ int row_group_idx = 0; + + // NB: This lambda will be invoked outside the scope of this call to + // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. + // `this` is a non-owning pointer so we are relying on the parent FileReader outliving + // this RecordBatchReader. + ::arrow::Iterator batches = ::arrow::MakeFunctionIterator( +- [readers, batch_schema, num_rows, ++ [readers, batch_schema, num_rows, row_group_idx, + this]() mutable -> ::arrow::Result { + ::arrow::ChunkedArrayVector columns(readers.size()); + +- // don't reserve more rows than necessary +- int64_t batch_size = std::min(properties().batch_size(), num_rows); +- num_rows -= batch_size; ++ int64_t batch_size = 0; ++ if (!num_rows.empty()) { ++ // don't reserve more rows than necessary ++ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); ++ num_rows[row_group_idx] -= batch_size; ++ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) != row_group_idx) { ++ row_group_idx++; ++ } ++ } + + RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( + reader_properties_.use_threads(), static_cast(readers.size()), +diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc +index 4fd7ef1b47..87326a54f1 100644 +--- a/cpp/src/parquet/arrow/writer.cc ++++ b/cpp/src/parquet/arrow/writer.cc +@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter { + return Status::OK(); + } + ++ int64_t GetBufferedSize() override { ++ if (row_group_writer_ == nullptr) { ++ return 0; ++ } ++ return row_group_writer_->total_compressed_bytes() + ++ row_group_writer_->total_compressed_bytes_written(); ++ } ++ + Status Close() override { + if (!closed_) { + // Make idempotent +@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter { + + // Max number of rows allowed in a row group. + const int64_t max_row_group_length = this->properties().max_row_group_length(); ++ const int64_t max_row_group_size = this->properties().max_row_group_size(); + + // Initialize a new buffered row group writer if necessary. + if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || +- row_group_writer_->num_rows() >= max_row_group_length) { ++ row_group_writer_->num_rows() >= max_row_group_length || ++ (row_group_writer_->total_compressed_bytes_written() + ++ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) { + RETURN_NOT_OK(NewBufferedRowGroup()); + } + +diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h +index 4a1a033a7b..0f13d05e44 100644 +--- a/cpp/src/parquet/arrow/writer.h ++++ b/cpp/src/parquet/arrow/writer.h +@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter { + /// option in this case. + virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; + ++ /// \brief Return the buffered size in bytes. ++ virtual int64_t GetBufferedSize() = 0; ++ + /// \brief Write the footer and close the file. + virtual ::arrow::Status Close() = 0; + virtual ~FileWriter(); +diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h +index 4d3acb491e..3906ff3c59 100644 +--- a/cpp/src/parquet/properties.h ++++ b/cpp/src/parquet/properties.h +@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true; + static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; + static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; + static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; ++static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024; + static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; + static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; + static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; +@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties { + dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), + write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), + max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), ++ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), + pagesize_(kDefaultDataPageSize), + version_(ParquetVersion::PARQUET_2_6), + data_page_version_(ParquetDataPageVersion::V1), +@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties { + dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), + write_batch_size_(properties.write_batch_size()), + max_row_group_length_(properties.max_row_group_length()), ++ max_row_group_size_(properties.max_row_group_size()), + pagesize_(properties.data_pagesize()), + version_(properties.version()), + data_page_version_(properties.data_page_version()), +@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties { + return this; + } + ++ /// Specify the max bytes size to put in a single row group. ++ /// Default 128 M. ++ Builder* max_row_group_size(int64_t max_row_group_size) { ++ max_row_group_size_ = max_row_group_size; ++ return this; ++ } ++ + /// Specify the data page size. + /// Default 1MB. + Builder* data_pagesize(int64_t pg_size) { +@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties { + + return std::shared_ptr(new WriterProperties( + pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, +- pagesize_, version_, created_by_, page_checksum_enabled_, ++ max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_, + std::move(file_encryption_properties_), default_column_properties_, + column_properties, data_page_version_, store_decimal_as_integer_, + std::move(sorting_columns_))); +@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties { + int64_t dictionary_pagesize_limit_; + int64_t write_batch_size_; + int64_t max_row_group_length_; ++ int64_t max_row_group_size_; + int64_t pagesize_; + ParquetVersion::type version_; + ParquetDataPageVersion data_page_version_; +@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties { + + inline int64_t max_row_group_length() const { return max_row_group_length_; } + ++ inline int64_t max_row_group_size() const { return max_row_group_size_; } ++ + inline int64_t data_pagesize() const { return pagesize_; } + + inline ParquetDataPageVersion data_page_version() const { +@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties { + private: + explicit WriterProperties( + MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, +- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version, ++ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version, + const std::string& created_by, bool page_write_checksum_enabled, + std::shared_ptr file_encryption_properties, + const ColumnProperties& default_column_properties, +@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties { + dictionary_pagesize_limit_(dictionary_pagesize_limit), + write_batch_size_(write_batch_size), + max_row_group_length_(max_row_group_length), ++ max_row_group_size_(max_row_group_size), + pagesize_(pagesize), + parquet_data_page_version_(data_page_version), + parquet_version_(version), +@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties { + int64_t dictionary_pagesize_limit_; + int64_t write_batch_size_; + int64_t max_row_group_length_; ++ int64_t max_row_group_size_; + int64_t pagesize_; + ParquetDataPageVersion parquet_data_page_version_; + ParquetVersion::type parquet_version_; +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +index 9df922afa2..5c8b3d4d07 100644 +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) + REQUIRED_VERSION + 0.11.0) + +- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) ++ if(NOT Thrift_VERSION) ++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) ++ set(Thrift_VERSION "${thrift_PC_VERSION}") ++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) ++ set(Thrift_VERSION "${ThriftAlt_VERSION}") ++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) ++ set(Thrift_VERSION "${THRIFT_VERSION}") ++ endif() ++ endif() ++ if(NOT Thrift_VERSION) ++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") ++ endif() ++ ++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") + list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) + list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) + list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh index ab48d3bcd2278b..ba65efb7a2d1da 100755 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh @@ -60,10 +60,14 @@ create_archive() { arrow_source="arrow-apache-arrow-24.0.0" arrow_archive="apache-arrow-24.0.0.tar.gz" +arrow_17_source="arrow-apache-arrow-17.0.0" +arrow_17_archive="apache-arrow-17.0.0.tar.gz" paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2" +paimon_17_source="${paimon_source}-arrow-17" paimon_archive="paimon-cpp-0a4f4e2.tar.gz" create_archive "${arrow_source}" "${arrow_archive}" arrow +create_archive "${arrow_17_source}" "${arrow_17_archive}" arrow17 create_archive "${paimon_source}" "${paimon_archive}" paimon arrow_patches=( @@ -71,6 +75,11 @@ arrow_patches=( apache-arrow-24.0.0-force-write-int96-timestamps.patch apache-arrow-24.0.0-lzo.patch ) +arrow_17_patches=( + apache-arrow-17.0.0-paimon.patch + apache-arrow-17.0.0-force-write-int96-timestamps.patch + apache-arrow-17.0.0-lzo.patch +) paimon_patches=( paimon-cpp-buildutils-static-deps.patch paimon-cpp-arrow-24-compatibility.patch @@ -79,16 +88,22 @@ paimon_patches=( for index in 0 1 2; do create_patch "${harness}/patches/${arrow_patches[${index}]}" "arrow-$((index + 1)).txt" + create_patch "${harness}/patches/${arrow_17_patches[${index}]}" "arrow17-$((index + 1)).txt" create_patch "${harness}/patches/${paimon_patches[${index}]}" "paimon-$((index + 1)).txt" done arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')" +arrow_17_md5="$(md5sum "${harness}/src/${arrow_17_archive}" | awk '{print $1}')" paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" { printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src" printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed" printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches" - printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)' + printf '%s\n' 'TP_ARCHIVES=(ARROW_17 ARROW PAIMON_CPP_17 PAIMON_CPP)' + printf 'ARROW_17_NAME="%s"\n' "${arrow_17_archive}" + printf 'ARROW_17_SOURCE="%s"\n' "${arrow_17_source}" + printf 'ARROW_17_MD5SUM="%s"\n' "${arrow_17_md5}" + printf '%s\n' 'ARROW_17_DOWNLOAD="unused"' printf 'ARROW_NAME="%s"\n' "${arrow_archive}" printf 'ARROW_SOURCE="%s"\n' "${arrow_source}" printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}" @@ -97,7 +112,13 @@ paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}" printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}" printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"' + printf 'PAIMON_CPP_17_NAME="%s"\n' "${paimon_archive}" + printf 'PAIMON_CPP_17_ARCHIVE_SOURCE="%s"\n' "${paimon_source}" + printf 'PAIMON_CPP_17_SOURCE="%s"\n' "${paimon_17_source}" + printf 'PAIMON_CPP_17_MD5SUM="%s"\n' "${paimon_md5}" + printf '%s\n' 'PAIMON_CPP_17_DOWNLOAD="unused"' printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n" test-fingerprint; }' + printf '%s\n' 'arrow_paimon_17_build_fingerprint() { printf "%s\n" test-17-fingerprint; }' } >"${harness}/vars.sh" exercise_interrupted_patch_set() { @@ -143,6 +164,34 @@ exercise_interrupted_patch_set ARROW "${arrow_source}" "${arrow_archive}" arrow exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}" "${paimon_archive}" paimon \ "${paimon_patches[@]}" +exercise_legacy_source_isolation() { + local index + + touch "${harness}/src/${arrow_source}/current-arrow-sentinel" + TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ + bash "${harness}/download-thirdparty.sh" ARROW_17 >/dev/null + for index in 1 2 3; do + [[ "$(<"${harness}/src/${arrow_17_source}/arrow17-${index}.txt")" == "patched" ]] || + fail "Arrow 17 patch ${index} was not applied" + done + [[ -f "${harness}/src/${arrow_source}/current-arrow-sentinel" ]] || + fail "extracting Arrow 17 modified the Arrow 24 source" + + touch "${harness}/src/${paimon_source}/current-paimon-sentinel" + TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ + bash "${harness}/download-thirdparty.sh" PAIMON_CPP_17 >/dev/null + [[ "$(<"${harness}/src/${paimon_17_source}/paimon-1.txt")" == "patched" ]] || + fail "Paimon Arrow 17 static dependency patch was not applied" + for index in 2 3; do + [[ "$(<"${harness}/src/${paimon_17_source}/paimon-${index}.txt")" == "original" ]] || + fail "Paimon Arrow 17 received an Arrow 24-only patch" + done + [[ -f "${harness}/src/${paimon_source}/current-paimon-sentinel" ]] || + fail "extracting Paimon for Arrow 17 modified the Arrow 24 source" +} + +exercise_legacy_source_isolation + exercise_generic_recovery_dispatch() { local generic="${tmpdir}/generic-recovery" local thirdparty_dir="${generic}/thirdparty" @@ -159,6 +208,8 @@ exercise_generic_recovery_dispatch() { local clean local package1 local package2 + local package3 + local package4 local extra mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ @@ -230,9 +281,10 @@ exercise_generic_recovery_dispatch() { status=$? fi [[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking its builder" - read -r flag parallel package1 package2 extra <"${args_file}" + read -r flag parallel package1 package2 package3 package4 extra <"${args_file}" [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || + "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && + "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || fail "generic recovery dispatched the wrong build package set" if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ @@ -242,9 +294,10 @@ exercise_generic_recovery_dispatch() { status=$? fi [[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before invoking its builder" - read -r flag parallel clean package1 package2 extra <"${args_file}" + read -r flag parallel clean package1 package2 package3 package4 extra <"${args_file}" [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" == "--clean" && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || + "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && + "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || fail "generic clean recovery dispatched the wrong build package set" if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ @@ -284,18 +337,24 @@ exercise_generic_recovery_dispatch # stale Arrow installation pass the shared prebuilt validation. . "${ROOT}/arrow-paimon-vars.sh" prebuilt="${tmpdir}/prebuilt" -mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" +selected_prebuilt="$(arrow_install_dir "${prebuilt}")" +mkdir -p "${selected_prebuilt}/include/arrow/util" "${selected_prebuilt}/lib64" printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \ - >"${prebuilt}/include/arrow/util/config.h" + >"${selected_prebuilt}/include/arrow/util/config.h" for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${prebuilt}/lib64/${library}" + touch "${selected_prebuilt}/lib64/${library}" done prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}" [[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || fail "focused recovery dispatches a bundled source package as a build target" +[[ "${ARROW_PAIMON_SHARED_BUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 arrow paimon_cpp" ]] || + fail "shared recovery does not cover both installed Arrow/Paimon stacks" [[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli" ]] || fail "focused recovery does not download the complete Arrow source closure" +prepare_arrow_paimon_download_packages arrow_17 paimon_cpp_17 +[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 xsimd_17 brotli" ]] || + fail "Arrow 17 build does not download its independent source closure" # A legacy prebuilt may have the old combined marker but no component markers. # Generic build.sh consumers must reject it before importing Arrow Compute. @@ -306,13 +365,13 @@ fi publish_arrow_prebuilt_marker "${prebuilt}" publish_paimon_prebuilt_marker "${prebuilt}" -rm "${prebuilt}/lib64/libarrow_compute.a" +rm "${selected_prebuilt}/lib64/libarrow_compute.a" if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fail "prebuilt validation accepted a missing Arrow Compute archive" fi -touch "${prebuilt}/lib64/libarrow_compute.a" +touch "${selected_prebuilt}/lib64/libarrow_compute.a" -printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt" +printf '%s\n' stale-arrow >"${selected_prebuilt}/arrow-build-fingerprint.txt" if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fail "Paimon-only marker update certified a stale Arrow build" fi @@ -338,5 +397,92 @@ fi publish_arrow_prebuilt_marker "${prebuilt}" arrow_paimon_prebuilt_valid "${prebuilt}" || fail "republished component markers were rejected" +select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 +[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] || + fail "recovery did not select only the missing Arrow 17 stack" + +# The legacy stack remains at the unversioned prefix and is validated +# independently. Rebuilding or invalidating either stack must not affect the +# other branch's artifacts or fingerprints. +mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" +printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_17_VERSION}" \ + >"${prebuilt}/include/arrow/util/config.h" +for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}"; do + touch "${prebuilt}/lib64/${library}" +done +publish_arrow_17_prebuilt_marker "${prebuilt}" +publish_paimon_17_prebuilt_marker "${prebuilt}" +arrow_paimon_17_prebuilt_valid "${prebuilt}" || + fail "matching Arrow 17 and Paimon artifacts were rejected" +arrow_paimon_prebuilt_valid "${prebuilt}" || + fail "publishing the Arrow 17 stack invalidated Arrow 24" +shared_arrow_paimon_prebuilt_valid "${prebuilt}" || + fail "matching shared Arrow/Paimon stacks were rejected" +select_arrow_paimon_rebuild_packages "${prebuilt}" +[[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]] || + fail "recovery rebuilt an already valid shared stack" + +invalidate_arrow_prebuilt_marker "${prebuilt}" +arrow_paimon_17_prebuilt_valid "${prebuilt}" || + fail "invalidating Arrow 24 affected the Arrow 17 stack" +select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 +[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || + fail "recovery did not isolate an invalid Arrow 24 stack" +publish_arrow_prebuilt_marker "${prebuilt}" + +invalidate_arrow_17_prebuilt_marker "${prebuilt}" +arrow_paimon_prebuilt_valid "${prebuilt}" || + fail "invalidating Arrow 17 affected the Arrow 24 stack" +select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 +[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] || + fail "recovery did not isolate an invalid Arrow 17 stack" +publish_arrow_17_prebuilt_marker "${prebuilt}" +arrow_paimon_17_prebuilt_valid "${prebuilt}" || + fail "republished Arrow 17 component markers were rejected" + +# Reinstalling one stack cleans only files owned by that stack. In particular, +# a downgrade of the legacy prefix must remove Arrow 24-only artifacts without +# deleting Paimon or unrelated thirdparty files. +cleanup_prefix="${tmpdir}/cleanup-prefix" +mkdir -p \ + "${cleanup_prefix}/include/arrow" \ + "${cleanup_prefix}/include/parquet" \ + "${cleanup_prefix}/include/paimon" \ + "${cleanup_prefix}/include/unrelated" \ + "${cleanup_prefix}/lib64/cmake/ArrowCompute" \ + "${cleanup_prefix}/lib64/cmake/Paimon" \ + "${cleanup_prefix}/lib64/pkgconfig" \ + "${cleanup_prefix}/share/arrow" \ + "${cleanup_prefix}/share/doc/arrow" +touch \ + "${cleanup_prefix}/lib64/libarrow_compute.a" \ + "${cleanup_prefix}/lib64/libparquet.a" \ + "${cleanup_prefix}/lib64/libpaimon.a" \ + "${cleanup_prefix}/lib64/libfmt_paimon.a" \ + "${cleanup_prefix}/lib64/libunrelated.a" \ + "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" \ + "${cleanup_prefix}/include/unrelated/sentinel" + +clean_arrow_artifacts_in "${cleanup_prefix}" +[[ ! -e "${cleanup_prefix}/include/arrow" && + ! -e "${cleanup_prefix}/include/parquet" && + ! -e "${cleanup_prefix}/lib64/libarrow_compute.a" && + ! -e "${cleanup_prefix}/lib64/cmake/ArrowCompute" && + ! -e "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" ]] || + fail "Arrow cleanup left stale artifacts in the selected prefix" +[[ -e "${cleanup_prefix}/include/paimon" && + -e "${cleanup_prefix}/lib64/libpaimon.a" && + -e "${cleanup_prefix}/include/unrelated/sentinel" ]] || + fail "Arrow cleanup removed another package's artifacts" + +clean_paimon_artifacts_in "${cleanup_prefix}" +[[ ! -e "${cleanup_prefix}/include/paimon" && + ! -e "${cleanup_prefix}/lib64/libpaimon.a" && + ! -e "${cleanup_prefix}/lib64/libfmt_paimon.a" && + ! -e "${cleanup_prefix}/lib64/cmake/Paimon" ]] || + fail "Paimon cleanup left stale artifacts in the selected prefix" +[[ -e "${cleanup_prefix}/lib64/libunrelated.a" && + -e "${cleanup_prefix}/include/unrelated/sentinel" ]] || + fail "Paimon cleanup removed another package's artifacts" echo "PASS" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index f8eedabe0fcbff..142ed38805b94f 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -385,7 +385,7 @@ AWS_SDK_SOURCE="aws-sdk-cpp-1.11.219" AWS_SDK_MD5SUM="80aa616efe1a3e7a9bf0dfbc44a97864" # tsan_header -TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7" +TSAN_HEADER_DOWNLOAD="https://raw.githubusercontent.com/gcc-mirror/gcc/b2d961e7342b5ba4e57adfa81cb189b738d10901/libsanitizer/include/sanitizer/tsan_interface_atomic.h" TSAN_HEADER_NAME="tsan_interface_atomic.h" TSAN_HEADER_FILE="tsan_interface_atomic.h" TSAN_HEADER_MD5SUM="d72679bea167d6a513d959f5abd149dc" @@ -619,6 +619,7 @@ export TP_ARCHIVES=( 'CYRUS_SASL' 'LIBRDKAFKA' 'FLATBUFFERS' + 'ARROW_17' 'ARROW' 'ARROW_ADBC' 'BROTLI' @@ -651,6 +652,7 @@ export TP_ARCHIVES=( 'PDQSORT' 'TIMSORT' 'BENCHMARK' + 'XSIMD_17' 'XSIMD' 'SIMDJSON' 'NLOHMANN_JSON' @@ -674,6 +676,7 @@ export TP_ARCHIVES=( 'JINDOFS' 'JUICEFS' 'PUGIXML' + 'PAIMON_CPP_17' 'PAIMON_CPP' 'LANCE_C' )