Skip to content

[fix](build) Keep Arrow 17 and 24 in shared thirdparty #66546

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hubgeter wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[fix](build) Keep Arrow 17 and 24 in shared thirdparty #66546

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a5c653
[fix](build) Keep Arrow 17 and 24 in shared thirdparty
hubgeter Aug 6, 2026
4f867ef
[fix](regression) Load packaged ADBC JNI library in external tests
hubgeter Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions .github/workflows/be-ut-mac.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,24 +129,27 @@ jobs:
fi
tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz

# Rebuild the Arrow/Paimon stack when its inputs change or the shared
# prebuilt predates the selected Arrow version/component closure.
# Rebuild either Arrow/Paimon stack when its inputs change or the
# shared prebuilt predates the dual-version component closure.
# The artifact check also covers scheduled and later BE-only builds,
# where Paths Filter is skipped or reports no Arrow/Paimon changes.
# shellcheck source=thirdparty/arrow-paimon-vars.sh
. ./arrow-paimon-vars.sh
select_arrow_paimon_rebuild_packages installed
arrow_paimon_prebuilt_is_valid=false
if arrow_paimon_prebuilt_valid installed; then
if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]]; then
arrow_paimon_prebuilt_is_valid=true
fi
if [[ "${{ steps.filter.outputs.arrow_paimon_changes }}" == "true" ||
"${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then
if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]]; then
ARROW_PAIMON_REBUILD_PACKAGES=("${ARROW_PAIMON_SHARED_BUILD_PACKAGES[@]}")
fi
curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \
-o doris-thirdparty-source.tgz
tar -zxvf doris-thirdparty-source.tgz
./download-thirdparty.sh arrow paimon_cpp xsimd brotli
export MACOSX_DEPLOYMENT_TARGET=12.0
./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp
./build-thirdparty.sh -j "$(nproc)" "${ARROW_PAIMON_REBUILD_PACKAGES[@]}"
fi
popd

Expand Down
35 changes: 29 additions & 6 deletions be/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")

option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH "Selected Arrow install prefix")
set(PAIMON_HOME "" CACHE PATH "Selected Paimon install prefix")
option(ENABLE_TDE "Enable TDE feature module" OFF)
set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
option(ENABLE_TLS "Enable TLS feature module" OFF)
Expand All @@ -178,8 +179,24 @@ endif()
if (DEFINED ENV{ENABLE_PAIMON_CPP})
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
endif()
if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
if (DEFINED ENV{ARROW_HOME})
set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "" FORCE)
endif()
if (DEFINED ENV{PAIMON_HOME})
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
elseif (NOT PAIMON_HOME)
set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "" FORCE)
endif()

if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h")
message(FATAL_ERROR "Selected Arrow installation is incomplete: ${ARROW_HOME}")
endif()
message(STATUS "ARROW_HOME is ${ARROW_HOME}")
if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon")
message(FATAL_ERROR "Selected Paimon installation is incomplete: ${PAIMON_HOME}")
endif()
if (ENABLE_PAIMON_CPP)
message(STATUS "PAIMON_HOME is ${PAIMON_HOME}")
endif()

set(CMAKE_SKIP_RPATH TRUE)
Expand Down Expand Up @@ -520,6 +537,12 @@ include_directories(
${SRC_DIR}/
)

include_directories(
BEFORE SYSTEM
${ARROW_HOME}/include
${PAIMON_HOME}/include
)

include_directories(
SYSTEM
${COMMON_SRC_DIR}
Expand Down Expand Up @@ -665,10 +688,10 @@ endif()

set(PAIMON_FACTORY_REGISTRY_LIBS)
if (ENABLE_PAIMON_CPP)
# Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
# thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
# No dual-stack selection needed — single Arrow for everything.
# The Arrow selected for this BE build includes COMPUTE/DATASET/ACERO/FILESYSTEM,
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY
# via thirdparty.cmake. Paimon uses this same selected Arrow (no paimon_deps);
# the legacy Arrow 17 stack remains installed only for branch-4.1 consumers.

# paimon_parquet_file_format depends on Arrow Dataset symbols.
# Force-link it with --whole-archive so its factory registration runs.
Expand Down
59 changes: 33 additions & 26 deletions be/cmake/thirdparty.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@ set(COMMON_THIRDPARTY)
# if arg exist noadd, not append to COMMON_THIRDPARTY variable
# if arg exist libname, use libname to find library
# if arg exist wholelibpath, use wholelibpath to find library
# if arg exist root, resolve the library below that install prefix
function(add_thirdparty)
cmake_parse_arguments(DORIS_THIRDPARTY
"NOTADD;LIB64"
"LIBNAME;WHOLELIBPATH"
"LIBNAME;WHOLELIBPATH;ROOT"
""
${ARGN})

Expand All @@ -39,14 +40,20 @@ function(add_thirdparty)
set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME} PARENT_SCOPE)
endif()

if (DORIS_THIRDPARTY_ROOT)
set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT})
else()
set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR})
endif()

if (DORIS_THIRDPARTY_LIB64)
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
elseif (DORIS_THIRDPARTY_LIBNAME)
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME})
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME})
elseif (DORIS_THIRDPARTY_WHOLELIBPATH)
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH})
else()
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
endif()
set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH})
endfunction()
Expand Down Expand Up @@ -100,18 +107,18 @@ add_thirdparty(cares LIB64)
add_thirdparty(address_sorting LIB64)
add_thirdparty(z LIB64)

add_thirdparty(brotlicommon LIB64)
add_thirdparty(brotlidec LIB64)
add_thirdparty(brotlienc LIB64)
add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME})
add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME})
add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME})
add_thirdparty(zstd LIB64)
add_thirdparty(arrow LIB64)
add_thirdparty(arrow_compute LIB64)
add_thirdparty(arrow_flight LIB64)
add_thirdparty(arrow_flight_sql LIB64)
add_thirdparty(arrow_dataset LIB64)
add_thirdparty(arrow_acero LIB64)
add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME})
add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME})
add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME})
add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME})
add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME})
add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME})
add_thirdparty(adbc_driver_manager LIB64)
add_thirdparty(parquet LIB64)
add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME})
# liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it
# so the final linker resolves C math symbols from the system library first.
add_thirdparty(lance_c LIB64 NOTADD)
Expand Down Expand Up @@ -191,16 +198,16 @@ add_thirdparty(icudata LIB64)
add_thirdparty(pugixml LIB64)

if (ENABLE_PAIMON_CPP)
add_thirdparty(paimon LIB64)
add_thirdparty(paimon_parquet_file_format LIB64)
add_thirdparty(paimon_orc_file_format LIB64)
add_thirdparty(paimon_blob_file_format LIB64)
add_thirdparty(paimon_local_file_system LIB64)
add_thirdparty(paimon_file_index LIB64)
add_thirdparty(paimon_global_index LIB64)

add_thirdparty(roaring_bitmap_paimon LIB64)
add_thirdparty(xxhash_paimon LIB64)
add_thirdparty(fmt_paimon LIB64)
add_thirdparty(tbb_paimon LIB64)
add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME})

add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME})
add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME})
endif()
12 changes: 7 additions & 5 deletions build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ rebuild_thirdparty_libraries() {
build_args+=(--clean)
fi
bash "${build_script}" "${build_args[@]}" "$@"
if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2
exit 1
fi
Expand All @@ -527,10 +527,12 @@ rebuild_thirdparty_libraries() {
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
echo "Thirdparty libraries need to be build ..."
rebuild_thirdparty_libraries true
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] &&
! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed"
if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
rebuild_thirdparty_libraries false "${ARROW_PAIMON_REBUILD_PACKAGES[@]}"
fi
fi

update_submodule() {
Expand Down
2 changes: 1 addition & 1 deletion regression-test/pipeline/external/conf/fe.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ DATE = `date +%Y%m%d-%H%M%S`
JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true"

# For jdk 17+, this JAVA_OPTS will be used as default JVM options
JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED"
JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false -Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED"

##
## the lowercase properties are read by main program.
Expand Down
Loading
Loading