diff --git a/build.sh b/build.sh index ceaa09a0779593..792b6cbeccdf99 100755 --- a/build.sh +++ b/build.sh @@ -68,6 +68,11 @@ Usage: $0 --be-extension-ignore build be-java-extensions package, choose which modules to ignore. Multiple modules separated by commas. --enable-dynamic-arch enable dynamic CPU detection in OpenBLAS. Default ON. --disable-dynamic-arch disable dynamic CPU detection in OpenBLAS. + --exclude-obs-dependencies exclude all Huawei Cloud OBS (com.huaweicloud) dependencies and the + fe-filesystem-obs module; nothing from Huawei is resolved, compiled, + or bundled. Use when repo.huaweicloud.com is unreachable or forbidden. + --exclude-cos-dependencies exclude all Tencent Cloud COS dependencies and the fe-filesystem-cos + module; nothing from Tencent COS is resolved, compiled, or bundled. --clean clean and build target --compile-bench BE compile-speed benchmark: cold, cache-free BE-only build (fresh dedicated build dir, ccache disabled) with a per-phase @@ -270,6 +275,8 @@ if ! OPTS="$(getopt \ -l 'be-extension-ignore:' \ -l 'enable-dynamic-arch' \ -l 'disable-dynamic-arch' \ + -l 'exclude-obs-dependencies' \ + -l 'exclude-cos-dependencies' \ -l 'clean' \ -l 'compile-bench' \ -l 'coverage' \ @@ -802,6 +809,14 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then modules+=("fe-filesystem/fe-filesystem-api") modules+=("fe-filesystem/fe-filesystem-spi") for _fs_mod in s3-base s3 gcs minio ozone oss cos obs azure hdfs-base hdfs oss-hdfs jfs local broker http; do + # Skip the modules whose Maven profile is deactivated so the -pl list stays consistent with + # the reactor: obs is absent under -Ddisable.obs=true, cos under -Ddisable.cos=true. + if [[ "${_fs_mod}" == "obs" && "${BUILD_OBS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi + if [[ "${_fs_mod}" == "cos" && "${BUILD_COS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi if [[ -d "${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${_fs_mod}" ]]; then modules+=("fe-filesystem/fe-filesystem-${_fs_mod}") fi @@ -1070,10 +1085,14 @@ function build_fe_modules() { extra_mvn_opts=(${MVN_OPT}) fi if [[ "${BUILD_OBS_DEPENDENCIES}" -eq 0 ]]; then - dependency_mvn_opts+=("-Dobs.dependency.scope=provided") + # Deactivates the `obs` Maven profile in fe-core, hadoop-deps and fe-filesystem, so no + # com.huaweicloud artifact is resolved and the Huawei OBS module is not built or bundled. + dependency_mvn_opts+=("-Ddisable.obs=true") fi if [[ "${BUILD_COS_DEPENDENCIES}" -eq 0 ]]; then - dependency_mvn_opts+=("-Dcos.dependency.scope=provided") + # Deactivates the `cos` Maven profile in fe-core and fe-filesystem, so no Tencent COS + # artifact is resolved and the fe-filesystem-cos module is not built or bundled. + dependency_mvn_opts+=("-Ddisable.cos=true") fi if [[ -n "${USER_SETTINGS_MVN_REPO}" && -f "${USER_SETTINGS_MVN_REPO}" ]]; then user_settings_opts=(-gs "${USER_SETTINGS_MVN_REPO}") @@ -1191,6 +1210,14 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then if [ ! -d "${fs_module_dir}" ]; then continue fi + # These modules are not built when their Maven profile is deactivated, so their plugin zip + # does not exist; skip the unpack to keep packaging consistent with the reactor. + if [[ "${fs_module}" == "obs" && "${BUILD_OBS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi + if [[ "${fs_module}" == "cos" && "${BUILD_COS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi mkdir -p "${fs_plugin_target}" # Unpack the self-contained plugin zip produced by maven-assembly-plugin. # Layout inside the zip: .jar at root + lib/*.jar for runtime deps. diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml b/fe/be-java-extensions/hadoop-deps/pom.xml index aba07779a80646..8c4c71aa486f8f 100644 --- a/fe/be-java-extensions/hadoop-deps/pom.xml +++ b/fe/be-java-extensions/hadoop-deps/pom.xml @@ -67,10 +67,8 @@ under the License. org.apache.hadoop.thirdparty hadoop-shaded-protobuf_3_25 - - com.huaweicloud - hadoop-huaweicloud - + org.apache.hadoop hadoop-client @@ -95,13 +93,32 @@ under the License. - - - - huawei-obs-sdk - https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ - - + + + + obs + + + !disable.obs + + + + + com.huaweicloud + hadoop-huaweicloud + + + + + + huawei-obs-sdk + https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ + + + + diff --git a/fe/be-java-extensions/preload-extensions/pom.xml b/fe/be-java-extensions/preload-extensions/pom.xml index 0b32dbed8e207a..d06b118bf39edf 100644 --- a/fe/be-java-extensions/preload-extensions/pom.xml +++ b/fe/be-java-extensions/preload-extensions/pom.xml @@ -168,18 +168,8 @@ under the License. com.zaxxer HikariCP - - - com.qcloud.cos - hadoop-cos - ${tencentcos.version} - - - org.json - json - - - + com.google.guava guava @@ -236,6 +226,34 @@ under the License. + + + + cos + + + !disable.cos + + + + + + com.qcloud.cos + hadoop-cos + ${tencentcos.version} + + + org.json + json + + + + + + + preload-extensions ${project.basedir}/target/ diff --git a/fe/fe-connector/fe-connector-paimon/pom.xml b/fe/fe-connector/fe-connector-paimon/pom.xml index 8a360e57bdfdbf..72da5fb2a1d6cd 100644 --- a/fe/fe-connector/fe-connector-paimon/pom.xml +++ b/fe/fe-connector/fe-connector-paimon/pom.xml @@ -192,24 +192,8 @@ under the License. hadoop-aws - - - com.huaweicloud - hadoop-huaweicloud - runtime - + - - - huawei-obs-sdk - https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ - - + + + + obs + + + !disable.obs + + + + + + com.huaweicloud + hadoop-huaweicloud + runtime + + + + + + huawei-obs-sdk + https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ + + + + doris-fe-connector-paimon diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index b02fc027d1572e..404f0a2cac0715 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -70,6 +70,70 @@ under the License. ${env.FE_UT_PARALLEL} + + + obs + + + !disable.obs + + + + + ${project.groupId} + fe-filesystem-obs + ${project.version} + test + + + + com.squareup.okhttp3 + * + + + com.squareup.okio + * + + + + + + + + cos + + + !disable.cos + + + + + ${project.groupId} + fe-filesystem-cos + ${project.version} + test + + + + com.squareup.okhttp3 + * + + + com.squareup.okio + * + + + + + @@ -219,42 +283,10 @@ under the License. ${project.version} test - - ${project.groupId} - fe-filesystem-obs - ${project.version} - test - - - - com.squareup.okhttp3 - * - - - com.squareup.okio - * - - - - - ${project.groupId} - fe-filesystem-cos - ${project.version} - test - - - - com.squareup.okhttp3 - * - - - com.squareup.okio - * - - - + + ${project.groupId} fe-filesystem-gcs diff --git a/fe/fe-filesystem/pom.xml b/fe/fe-filesystem/pom.xml index 15392808f1a226..88fe040e7cb880 100644 --- a/fe/fe-filesystem/pom.xml +++ b/fe/fe-filesystem/pom.xml @@ -65,8 +65,10 @@ under the License. fe-filesystem-minio fe-filesystem-ozone fe-filesystem-oss - fe-filesystem-cos - fe-filesystem-obs + + fe-filesystem-local fe-filesystem-azure fe-filesystem-hdfs-base @@ -111,5 +113,38 @@ under the License. + + + + obs + + + !disable.obs + + + + fe-filesystem-obs + + + + + cos + + + !disable.cos + + + + fe-filesystem-cos + + + +