From 17c527a80a1d6afd61b8335382bc6f56b117d815 Mon Sep 17 00:00:00 2001 From: Mryange Date: Mon, 3 Aug 2026 14:24:53 +0800 Subject: [PATCH 1/3] [opt](column) reuse nullable column metadata during function execution (#66031) The default nullable function path independently inspected the same null maps when checking for all-NULL arguments, unnesting nullable inputs, and wrapping function results. This caused redundant full null-map scans and made type-level nullable checks difficult to distinguish from runtime NULL-value checks. This change introduces `NullableColumnInfo` to collect the nested column, typed null-map column, constness, and non-NULL count once per input column. The information is reused across nullable unnesting, CAST handling, and result wrapping. It also renames the type-level helper to `has_nullable_argument_type` and preserves the existing copy-on-write behavior by copying nested data only when NULL payloads must be replaced with defaults. (cherry picked from commit a4423dc2a02cb60fd45c4f31343f768715b0100d) --- .../core/block/column_with_type_and_name.cpp | 83 ++++++---- be/src/core/block/column_with_type_and_name.h | 20 ++- be/src/core/column/column.h | 11 ++ be/src/core/column/column_const.h | 7 + be/src/core/column/column_nullable.cpp | 23 +++ be/src/core/column/column_nullable.h | 8 + be/src/exprs/function/cast/function_cast.cpp | 27 +++- be/src/exprs/function/function.cpp | 60 +++++--- be/src/exprs/function/function.h | 3 + .../block/column_with_type_and_name_test.cpp | 142 +++++++++++++++++- .../function/function_arithmetic_test.cpp | 8 + 11 files changed, 331 insertions(+), 61 deletions(-) diff --git a/be/src/core/block/column_with_type_and_name.cpp b/be/src/core/block/column_with_type_and_name.cpp index ec3f4a61386fdd..917bfd3d1421ae 100644 --- a/be/src/core/block/column_with_type_and_name.cpp +++ b/be/src/core/block/column_with_type_and_name.cpp @@ -30,10 +30,10 @@ #include "core/column/column.h" #include "core/column/column_const.h" #include "core/column/column_nothing.h" +#include "core/column/column_nullable.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_nullable.h" #include "core/types.h" -#include "util/simd/bits.h" namespace doris { @@ -105,41 +105,58 @@ void ColumnWithTypeAndName::to_pb_column_meta(PColumnMeta* col_meta) const { type->to_pb_column_meta(col_meta); } +const ColumnNullable& ColumnWithTypeAndName::get_nullable_column() const { + DCHECK(type->is_nullable()); + DCHECK(column); + const auto& [physical_column, _] = unpack_if_const(column); + return assert_cast(*physical_column); +} + +const ColumnUInt8::Ptr& ColumnWithTypeAndName::get_nullable_null_map_column() const { + return get_nullable_column().get_null_map_column_ptr(); +} + +NullableColumnInfo ColumnWithTypeAndName::get_nullable_column_info() const { + DCHECK(type->is_nullable()); + DCHECK(column); + + const auto [has_null, only_null] = get_nullable_column().get_null_map_state(); + return {.has_null = has_null, + .only_null = only_null, + .is_const = is_column_const(*column), + .is_nullable = true}; +} + ColumnWithTypeAndName ColumnWithTypeAndName::unnest_nullable( - bool replace_null_data_to_default) const { - if (type->is_nullable()) { - auto nested_type = - assert_cast(type.get()) - ->get_nested_type(); - ColumnPtr nested_column = column; - if (column) { - // A column_ptr is needed here to ensure that the column in convert_to_full_column_if_const is not released. - auto [column_ptr, is_const] = unpack_if_const(column); - const auto* source_column = - assert_cast( - column_ptr.get()); - if (is_const) { - nested_column = - ColumnConst::create(source_column->get_nested_column_ptr(), column->size()); - } else { - nested_column = source_column->get_nested_column_ptr(); - } - - if (replace_null_data_to_default) { - const auto& null_map = source_column->get_null_map_data(); - // only need to mutate nested column, avoid to copy nullmap - auto mutable_nested_col = (*std::move(nested_column)).mutate(); - if (simd::contain_one(null_map.data(), null_map.size())) { - mutable_nested_col->replace_column_null_data(null_map.data()); - } - - return {std::move(mutable_nested_col), nested_type, ""}; - } - } - return {nested_column, nested_type, ""}; - } else { + const NullableColumnInfo& info, bool replace_null_data_to_default) const { + if (!type->is_nullable()) { return {column, type, ""}; } + DCHECK(info.is_nullable); + + const auto& nullable_column = get_nullable_column(); + const auto get_nested_column = [&]() -> ColumnPtr { + const auto& nested_column = nullable_column.get_nested_column_ptr(); + if (info.is_const) { + return ColumnConst::create(nested_column, column->size()); + } + return nested_column; + }; + + auto nested_type = assert_cast(type.get()) + ->get_nested_type(); + if (replace_null_data_to_default && info.has_null) { + if (column->try_replace_null_payload_with_default_without_cow()) { + return {get_nested_column(), nested_type, ""}; + } + + // Only copy the nested column because the original nullable column must remain unchanged. + const auto nested_column = get_nested_column(); + auto mutable_nested_col = nested_column->clone_resized(nested_column->size()); + mutable_nested_col->replace_column_null_data(nullable_column.get_null_map_data().data()); + return {std::move(mutable_nested_col), nested_type, ""}; + } + return {get_nested_column(), nested_type, ""}; } Status ColumnWithTypeAndName::check_type_and_column_match() const { diff --git a/be/src/core/block/column_with_type_and_name.h b/be/src/core/block/column_with_type_and_name.h index 9b1c357ac66c98..d6a1954101b18a 100644 --- a/be/src/core/block/column_with_type_and_name.h +++ b/be/src/core/block/column_with_type_and_name.h @@ -26,17 +26,29 @@ #include #include #include +#include +#include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" namespace doris { +class ColumnNullable; class PColumnMeta; } // namespace doris namespace doris { +struct NullableColumnInfo { + bool has_null = false; + bool only_null = false; + bool is_const = false; + bool is_nullable = false; +}; + +using NullableColumnInfos = std::vector; + // class WriteBuffer; /** Column data along with its data type and name. @@ -69,9 +81,15 @@ struct ColumnWithTypeAndName { void to_pb_column_meta(PColumnMeta* col_meta) const; - ColumnWithTypeAndName unnest_nullable(bool replace_null_data_to_default = false) const; + const ColumnUInt8::Ptr& get_nullable_null_map_column() const; + NullableColumnInfo get_nullable_column_info() const; + ColumnWithTypeAndName unnest_nullable(const NullableColumnInfo& info, + bool replace_null_data_to_default) const; Status check_type_and_column_match() const; + +private: + const ColumnNullable& get_nullable_column() const; }; } // namespace doris diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index 3b727285e64613..aa29ca601e7c86 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -724,6 +724,17 @@ class IColumn : public COW { // column_vector and column_decimal override this method to return true virtual bool support_replace_column_null_data() const { return false; } + /** + * Try to replace the payload of NULL rows with the nested column's default value without + * going through COW. Implementations must return false without modifying data unless the + * complete column ownership chain is exclusive. This is only safe because payloads of rows + * that are already NULL are not observable through the nullable column. In particular, a + * shared nested column may belong to another nullable column with a different null map. + * + * This bypasses the normal COW mutation path. Do not use it for general column mutation. + */ + virtual bool try_replace_null_payload_with_default_without_cow() const { return false; } + // For float/double types, replace -0.0 with 0.0, set NaN to quiet NaN, // used to ensure data hash equality for -0.0 and +0.0, e.g. aggregate and join virtual void replace_float_special_values() {} diff --git a/be/src/core/column/column_const.h b/be/src/core/column/column_const.h index dd5296cc1ef726..6823416ad819f6 100644 --- a/be/src/core/column/column_const.h +++ b/be/src/core/column/column_const.h @@ -307,6 +307,13 @@ class ColumnConst final : public COWHelper { return data->support_replace_column_null_data(); } + bool try_replace_null_payload_with_default_without_cow() const override { + if (!IColumn::is_exclusive()) { + return false; + } + return data->try_replace_null_payload_with_default_without_cow(); + } + void finalize() override { data->finalize(); } void erase(size_t start, size_t length) override { diff --git a/be/src/core/column/column_nullable.cpp b/be/src/core/column/column_nullable.cpp index 01d0fc072ed41d..c92ab615380322 100644 --- a/be/src/core/column/column_nullable.cpp +++ b/be/src/core/column/column_nullable.cpp @@ -699,6 +699,29 @@ bool ColumnNullable::only_null() const { return !simd::contain_zero(get_null_map_data().data(), size()); } +ColumnNullable::NullMapState ColumnNullable::get_null_map_state() const { + const auto& null_map = get_null_map_data(); + if (null_map.empty()) { + return {.has_null = false, .only_null = true}; + } + + if (null_map[0]) { + return {.has_null = true, + .only_null = !simd::contain_zero(null_map.data() + 1, null_map.size() - 1)}; + } + return {.has_null = simd::contain_one(null_map.data() + 1, null_map.size() - 1), + .only_null = false}; +} + +bool ColumnNullable::try_replace_null_payload_with_default_without_cow() const { + if (!is_exclusive()) { + return false; + } + + const_cast(get_nested_column()).replace_column_null_data(get_null_map_data().data()); + return true; +} + bool ColumnNullable::has_null(size_t begin, size_t end) const { return simd::contain_one(get_null_map_data().data() + begin, end - begin); } diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index 3e7afb904e6c81..2c611e2732419b 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -64,6 +64,11 @@ class ColumnNullable final : public COWHelper { ColumnNullable(const ColumnNullable&) = default; public: + struct NullMapState { + bool has_null; + bool only_null; + }; + /** Create a column from immutable/shared subcolumns without cloning them. * Call IColumn::mutate before modifying the returned column tree. */ @@ -270,7 +275,10 @@ class ColumnNullable final : public COWHelper { get_null_map_column().is_exclusive(); } + bool try_replace_null_payload_with_default_without_cow() const override; + bool only_null() const override; + NullMapState get_null_map_state() const; // used in schema change void change_nested_column(ColumnPtr& other) { ((ColumnPtr&)_nested_column) = other; } diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index 99b60627d1b20f..75879ebf6b3461 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -182,24 +182,37 @@ WrapperType prepare_remove_nullable(FunctionContext* context, const DataTypePtr& bool replace_null_data_to_default = need_replace_null_data_to_default( context, from_type_not_nullable, to_type_not_nullable); + NullableColumnInfo source_info; + if (block.get_by_position(arguments[0]).type->is_nullable()) { + source_info = block.get_by_position(arguments[0]).get_nullable_column_info(); + } auto nested_result_index = block.columns(); - block.insert(block.get_by_position(result).unnest_nullable()); + const auto& result_column = block.get_by_position(result); + block.insert({nullptr, to_type_not_nullable, result_column.name}); auto nested_source_index = block.columns(); - block.insert(block.get_by_position(arguments[0]) - .unnest_nullable(replace_null_data_to_default)); + if (source_info.is_nullable) { + block.insert(block.get_by_position(arguments[0]) + .unnest_nullable(source_info, replace_null_data_to_default)); + } else { + block.insert(block.get_by_position(arguments[0])); + } - const auto& arg_col = block.get_by_position(arguments[0]); const NullMap::value_type* arg_null_map = nullptr; - if (const auto* nullable = check_and_get_column(*arg_col.column)) { - arg_null_map = nullable->get_null_map_data().data(); + if (source_info.is_nullable) { + arg_null_map = block.get_by_position(arguments[0]) + .get_nullable_null_map_column() + ->get_data() + .data(); } RETURN_IF_ERROR(prepare_impl(context, from_type_not_nullable, to_type_not_nullable)( context, block, {nested_source_index}, nested_result_index, input_rows_count, arg_null_map)); + NullableColumnInfos nullable_column_infos(block.columns()); + nullable_column_infos[arguments[0]] = std::move(source_info); block.get_by_position(result).column = wrap_in_nullable(block.get_by_position(nested_result_index).column, block, - arguments, input_rows_count); + arguments, nullable_column_infos, input_rows_count); block.erase(nested_source_index); block.erase(nested_result_index); diff --git a/be/src/exprs/function/function.cpp b/be/src/exprs/function/function.cpp index c7b35e8260f209..552fafdb45959a 100644 --- a/be/src/exprs/function/function.cpp +++ b/be/src/exprs/function/function.cpp @@ -44,6 +44,7 @@ namespace doris { #include "common/compile_check_begin.h" ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const ColumnNumbers& args, + const NullableColumnInfos& nullable_column_infos, size_t input_rows_count) { ColumnPtr result_null_map_column; /// If result is already nullable. @@ -56,14 +57,13 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum } for (const auto& arg : args) { - const ColumnWithTypeAndName& elem = block.get_by_position(arg); - if (!elem.type->is_nullable() || is_column_const(*elem.column)) { + const auto& info = nullable_column_infos[arg]; + if (!info.is_nullable || info.is_const) { continue; } - if (const auto* nullable = assert_cast(elem.column.get()); - nullable->has_null()) { - const ColumnPtr& null_map_column = nullable->get_null_map_column_ptr(); + if (info.has_null) { + const auto& null_map_column = block.get_by_position(arg).get_nullable_null_map_column(); if (!result_null_map_column) { // NOLINT(bugprone-use-after-move) result_null_map_column = null_map_column->clone_resized(input_rows_count); continue; @@ -75,8 +75,7 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum NullMap& result_null_map = assert_cast(*mutable_result_null_map_column).get_data(); - const NullMap& src_null_map = - assert_cast(*null_map_column).get_data(); + const NullMap& src_null_map = null_map_column->get_data(); VectorizedUtils::update_null_map(result_null_map, src_null_map); } @@ -101,6 +100,18 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum return ColumnNullable::create(src_not_nullable, result_null_map_column); } +ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const ColumnNumbers& args, + size_t input_rows_count) { + NullableColumnInfos nullable_column_infos(block.columns()); + for (const auto arg : args) { + const auto& column = block.get_by_position(arg); + if (column.type->is_nullable()) { + nullable_column_infos[arg] = column.get_nullable_column_info(); + } + } + return wrap_in_nullable(src, block, args, nullable_column_infos, input_rows_count); +} + bool have_null_column(const Block& block, const ColumnNumbers& args) { return std::ranges::any_of(args, [&block](const auto& elem) { return block.get_by_position(elem).type->is_nullable(); @@ -197,16 +208,25 @@ Status PreparedFunctionImpl::default_implementation_for_nulls( return Status::OK(); } - if (std::ranges::any_of(args, [&block](const auto& elem) { - return block.get_by_position(elem).column->only_null(); - })) { - block.get_by_position(result).column = - block.get_by_position(result).type->create_column_const(input_rows_count, Field()); - *executed = true; - return Status::OK(); - } - if (have_null_column(block, args)) { + NullableColumnInfos nullable_column_infos(block.columns()); + for (const auto arg : args) { + const auto& argument = block.get_by_position(arg); + if (!argument.type->is_nullable()) { + continue; + } + + auto info = argument.get_nullable_column_info(); + if (info.only_null) { + auto& result_column = block.get_by_position(result); + result_column.column = + result_column.type->create_column_const(input_rows_count, Field()); + *executed = true; + return Status::OK(); + } + nullable_column_infos[arg] = info; + } + bool need_to_default = need_replace_null_data_to_default(); // extract nested column from nulls ColumnNumbers new_args; @@ -215,7 +235,8 @@ Status PreparedFunctionImpl::default_implementation_for_nulls( for (int i = 0; i < args.size(); ++i) { uint32_t arg = args[i]; new_args.push_back(i); - new_block.simple_insert(block.get_by_position(arg).unnest_nullable(need_to_default)); + new_block.simple_insert(block.get_by_position(arg).unnest_nullable( + nullable_column_infos[arg], need_to_default)); } new_block.simple_insert(block.get_by_position(result)); int new_result = new_block.columns() - 1; @@ -224,8 +245,9 @@ Status PreparedFunctionImpl::default_implementation_for_nulls( // After run with nested, wrap them in null. Before this, block.get_by_position(result).type // is not compatible with get_by_position(result).column - block.get_by_position(result).column = wrap_in_nullable( - new_block.get_by_position(new_result).column, block, args, input_rows_count); + block.get_by_position(result).column = + wrap_in_nullable(new_block.get_by_position(new_result).column, block, args, + nullable_column_infos, input_rows_count); *executed = true; return Status::OK(); diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 13d4bfd6bb573b..807b01b004035d 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -680,5 +680,8 @@ using FunctionPtr = std::shared_ptr; */ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const ColumnNumbers& args, size_t input_rows_count); +ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const ColumnNumbers& args, + const NullableColumnInfos& nullable_column_infos, + size_t input_rows_count); } // namespace doris diff --git a/be/test/core/block/column_with_type_and_name_test.cpp b/be/test/core/block/column_with_type_and_name_test.cpp index 8a5fd999d4297f..bafce0c4350eae 100644 --- a/be/test/core/block/column_with_type_and_name_test.cpp +++ b/be/test/core/block/column_with_type_and_name_test.cpp @@ -35,9 +35,149 @@ TEST(ColumnWithTypeAndNameTest, get_nested_test) { column_with_type_and_name.type = std::make_shared(std::make_shared()); column_with_type_and_name.name = "column_with_type_and_name"; - auto result = column_with_type_and_name.unnest_nullable(true); + auto result = column_with_type_and_name.unnest_nullable( + column_with_type_and_name.get_nullable_column_info(), true); EXPECT_TRUE(is_column_const(*result.column)); EXPECT_EQ(result.column->size(), 3); + EXPECT_EQ(result.column->get_int(0), 0); +} + +TEST(ColumnWithTypeAndNameTest, get_nullable_column_info_for_const_column) { + auto nullable_type = std::make_shared(std::make_shared()); + + auto null_column = ColumnNullable::create(ColumnHelper::create_column({1}), + ColumnHelper::create_column({true})); + ColumnWithTypeAndName const_null {ColumnConst::create(std::move(null_column), 3), nullable_type, + "const_null"}; + auto null_info = const_null.get_nullable_column_info(); + EXPECT_TRUE(null_info.is_const); + EXPECT_TRUE(null_info.has_null); + EXPECT_TRUE(null_info.only_null); + EXPECT_EQ(const_null.get_nullable_null_map_column()->size(), 1); + + auto non_null_column = + ColumnNullable::create(ColumnHelper::create_column({1}), + ColumnHelper::create_column({false})); + ColumnWithTypeAndName const_non_null {ColumnConst::create(std::move(non_null_column), 3), + nullable_type, "const_non_null"}; + auto non_null_info = const_non_null.get_nullable_column_info(); + EXPECT_TRUE(non_null_info.is_const); + EXPECT_FALSE(non_null_info.has_null); + EXPECT_FALSE(non_null_info.only_null); + EXPECT_EQ(const_non_null.get_nullable_null_map_column()->size(), 1); +} + +TEST(ColumnWithTypeAndNameTest, get_nullable_column_info_null_map_states) { + auto nullable_type = std::make_shared(std::make_shared()); + + const auto check_state = [&](std::initializer_list values, + std::initializer_list null_map, bool has_null, + bool only_null) { + ColumnWithTypeAndName column { + ColumnNullable::create(ColumnHelper::create_column(values), + ColumnHelper::create_column(null_map)), + nullable_type, "nullable"}; + const auto info = column.get_nullable_column_info(); + EXPECT_EQ(info.has_null, has_null); + EXPECT_EQ(info.only_null, only_null); + }; + + check_state({}, {}, false, true); + check_state({1, 2, 3}, {false, false, false}, false, false); + check_state({1, 2, 3}, {true, true, true}, true, true); + check_state({1, 2, 3}, {false, true, false}, true, false); + check_state({1, 2, 3}, {true, false, true}, true, false); +} + +TEST(ColumnWithTypeAndNameTest, unnest_nullable_without_null_reuses_nested_column) { + auto nested_column = ColumnHelper::create_column({1, 2, 3}); + auto nullable_column = ColumnNullable::create( + nested_column, ColumnHelper::create_column({false, false, false})); + ColumnWithTypeAndName column_with_type_and_name { + std::move(nullable_column), + std::make_shared(std::make_shared()), "nullable"}; + + auto result = column_with_type_and_name.unnest_nullable( + column_with_type_and_name.get_nullable_column_info(), true); + + EXPECT_EQ(result.column.get(), nested_column.get()); +} + +TEST(ColumnWithTypeAndNameTest, unnest_nullable_with_unique_nested_replaces_data_in_place) { + auto nullable_column = ColumnNullable::create( + ColumnHelper::create_column({1, 2, 3}), + ColumnHelper::create_column({false, true, false})); + const auto* original_nested_column = + static_cast(*nullable_column).get_nested_column_ptr().get(); + ColumnWithTypeAndName column_with_type_and_name { + std::move(nullable_column), + std::make_shared(std::make_shared()), "nullable"}; + + const auto info = column_with_type_and_name.get_nullable_column_info(); + auto result = column_with_type_and_name.unnest_nullable(info, true); + + EXPECT_EQ(result.column.get(), original_nested_column); + EXPECT_EQ(assert_cast(*result.column).get_data()[1], 0); +} + +TEST(ColumnWithTypeAndNameTest, unnest_nullable_with_shared_nested_preserves_visible_alias) { + auto nested_column = ColumnHelper::create_column({1, 2, 3}); + auto nullable_column = ColumnNullable::create( + nested_column, ColumnHelper::create_column({false, true, false})); + auto visible_alias = ColumnNullable::create( + nested_column, ColumnHelper::create_column({false, false, false})); + ColumnWithTypeAndName column_with_type_and_name { + std::move(nullable_column), + std::make_shared(std::make_shared()), "nullable"}; + + const auto info = column_with_type_and_name.get_nullable_column_info(); + auto result = column_with_type_and_name.unnest_nullable(info, true); + + EXPECT_NE(result.column.get(), nested_column.get()); + EXPECT_EQ(assert_cast(*result.column).get_data()[1], 0); + EXPECT_FALSE(visible_alias->is_null_at(1)); + const ColumnNullable& visible_alias_column = *visible_alias; + EXPECT_EQ( + assert_cast(visible_alias_column.get_nested_column()).get_data()[1], + 2); +} + +TEST(ColumnWithTypeAndNameTest, unnest_nullable_with_shared_source_replaces_data_on_copy) { + auto nullable_column = ColumnNullable::create( + ColumnHelper::create_column({1, 2, 3}), + ColumnHelper::create_column({false, true, false})); + ColumnWithTypeAndName column_with_type_and_name { + std::move(nullable_column), + std::make_shared(std::make_shared()), "nullable"}; + ColumnPtr source_alias = column_with_type_and_name.column; + const auto& original_nested_column = + assert_cast(*source_alias).get_nested_column(); + + const auto info = column_with_type_and_name.get_nullable_column_info(); + auto result = column_with_type_and_name.unnest_nullable(info, true); + + EXPECT_NE(result.column.get(), &original_nested_column); + EXPECT_EQ(assert_cast(*result.column).get_data()[1], 0); + EXPECT_EQ(assert_cast(original_nested_column).get_data()[1], 2); +} + +TEST(ColumnWithTypeAndNameTest, unnest_const_nullable_with_shared_source_replaces_data_on_copy) { + auto nullable_column = + ColumnNullable::create(ColumnHelper::create_column({1}), + ColumnHelper::create_column({true})); + ColumnWithTypeAndName column_with_type_and_name { + ColumnConst::create(std::move(nullable_column), 3), + std::make_shared(std::make_shared()), "nullable"}; + ColumnPtr source_alias = column_with_type_and_name.column; + const auto& original_nullable_column = assert_cast( + assert_cast(*source_alias).get_data_column()); + + const auto info = column_with_type_and_name.get_nullable_column_info(); + auto result = column_with_type_and_name.unnest_nullable(info, true); + + EXPECT_TRUE(is_column_const(*result.column)); + EXPECT_EQ(result.column->get_int(0), 0); + EXPECT_EQ(original_nullable_column.get_nested_column().get_int(0), 1); } } // namespace doris diff --git a/be/test/exprs/function/function_arithmetic_test.cpp b/be/test/exprs/function/function_arithmetic_test.cpp index 09c66ba9bf8860..4d3829bf30abb5 100644 --- a/be/test/exprs/function/function_arithmetic_test.cpp +++ b/be/test/exprs/function/function_arithmetic_test.cpp @@ -32,6 +32,14 @@ namespace doris { +TEST(function_arithmetic_test, add_mixed_nullable_arguments_test) { + InputTypeSet input_types = {Nullable {PrimitiveType::TYPE_INT}, + Notnull {PrimitiveType::TYPE_INT}}; + DataSet data_set = {{{int32_t {1}, int32_t {2}}, int32_t {3}}, {{Null(), int32_t {4}}, Null()}}; + + static_cast(check_function("add", input_types, data_set)); +} + TEST(function_arithmetic_test, function_arithmetic_mod_test) { std::string func_name = "mod"; From 7f5ffa90adb2a43d41355aa726c0d01878652ab9 Mon Sep 17 00:00:00 2001 From: Mryange Date: Fri, 31 Jul 2026 16:14:46 +0800 Subject: [PATCH 2/3] [opt](exec) Avoid copies when publishing projection results (#66085) Projection results were converted to mutable columns before being published. Shared results such as SlotRef columns therefore triggered COW clones and full-column copies. This change keeps scoped output-block reuse while moving exclusive results directly and publishing shared immutable columns after restore. It also makes `ColumnConst` and `ColumnVariant` ownership checks include their nested columns, and uses actual peak memory tracking instead of charging shared input buffers as projection allocations. None - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label (cherry picked from commit c611239b7a3ee0c42c61136248a8fda44a838f43) --- be/src/core/column/column_const.h | 2 + be/src/core/column/column_variant.cpp | 14 ++++ be/src/core/column/column_variant.h | 2 + be/src/exec/operator/operator.cpp | 80 +++++++++--------- be/src/exec/operator/operator.h | 4 - be/src/exec/scan/scanner.cpp | 62 ++++++++------ be/test/core/column/column_const_test.cpp | 11 ++- be/test/core/column/column_nullable_test.cpp | 7 ++ be/test/core/column/column_variant_test.cpp | 38 ++++++--- .../operator/operator_projection_test.cpp | 81 +++++++++++++++++++ .../scan/scanner_late_arrival_rf_test.cpp | 45 +++++++++++ 11 files changed, 266 insertions(+), 80 deletions(-) create mode 100644 be/test/exec/operator/operator_projection_test.cpp diff --git a/be/src/core/column/column_const.h b/be/src/core/column/column_const.h index 6823416ad819f6..9a79d8d0cb5384 100644 --- a/be/src/core/column/column_const.h +++ b/be/src/core/column/column_const.h @@ -121,6 +121,8 @@ class ColumnConst final : public COWHelper { bool is_variable_length() const override { return data->is_variable_length(); } + bool is_exclusive() const override { return IColumn::is_exclusive() && data->is_exclusive(); } + std::string get_name() const override { return "Const(" + data->get_name() + ")"; } void resize(size_t new_size) override { s = new_size; } diff --git a/be/src/core/column/column_variant.cpp b/be/src/core/column/column_variant.cpp index 1efeb159b2a984..820fbe2794f11c 100644 --- a/be/src/core/column/column_variant.cpp +++ b/be/src/core/column/column_variant.cpp @@ -832,6 +832,20 @@ size_t ColumnVariant::allocated_bytes() const { return res; } +bool ColumnVariant::is_exclusive() const { + if (!IColumn::is_exclusive()) { + return false; + } + for (const auto& entry : subcolumns) { + for (const auto& part : entry->data.data) { + if (!part->is_exclusive()) { + return false; + } + } + } + return serialized_sparse_column->is_exclusive() && serialized_doc_value_column->is_exclusive(); +} + void ColumnVariant::for_each_subcolumn(ColumnCallback callback) { for (auto& entry : subcolumns) { for (auto& part : entry->data.data) { diff --git a/be/src/core/column/column_variant.h b/be/src/core/column/column_variant.h index 1d5c4eed1378a2..070ddc957f70fc 100644 --- a/be/src/core/column/column_variant.h +++ b/be/src/core/column/column_variant.h @@ -469,6 +469,8 @@ class ColumnVariant final : public COWHelper { bool has_enough_capacity(const IColumn& src) const override { return false; } + bool is_exclusive() const override; + void for_each_subcolumn(ColumnCallback callback) override; // Do nothing, call try_insert instead diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp index a7325981fb357e..1c072d9565c6aa 100644 --- a/be/src/exec/operator/operator.cpp +++ b/be/src/exec/operator/operator.cpp @@ -333,58 +333,54 @@ Status OperatorXBase::do_projections(RuntimeState* state, Block* origin_block, if (rows == 0) { return Status::OK(); } - Block input_block = *origin_block; - - size_t bytes_usage = 0; - ColumnsWithTypeAndName new_columns; - for (const auto& projections : local_state->_intermediate_projections) { - new_columns.resize(projections.size()); - for (int i = 0; i < projections.size(); i++) { - RETURN_IF_ERROR(projections[i]->execute(&input_block, new_columns[i])); - } - Block tmp_block {new_columns}; - bytes_usage += tmp_block.allocated_bytes(); - input_block.swap(tmp_block); - } - - DCHECK_EQ(rows, input_block.rows()); - auto insert_column_datas = [&](auto& to, ColumnPtr& from, size_t rows) { - if (to->is_nullable() && !from->is_nullable()) { - if (_keep_origin || !from->is_exclusive()) { - auto& null_column = reinterpret_cast(*to); - null_column.get_nested_column().insert_range_from(*from, 0, rows); - null_column.get_null_map_column().get_data().resize_fill(rows, 0); - bytes_usage += null_column.allocated_bytes(); - } else { - to = ColumnNullable::create(IColumn::mutate(std::move(from)), - ColumnUInt8::create(rows, 0)); - } - } else { - if (_keep_origin || !from->is_exclusive()) { - to->insert_range_from(*from, 0, rows); - bytes_usage += from->allocated_bytes(); - } else { - to = IColumn::mutate(std::move(from)); + SCOPED_PEAK_MEM(&local_state->_estimate_memory_usage); + + { + Block input_block = *origin_block; + + ColumnsWithTypeAndName new_columns; + for (const auto& projections : local_state->_intermediate_projections) { + new_columns.resize(projections.size()); + for (int i = 0; i < projections.size(); i++) { + RETURN_IF_ERROR(projections[i]->execute(&input_block, new_columns[i])); } + Block tmp_block {new_columns}; + input_block.swap(tmp_block); } - }; - auto scoped_mutable_block = VectorizedUtils::build_scoped_mutable_mem_reuse_block( - output_block, *_output_row_descriptor); - auto& mutable_block = scoped_mutable_block.mutable_block(); - auto& mutable_columns = mutable_block.mutable_columns(); - if (rows != 0) { + DCHECK_EQ(rows, input_block.rows()); + + auto scoped_mutable_block = VectorizedUtils::build_scoped_mutable_mem_reuse_block( + output_block, *_output_row_descriptor); + auto& mutable_columns = scoped_mutable_block.mutable_columns(); DCHECK_EQ(mutable_columns.size(), local_state->_projections.size()) << debug_string(); + Columns shared_columns(mutable_columns.size()); + for (int i = 0; i < mutable_columns.size(); ++i) { ColumnPtr column_ptr; RETURN_IF_ERROR(local_state->_projections[i]->execute(&input_block, column_ptr)); column_ptr = column_ptr->convert_to_full_column_if_const(); - bytes_usage += column_ptr->allocated_bytes(); - insert_column_datas(mutable_columns[i], column_ptr, rows); + if (is_column_nullable(*mutable_columns[i]) && !is_column_nullable(*column_ptr)) { + column_ptr = make_nullable(column_ptr, false); + } + if (column_ptr->is_exclusive()) { + mutable_columns[i] = IColumn::mutate(std::move(column_ptr)); + } else { + shared_columns[i] = std::move(column_ptr); + } + } + + scoped_mutable_block.restore(); + for (int i = 0; i < shared_columns.size(); ++i) { + if (shared_columns[i]) { + output_block->replace_by_position(i, std::move(shared_columns[i])); + } } - DCHECK(mutable_block.rows() == rows); } - local_state->_estimate_memory_usage += bytes_usage; + + origin_block->clear_column_data( + local_state->_parent->intermediate_row_desc().num_materialized_slots()); + DCHECK_EQ(output_block->rows(), rows); return Status::OK(); } diff --git a/be/src/exec/operator/operator.h b/be/src/exec/operator/operator.h index 676a8b67b18226..e193d57163b364 100644 --- a/be/src/exec/operator/operator.h +++ b/be/src/exec/operator/operator.h @@ -1020,10 +1020,6 @@ class OperatorXBase : public OperatorBase { std::string _op_name; int _parallel_tasks = 0; - //_keep_origin is used to avoid copying during projection, - // currently set to false only in the nestloop join. - bool _keep_origin = true; - // _blockable is true if the operator contains expressions that may block execution bool _blockable = false; }; diff --git a/be/src/exec/scan/scanner.cpp b/be/src/exec/scan/scanner.cpp index 41a1e0328ac23e..39e22312e75434 100644 --- a/be/src/exec/scan/scanner.cpp +++ b/be/src/exec/scan/scanner.cpp @@ -196,37 +196,51 @@ Status Scanner::_do_projections(Block* origin_block, Block* output_block) { if (rows == 0) { return Status::OK(); } - Block input_block = *origin_block; - std::vector result_column_ids; - for (auto& projections : _intermediate_projections) { - result_column_ids.resize(projections.size()); - for (int i = 0; i < projections.size(); i++) { - RETURN_IF_ERROR(projections[i]->execute(&input_block, &result_column_ids[i])); - } - input_block.shuffle_columns(result_column_ids); - } - - DCHECK_EQ(rows, input_block.rows()); - auto scoped_mutable_block = VectorizedUtils::build_scoped_mutable_mem_reuse_block( - output_block, *_output_row_descriptor); - auto& mutable_block = scoped_mutable_block.mutable_block(); + { + Block input_block = *origin_block; - auto& mutable_columns = mutable_block.mutable_columns(); + std::vector result_column_ids; + for (auto& projections : _intermediate_projections) { + result_column_ids.resize(projections.size()); + for (int i = 0; i < projections.size(); i++) { + RETURN_IF_ERROR(projections[i]->execute(&input_block, &result_column_ids[i])); + } + input_block.shuffle_columns(result_column_ids); + } - DCHECK_EQ(mutable_columns.size(), _projections.size()); + DCHECK_EQ(rows, input_block.rows()); + auto scoped_mutable_block = VectorizedUtils::build_scoped_mutable_mem_reuse_block( + output_block, *_output_row_descriptor); + auto& mutable_columns = scoped_mutable_block.mutable_columns(); + DCHECK_EQ(mutable_columns.size(), _projections.size()); + Columns shared_columns(mutable_columns.size()); + + for (int i = 0; i < mutable_columns.size(); ++i) { + ColumnPtr column_ptr; + RETURN_IF_ERROR(_projections[i]->execute(&input_block, column_ptr)); + column_ptr = column_ptr->convert_to_full_column_if_const(); + if (mutable_columns[i]->is_nullable() != column_ptr->is_nullable()) { + throw Exception(ErrorCode::INTERNAL_ERROR, "Nullable mismatch"); + } + if (column_ptr->is_exclusive()) { + mutable_columns[i] = IColumn::mutate(std::move(column_ptr)); + } else { + shared_columns[i] = std::move(column_ptr); + } + } - for (int i = 0; i < mutable_columns.size(); ++i) { - ColumnPtr column_ptr; - RETURN_IF_ERROR(_projections[i]->execute(&input_block, column_ptr)); - column_ptr = column_ptr->convert_to_full_column_if_const(); - if (mutable_columns[i]->is_nullable() != column_ptr->is_nullable()) { - throw Exception(ErrorCode::INTERNAL_ERROR, "Nullable mismatch"); + scoped_mutable_block.restore(); + for (int i = 0; i < shared_columns.size(); ++i) { + if (shared_columns[i]) { + output_block->replace_by_position(i, std::move(shared_columns[i])); + } } - mutable_columns[i] = IColumn::mutate(std::move(column_ptr)); } - scoped_mutable_block.restore(); + origin_block->clear_column_data( + _local_state->_parent->row_descriptor().num_materialized_slots()); + DCHECK_EQ(output_block->rows(), rows); return Status::OK(); } diff --git a/be/test/core/column/column_const_test.cpp b/be/test/core/column/column_const_test.cpp index e9f57df213bce3..cc9980c654d66c 100644 --- a/be/test/core/column/column_const_test.cpp +++ b/be/test/core/column/column_const_test.cpp @@ -41,6 +41,15 @@ TEST(ColumnConstTest, TestCreate) { EXPECT_TRUE(!is_column_const(column_const2->get_data_column())); } +TEST(ColumnConstTest, IsExclusiveChecksNestedColumn) { + auto column_data = ColumnHelper::create_column({7}); + auto column_const = ColumnConst::create(column_data, 3); + + EXPECT_FALSE(column_const->is_exclusive()); + column_data.reset(); + EXPECT_TRUE(column_const->is_exclusive()); +} + TEST(ColumnConstTest, clone_resized_clones_nested_data) { auto column_data = ColumnHelper::create_column({7}); auto column_const = ColumnConst::create(column_data, 3); @@ -322,4 +331,4 @@ TEST(ColumnConstTest, replace_float_special_values) { column_const->finalize(); } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/core/column/column_nullable_test.cpp b/be/test/core/column/column_nullable_test.cpp index 77f167c9ea8f2e..088e2071795228 100644 --- a/be/test/core/column/column_nullable_test.cpp +++ b/be/test/core/column/column_nullable_test.cpp @@ -136,6 +136,13 @@ TEST(ColumnNullableTest, SharedCreatePreservesImmutableSubcolumns) { EXPECT_EQ(nullable_ref.get_null_map_column_ptr().get(), null_map_alias.get()); EXPECT_EQ(nested_alias->size(), 1); EXPECT_EQ(null_map_alias->size(), 1); + EXPECT_FALSE(nullable->is_exclusive()); + + nested.reset(); + nested_alias.reset(); + null_map.reset(); + null_map_alias.reset(); + EXPECT_TRUE(nullable->is_exclusive()); } TEST(ColumnNullableTest, UpdateCrc32cBatchKeepsBlockInsertable) { diff --git a/be/test/core/column/column_variant_test.cpp b/be/test/core/column/column_variant_test.cpp index b7c901c8c71b29..6f5568730f3658 100644 --- a/be/test/core/column/column_variant_test.cpp +++ b/be/test/core/column/column_variant_test.cpp @@ -1832,16 +1832,36 @@ TEST_F(ColumnVariantTest, is_scalar_variant) { } TEST_F(ColumnVariantTest, is_exclusive) { - auto test_func = [](const auto& source_column) { - auto src_size = source_column->size(); - EXPECT_TRUE(src_size > 0); + auto variant = VariantUtil::construct_basic_varint_column(); + EXPECT_GT(variant->size(), 0); + EXPECT_TRUE(variant->is_exclusive()); - // Test is_exclusive - bool is_exclusive = source_column->is_exclusive(); - // The result depends on the actual data structure - EXPECT_TRUE(is_exclusive); - }; - test_func(column_variant); + const auto& subcolumns = variant->get_subcolumns(); + const auto* root = subcolumns.get_root(); + ColumnPtr shared_subcolumn; + for (const auto& entry : subcolumns) { + if (entry.get() != root && !entry->data.data.empty()) { + shared_subcolumn = static_cast(entry->data.data[0]); + break; + } + } + ASSERT_TRUE(shared_subcolumn); + EXPECT_FALSE(variant->is_exclusive()); + + shared_subcolumn.reset(); + EXPECT_TRUE(variant->is_exclusive()); + + auto shared_sparse_column = variant->get_sparse_column(); + EXPECT_FALSE(variant->is_exclusive()); + + shared_sparse_column.reset(); + EXPECT_TRUE(variant->is_exclusive()); + + auto shared_doc_value_column = variant->get_doc_value_column(); + EXPECT_FALSE(variant->is_exclusive()); + + shared_doc_value_column.reset(); + EXPECT_TRUE(variant->is_exclusive()); } TEST_F(ColumnVariantTest, get_root_type) { diff --git a/be/test/exec/operator/operator_projection_test.cpp b/be/test/exec/operator/operator_projection_test.cpp new file mode 100644 index 00000000000000..02ddc5f7f9ef55 --- /dev/null +++ b/be/test/exec/operator/operator_projection_test.cpp @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "common/object_pool.h" +#include "core/data_type/data_type_number.h" +#include "exec/operator/mock_operator.h" +#include "runtime/runtime_profile.h" +#include "testutil/column_helper.h" +#include "testutil/mock/mock_descriptors.h" +#include "testutil/mock/mock_runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" + +namespace doris { + +TEST(OperatorProjectionTest, PublishesSharedColumnAndReusesOutputBlock) { + ObjectPool pool; + auto data_type = std::make_shared(); + auto row_descriptor = MockRowDescriptor({data_type}, &pool); + + MockOperatorX op; + op._row_descriptor = row_descriptor; + op._output_row_descriptor = + std::make_unique(std::vector {data_type}, &pool); + + MockRuntimeState state; + const auto max_operator_id = op.operator_id() - 1; + state.resize_op_id_to_local_state(max_operator_id); + state.set_max_operator_id(max_operator_id); + RuntimeProfile parent_profile("parent"); + LocalStateInfo info {&parent_profile, {}, nullptr, {}, 0}; + ASSERT_TRUE(op.setup_local_state(&state, info).ok()); + + auto* local_state = state.get_local_state(op.operator_id()); + local_state->_projections = MockSlotRef::create_mock_contexts(0, data_type); + + std::vector first_values(1 << 18, 7); + Block first_origin = ColumnHelper::create_block(first_values); + const auto* first_column = first_origin.get_by_position(0).column.get(); + const auto first_allocated_bytes = static_cast(first_origin.allocated_bytes()); + + Block output; + ASSERT_TRUE(op.do_projections(&state, &first_origin, &output).ok()); + EXPECT_EQ(output.get_by_position(0).column.get(), first_column); + EXPECT_EQ(output.rows(), first_values.size()); + EXPECT_EQ(output.get_by_position(0).column->get_int(0), 7); + EXPECT_EQ(first_origin.rows(), 0); + EXPECT_LT(local_state->estimate_memory_usage(), first_allocated_bytes); + + output.clear_column_data(); + Block second_origin = ColumnHelper::create_block({8, 9}); + const auto* second_column = second_origin.get_by_position(0).column.get(); + + ASSERT_TRUE(op.do_projections(&state, &second_origin, &output).ok()); + EXPECT_EQ(output.get_by_position(0).column.get(), second_column); + EXPECT_EQ(output.rows(), 2); + EXPECT_EQ(output.get_by_position(0).column->get_int(0), 8); + EXPECT_EQ(output.get_by_position(0).column->get_int(1), 9); + EXPECT_EQ(second_origin.rows(), 0); +} + +} // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 51729f2eb3541a..2b7a7154f24e27 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -213,4 +213,49 @@ TEST(ScannerProjectionTest, projects_incompatible_blocks_before_reading_the_next EXPECT_EQ(final_output.rows(), 0); } +TEST(ScannerProjectionTest, publishes_shared_column_and_reuses_output_block) { + ObjectPool pool; + auto data_type = std::make_shared(); + auto row_descriptor = MockRowDescriptor({data_type}, &pool); + + MockRuntimeState state; + state._batch_size = 4; + + auto op = std::make_shared(); + op->_row_descriptor = row_descriptor; + op->_output_row_descriptor = + std::make_unique(std::vector {data_type}, &pool); + op->_output_tuple_desc = op->_output_row_descriptor->tuple_descriptors()[0]; + + auto local_state = std::make_shared(&state, op.get()); + local_state->_projections = MockSlotRef::create_mock_contexts(0, data_type); + + RuntimeProfile profile("scanner"); + TestScanner scanner(&state, local_state.get(), -1, &profile); + ASSERT_TRUE(scanner.init(&state, {}).ok()); + + Block first_input = ColumnHelper::create_block({1, 2}); + const auto* first_column = first_input.get_by_position(0).column.get(); + scanner.add_block(std::move(first_input)); + + Block second_input = ColumnHelper::create_block({3, 4}); + const auto* second_column = second_input.get_by_position(0).column.get(); + scanner.add_block(std::move(second_input)); + + Block output; + bool eos = false; + ASSERT_TRUE(scanner.get_block_after_projects(&state, &output, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(output.get_by_position(0).column.get(), first_column); + EXPECT_EQ(output.get_by_position(0).column->get_int(0), 1); + EXPECT_EQ(output.get_by_position(0).column->get_int(1), 2); + + output.clear_column_data(); + ASSERT_TRUE(scanner.get_block_after_projects(&state, &output, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(output.get_by_position(0).column.get(), second_column); + EXPECT_EQ(output.get_by_position(0).column->get_int(0), 3); + EXPECT_EQ(output.get_by_position(0).column->get_int(1), 4); +} + } // namespace doris From 91c227f61b1d028bff96e1bbcefd89885af77bae Mon Sep 17 00:00:00 2001 From: Mryange Date: Sat, 8 Aug 2026 20:41:35 +0800 Subject: [PATCH 3/3] fix --- be/src/core/block/column_with_type_and_name.cpp | 9 +++++++++ be/src/core/block/column_with_type_and_name.h | 1 + 2 files changed, 10 insertions(+) diff --git a/be/src/core/block/column_with_type_and_name.cpp b/be/src/core/block/column_with_type_and_name.cpp index 917bfd3d1421ae..028f91fabb0dbd 100644 --- a/be/src/core/block/column_with_type_and_name.cpp +++ b/be/src/core/block/column_with_type_and_name.cpp @@ -127,6 +127,15 @@ NullableColumnInfo ColumnWithTypeAndName::get_nullable_column_info() const { .is_nullable = true}; } +ColumnWithTypeAndName ColumnWithTypeAndName::unnest_nullable( + bool replace_null_data_to_default) const { + NullableColumnInfo info; + if (type->is_nullable()) { + info = get_nullable_column_info(); + } + return unnest_nullable(info, replace_null_data_to_default); +} + ColumnWithTypeAndName ColumnWithTypeAndName::unnest_nullable( const NullableColumnInfo& info, bool replace_null_data_to_default) const { if (!type->is_nullable()) { diff --git a/be/src/core/block/column_with_type_and_name.h b/be/src/core/block/column_with_type_and_name.h index d6a1954101b18a..577c8edf6234c9 100644 --- a/be/src/core/block/column_with_type_and_name.h +++ b/be/src/core/block/column_with_type_and_name.h @@ -83,6 +83,7 @@ struct ColumnWithTypeAndName { const ColumnUInt8::Ptr& get_nullable_null_map_column() const; NullableColumnInfo get_nullable_column_info() const; + ColumnWithTypeAndName unnest_nullable(bool replace_null_data_to_default = false) const; ColumnWithTypeAndName unnest_nullable(const NullableColumnInfo& info, bool replace_null_data_to_default) const;