Skip to content

branch-4.1: [feature](variant) Support ColumnVariantV2 segment reads and writes #66607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
eldenmoon wants to merge 2 commits into
base: branch-4.1
Choose a base branch
from
Open

branch-4.1: [feature](variant) Support ColumnVariantV2 segment reads and writes #66607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f5286b9
[feature](variant) Support ColumnVariantV2 segment reads and writes
eldenmoon Aug 11, 2026
e0e3bec
[fix](be) Use JSON serialization in Variant Parquet test
eldenmoon Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
118 changes: 31 additions & 87 deletions be/src/core/column/column_variant.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
#include "core/column/column_array.h"
#include "core/column/column_string.h"
#include "core/column/column_vector.h"
#include "core/column/variant_column_utils.h"
#include "core/data_type/convert_field_to_type.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_decimal.h"
Expand All @@ -74,7 +75,6 @@
#include "util/jsonb_document.h"
#include "util/jsonb_document_cast.h"
#include "util/jsonb_parser_simd.h"
#include "util/jsonb_utils.h"
#include "util/simd/bits.h"
#include "util/unaligned.h"

Expand Down Expand Up @@ -883,7 +883,7 @@ void ColumnVariant::insert_from(const IColumn& src, size_t n) {

void ColumnVariant::try_insert(const Field& field) {
size_t old_size = size();
const auto& object = field.get<TYPE_VARIANT>();
const auto& object = field.get<TYPE_VARIANT>().legacy_map();
for (const auto& [key, value] : object) {
if (key.get_path() == "__DORIS_VARIANT_DOC_VALUE__") {
insert_to_doc_value_column(value.field);
Expand Down Expand Up @@ -1074,7 +1074,7 @@ void ColumnVariant::get(size_t n, Field& res) const {
size());
}
res = Field::create_field<TYPE_VARIANT>(VariantMap());
auto& object = res.get<TYPE_VARIANT>();
auto& object = res.get<TYPE_VARIANT>().legacy_map();

for (const auto& entry : subcolumns) {
FieldWithDataType field;
Expand Down Expand Up @@ -1122,7 +1122,7 @@ void ColumnVariant::try_get_from_doc_value_column(size_t n, Field& res) const {
CHECK(subcolumns.size() == 1) << "subcolumns size should be 1";
FieldWithDataType field_with_data_type;
serialized_doc_value_column->get(n, field_with_data_type.field);
auto& object = res.get<TYPE_VARIANT>();
auto& object = res.get<TYPE_VARIANT>().legacy_map();
object.try_emplace(PathInData("__DORIS_VARIANT_DOC_VALUE__"), std::move(field_with_data_type));
}

Expand Down Expand Up @@ -1703,31 +1703,6 @@ enum class NestedJsonSkipKind {
kInvalidType,
};

static bool is_semantically_empty_jsonb_value(const JsonbValue* value) {
if (value == nullptr || value->isNull()) {
return true;
}
if (value->isArray()) {
const auto* array = value->unpack<ArrayVal>();
for (auto it = array->begin(); it != array->end(); ++it) {
if (!is_semantically_empty_jsonb_value(&*it)) {
return false;
}
}
return true;
}
if (value->isObject()) {
const auto* object = value->unpack<ObjectVal>();
for (auto it = object->begin(); it != object->end(); ++it) {
if (!is_semantically_empty_jsonb_value(it->value())) {
return false;
}
}
return true;
}
return false;
}

static bool is_semantically_empty_nested_field(const Field& field) {
switch (field.get_type()) {
case PrimitiveType::TYPE_NULL:
Expand All @@ -1742,7 +1717,7 @@ static bool is_semantically_empty_nested_field(const Field& field) {
return true;
}
case PrimitiveType::TYPE_VARIANT: {
const auto& object = field.get<TYPE_VARIANT>();
const auto& object = field.get<TYPE_VARIANT>().legacy_map();
for (const auto& [_, value] : object) {
if (!is_semantically_empty_nested_field(value.field)) {
return false;
Expand All @@ -1758,7 +1733,7 @@ static bool is_semantically_empty_nested_field(const Field& field) {
if (!st.ok() || doc == nullptr) {
return false;
}
return is_semantically_empty_jsonb_value(doc->getValue());
return is_variant_jsonb_value_semantically_empty(doc->getValue());
}
default:
return false;
Expand Down Expand Up @@ -2695,58 +2670,6 @@ bool ColumnVariant::try_insert_default_from_nested(const Subcolumns::NodePtr& en
return true;
}

size_t ColumnVariant::find_path_lower_bound_in_sparse_data(StringRef path,
const ColumnString& sparse_data_paths,
size_t start, size_t end) {
// Simple random access iterator over values in ColumnString in specified range.
class Iterator {
public:
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-local-typedefs"
using difference_type = std::ptrdiff_t;
using value_type = StringRef;
using iterator_category = std::random_access_iterator_tag;
using pointer = StringRef*;
using reference = StringRef&;
#pragma GCC diagnostic pop

Iterator() = delete;
Iterator(const ColumnString* data_, size_t index_) : data(data_), index(index_) {}
Iterator(const Iterator& rhs) = default;
Iterator& operator=(const Iterator& rhs) = default;
inline Iterator& operator+=(difference_type rhs) {
index += rhs;
return *this;
}
inline StringRef operator*() const { return data->get_data_at(index); }

inline Iterator& operator++() {
++index;
return *this;
}
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-member-function"
#endif
inline Iterator& operator--() {
--index;
return *this;
}
#ifdef __clang__
#pragma clang diagnostic pop
#endif
inline difference_type operator-(const Iterator& rhs) const { return index - rhs.index; }

const ColumnString* data;
size_t index;
};

Iterator start_it(&sparse_data_paths, start);
Iterator end_it(&sparse_data_paths, end);
auto it = std::lower_bound(start_it, end_it, path);
return it.index;
}

void ColumnVariant::Subcolumn::deserialize_from_binary_column(const ColumnString* value,
size_t row) {
const auto& data_ref = value->get_data_at(row);
Expand Down Expand Up @@ -2785,6 +2708,26 @@ void ColumnVariant::Subcolumn::deserialize_from_binary_column(const ColumnString
}
}

void ColumnVariant::Subcolumn::deserialize_nullable_from_binary_column(const ColumnString* value,
size_t row) {
auto [field, info] = ColumnVariant::deserialize_from_binary_column(value, row);
bool is_null = field.get_type() == PrimitiveType::TYPE_NULL;
if (field.get_type() == PrimitiveType::TYPE_JSONB) {
const auto& jsonb = field.get<TYPE_JSONB>();
const JsonbDocument* document = nullptr;
const Status status = JsonbDocument::checkAndCreateDocument(jsonb.get_value(),
jsonb.get_size(), &document);
DORIS_CHECK(status.ok()) << status;
DORIS_CHECK(document != nullptr);
is_null = document->getValue()->isNull();
}
if (is_null) {
insert_default();
} else {
insert(std::move(field), std::move(info));
}
}

void ColumnVariant::fill_path_column_from_sparse_data(Subcolumn& subcolumn, NullMap* null_map,
StringRef path,
const ColumnPtr& sparse_data_column,
Expand All @@ -2807,13 +2750,14 @@ void ColumnVariant::fill_path_column_from_sparse_data(Subcolumn& subcolumn, Null
for (size_t i = start; i != end; ++i) {
size_t paths_start = sparse_data_offsets[static_cast<ssize_t>(i) - 1];
size_t paths_end = sparse_data_offsets[static_cast<ssize_t>(i)];
auto lower_bound_path_index = ColumnVariant::find_path_lower_bound_in_sparse_data(
path, sparse_data_paths, paths_start, paths_end);
auto lower_bound_path_index = find_variant_sparse_path_lower_bound(path, sparse_data_paths,
paths_start, paths_end);
bool is_null = false;
if (lower_bound_path_index != paths_end &&
sparse_data_paths.get_data_at(lower_bound_path_index) == path) {
subcolumn.deserialize_from_binary_column(&sparse_data_values, lower_bound_path_index);
is_null = false;
subcolumn.deserialize_nullable_from_binary_column(&sparse_data_values,
lower_bound_path_index);
is_null = subcolumn.is_null_at(subcolumn.size() - 1);
} else {
subcolumn.insert_default();
is_null = true;
Expand Down
8 changes: 4 additions & 4 deletions be/src/core/column/column_variant.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,10 @@ class ColumnVariant final : public COWHelper<IColumn, ColumnVariant> {

void deserialize_from_binary_column(const ColumnString* value, size_t row);

// Used only when a storage cell is the requested logical value. SQL NULL and encoded
// JSON null both become a missing value; object children keep their encoded null.
void deserialize_nullable_from_binary_column(const ColumnString* value, size_t row);

/// Returns single column if subcolumn in finalizes.
/// Otherwise -- undefined behaviour.
IColumn& get_finalized_column();
Expand Down Expand Up @@ -614,10 +618,6 @@ class ColumnVariant final : public COWHelper<IColumn, ColumnVariant> {
const ColumnPtr& sparse_data_column, size_t start,
size_t end);

static size_t find_path_lower_bound_in_sparse_data(StringRef path,
const ColumnString& sparse_data_paths,
size_t start, size_t end);

// Deserialize the i-th row of the column from the sparse column.
static std::pair<Field, FieldInfo> deserialize_from_binary_column(const ColumnString* value,
size_t row);
Expand Down
63 changes: 63 additions & 0 deletions be/src/core/column/variant_column_utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "core/column/variant_column_utils.h"

#include "core/column/column_string.h"
#include "util/jsonb_document.h"

namespace doris {

size_t find_variant_sparse_path_lower_bound(StringRef path, const ColumnString& sparse_paths,
size_t start, size_t end) {
while (start < end) {
const size_t middle = start + (end - start) / 2;
if (sparse_paths.get_data_at(middle) < path) {
start = middle + 1;
} else {
end = middle;
}
}
return start;
}

bool is_variant_jsonb_value_semantically_empty(const JsonbValue* value) {
if (value == nullptr || value->isNull()) {
return true;
}
if (value->isArray()) {
const auto* array = value->unpack<ArrayVal>();
for (auto it = array->begin(); it != array->end(); ++it) {
if (!is_variant_jsonb_value_semantically_empty(&*it)) {
return false;
}
}
return true;
}
if (value->isObject()) {
const auto* object = value->unpack<ObjectVal>();
for (auto it = object->begin(); it != object->end(); ++it) {
if (!is_variant_jsonb_value_semantically_empty(it->value())) {
return false;
}
}
return true;
}
return false;
}

} // namespace doris
37 changes: 37 additions & 0 deletions be/src/core/column/variant_column_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <cstddef>

#include "core/column/column_string.h"
#include "core/string_ref.h"

namespace doris {

struct JsonbValue;

// Returns the first sorted sparse path in [start, end) that is not less than path.
size_t find_variant_sparse_path_lower_bound(StringRef path, const ColumnString& sparse_paths,
size_t start, size_t end);

// True when a JSONB value recursively contains no visible scalar payload. Variant V1 and V2
// readers share this rule when deciding whether a materialized nested value contributes a path.
bool is_variant_jsonb_value_semantically_empty(const JsonbValue* value);

} // namespace doris
Loading
Loading