Skip to content

[improvement](cloud) Remove pre-rowset delete bitmaps by key #66598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mymeiyi wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[improvement](cloud) Remove pre-rowset delete bitmaps by key #66598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions be/src/cloud/cloud_cumulative_compaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
}
}
// agg delete bitmap for pre rowsets
if (config::enable_agg_and_remove_pre_rowsets_delete_bitmap &&
if (config::delete_bitmap_store_write_version != 2 &&
config::enable_agg_and_remove_pre_rowsets_delete_bitmap &&
_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) {
OlapStopWatch watch;
Expand All @@ -425,17 +426,23 @@ Status CloudCumulativeCompaction::modify_rowsets() {
std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
auto pre_rowsets_delete_bitmap = std::make_shared<DeleteBitmap>(_tablet->tablet_id());
std::map<std::string, int64_t> pre_rowset_to_versions;
std::unique_ptr<CloudTablet::PreRowsetDeleteBitmapStats> pre_rowset_delete_bitmap_stats;
if (config::enable_remove_pre_rowsets_delete_bitmap_by_keys) {
Comment thread
mymeiyi marked this conversation as resolved.
pre_rowset_delete_bitmap_stats =
std::make_unique<CloudTablet::PreRowsetDeleteBitmapStats>();
}
cloud_tablet()->agg_delete_bitmap_for_compaction(
_output_rowset->start_version(), _output_rowset->end_version(), pre_rowsets,
pre_rowsets_delete_bitmap, pre_rowset_to_versions);
pre_rowsets_delete_bitmap, pre_rowset_to_versions,
pre_rowset_delete_bitmap_stats.get());
// update delete bitmap to ms
DBUG_EXECUTE_IF(
"CumulativeCompaction.modify_rowsets.cloud_update_delete_bitmap_without_lock.block",
DBUG_BLOCK);
auto status = _engine.meta_mgr().cloud_update_delete_bitmap_without_lock(
*cloud_tablet(), pre_rowsets_delete_bitmap.get(), pre_rowset_to_versions,
cloud_tablet()->table_id(), _output_rowset->start_version(),
_output_rowset->end_version());
pre_rowset_delete_bitmap_stats.get(), cloud_tablet()->table_id(),
_output_rowset->start_version(), _output_rowset->end_version());
if (!status.ok()) {
LOG(WARNING) << "failed to agg pre rowsets delete bitmap to ms. tablet_id="
<< _tablet->tablet_id() << ", pre rowset num=" << pre_rowsets.size()
Expand Down
20 changes: 18 additions & 2 deletions be/src/cloud/cloud_meta_mgr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2126,8 +2126,10 @@ Status CloudMetaMgr::update_delete_bitmap(const CloudTablet& tablet, int64_t loc

Status CloudMetaMgr::cloud_update_delete_bitmap_without_lock(
const CloudTablet& tablet, DeleteBitmap* delete_bitmap,
std::map<std::string, int64_t>& rowset_to_versions, int64_t table_id,
int64_t pre_rowset_agg_start_version, int64_t pre_rowset_agg_end_version) {
std::map<std::string, int64_t>& rowset_to_versions,
const CloudTablet::PreRowsetDeleteBitmapStats* pre_rowset_delete_bitmap_stats,
int64_t table_id, int64_t pre_rowset_agg_start_version,
int64_t pre_rowset_agg_end_version) {
if (config::delete_bitmap_store_write_version == 2) {
VLOG_DEBUG << "no need to agg delete bitmap v1 in ms because use v2";
return Status::OK();
Expand Down Expand Up @@ -2166,6 +2168,20 @@ Status CloudMetaMgr::cloud_update_delete_bitmap_without_lock(
req.set_pre_rowset_agg_start_version(pre_rowset_agg_start_version);
req.set_pre_rowset_agg_end_version(pre_rowset_agg_end_version);
}
if (pre_rowset_delete_bitmap_stats != nullptr) {
for (const auto& [rowset_id, delete_bitmap_stats] : *pre_rowset_delete_bitmap_stats) {
auto* rowset_stats_pb = req.add_pre_rowset_delete_bitmap_stats();
rowset_stats_pb->set_rowset_id(rowset_id);
for (const auto& [segment_id, version, delete_bitmap_size] : delete_bitmap_stats) {
auto* delete_bitmap_stat_pb = rowset_stats_pb->add_delete_bitmap_stats();
delete_bitmap_stat_pb->set_segment_id(segment_id);
delete_bitmap_stat_pb->set_version(version);
delete_bitmap_stat_pb->set_delete_bitmap_size(delete_bitmap_size);
}
}
}
TEST_SYNC_POINT_RETURN_WITH_VALUE(
"CloudMetaMgr::cloud_update_delete_bitmap_without_lock.before_rpc", Status::OK(), &req);
return retry_rpc(MetaServiceRPC::UPDATE_DELETE_BITMAP, req, &res,
&MetaService_Stub::update_delete_bitmap,
{
Expand Down
6 changes: 4 additions & 2 deletions be/src/cloud/cloud_meta_mgr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,10 @@ class CloudMetaMgr {

Status cloud_update_delete_bitmap_without_lock(
const CloudTablet& tablet, DeleteBitmap* delete_bitmap,
std::map<std::string, int64_t>& rowset_to_versions, int64_t table_id,
int64_t pre_rowset_agg_start_version = 0, int64_t pre_rowset_agg_end_version = 0);
std::map<std::string, int64_t>& rowset_to_versions,
const CloudTablet::PreRowsetDeleteBitmapStats* pre_rowset_delete_bitmap_stats,
int64_t table_id, int64_t pre_rowset_agg_start_version = 0,
int64_t pre_rowset_agg_end_version = 0);

Status get_delete_bitmap_update_lock(const CloudTablet& tablet, int64_t lock_id,
int64_t initiator);
Expand Down
32 changes: 27 additions & 5 deletions be/src/cloud/cloud_tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1432,12 +1432,34 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(

void CloudTablet::agg_delete_bitmap_for_compaction(
int64_t start_version, int64_t end_version, const std::vector<RowsetSharedPtr>& pre_rowsets,
DeleteBitmapPtr& new_delete_bitmap,
std::map<std::string, int64_t>& pre_rowset_to_versions) {
for (auto& rowset : pre_rowsets) {
DeleteBitmapPtr& new_delete_bitmap, std::map<std::string, int64_t>& pre_rowset_to_versions,
PreRowsetDeleteBitmapStats* pre_rowset_delete_bitmap_stats) {
auto& delete_bitmap = tablet_meta()->delete_bitmap();
for (const auto& rowset : pre_rowsets) {
if (pre_rowset_delete_bitmap_stats != nullptr) {
auto& rowset_delete_bitmap_stats =
(*pre_rowset_delete_bitmap_stats)[rowset->rowset_id().to_string()];
std::shared_lock lock(delete_bitmap.lock);
const auto bitmap_start_version = static_cast<DeleteBitmap::Version>(start_version);
const auto bitmap_end_version = static_cast<DeleteBitmap::Version>(end_version);
for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
DeleteBitmap::BitmapKey segment_start {rowset->rowset_id(), seg_id,
bitmap_start_version};
for (auto it = delete_bitmap.delete_bitmap.lower_bound(segment_start);
it != delete_bitmap.delete_bitmap.end(); ++it) {
const auto& [key, bitmap] = *it;
if (std::get<0>(key) != rowset->rowset_id() || std::get<1>(key) != seg_id ||
std::get<2>(key) >= bitmap_end_version) {
break;
}
rowset_delete_bitmap_stats.emplace_back(seg_id, std::get<2>(key),
bitmap.getSizeInBytes());
}
}
}
for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
auto d = tablet_meta()->delete_bitmap().get_agg_without_cache(
{rowset->rowset_id(), seg_id, end_version}, start_version);
auto d = delete_bitmap.get_agg_without_cache({rowset->rowset_id(), seg_id, end_version},
start_version);
if (d->isEmpty()) {
continue;
}
Expand Down
15 changes: 11 additions & 4 deletions be/src/cloud/cloud_tablet.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#pragma once

#include <memory>
#include <tuple>

#include "storage/partial_update_info.h"
#include "storage/rowset/rowset.h"
Expand Down Expand Up @@ -76,6 +77,11 @@ struct RecycledRowsets {

class CloudTablet final : public BaseTablet {
public:
// rowset id -> [(segment id, version, serialized delete bitmap size)]
using PreRowsetDeleteBitmapStats = std::map<
std::string,
std::vector<std::tuple<DeleteBitmap::SegmentId, DeleteBitmap::Version, size_t>>>;

CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta);

~CloudTablet() override;
Expand Down Expand Up @@ -357,10 +363,11 @@ class CloudTablet final : public BaseTablet {
// check that if the delete bitmap in delete bitmap cache has the same cardinality with the expected_delete_bitmap's
Status check_delete_bitmap_cache(int64_t txn_id, DeleteBitmap* expected_delete_bitmap) override;

void agg_delete_bitmap_for_compaction(int64_t start_version, int64_t end_version,
const std::vector<RowsetSharedPtr>& pre_rowsets,
DeleteBitmapPtr& new_delete_bitmap,
std::map<std::string, int64_t>& pre_rowset_to_versions);
void agg_delete_bitmap_for_compaction(
int64_t start_version, int64_t end_version,
const std::vector<RowsetSharedPtr>& pre_rowsets, DeleteBitmapPtr& new_delete_bitmap,
std::map<std::string, int64_t>& pre_rowset_to_versions,
PreRowsetDeleteBitmapStats* pre_rowset_delete_bitmap_stats);

bool need_remove_unused_rowsets();

Expand Down
2 changes: 2 additions & 0 deletions be/src/common/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1431,6 +1431,8 @@ DEFINE_mBool(enable_mow_get_agg_by_cache, "true");
DEFINE_mBool(enable_mow_get_agg_correctness_check_core, "false");
DEFINE_mBool(enable_agg_and_remove_pre_rowsets_delete_bitmap, "true");
DEFINE_mBool(enable_check_agg_and_remove_pre_rowsets_delete_bitmap, "false");
// Send pre-rowset delete bitmap stats so Meta Service removes the split keys individually.
DEFINE_mBool(enable_remove_pre_rowsets_delete_bitmap_by_keys, "true");

// The secure path with user files, used in the `local` table function.
DEFINE_String(user_files_secure_path, "${DORIS_HOME}");
Expand Down
1 change: 1 addition & 0 deletions be/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1523,6 +1523,7 @@ DECLARE_mBool(enable_mow_get_agg_by_cache);
DECLARE_mBool(enable_mow_get_agg_correctness_check_core);
DECLARE_mBool(enable_agg_and_remove_pre_rowsets_delete_bitmap);
DECLARE_mBool(enable_check_agg_and_remove_pre_rowsets_delete_bitmap);
DECLARE_mBool(enable_remove_pre_rowsets_delete_bitmap_by_keys);

// The secure path with user files, used in the `local` table function.
DECLARE_String(user_files_secure_path);
Expand Down
75 changes: 75 additions & 0 deletions be/test/cloud/cloud_meta_mgr_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@

#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
#include "cloud/config.h"
#include "cpp/sync_point.h"
#include "load/stream_load/stream_load_context.h"
#include "storage/olap_common.h"
#include "storage/rowset/rowset_factory.h"
#include "storage/rowset/rowset_meta.h"
#include "storage/tablet/tablet_meta.h"
#include "util/defer_op.h"
#include "util/uid_util.h"

namespace doris {
Expand Down Expand Up @@ -67,6 +69,79 @@ TEST_F(CloudMetaMgrTest, response_status_falls_back_for_invalid_actual_code) {
EXPECT_EQ(get_response_code(status), MetaServiceCode::KV_TXN_CONFLICT);
}

TEST_F(CloudMetaMgrTest, PreRowsetDeleteBitmapStatsRequestEncoding) {
CloudStorageEngine engine(EngineOptions {});
CloudMetaMgr meta_mgr;
TabletMetaSharedPtr tablet_meta(
new TabletMeta(1001, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F));
auto tablet = std::make_shared<CloudTablet>(engine, std::make_shared<TabletMeta>(*tablet_meta));
DeleteBitmap delete_bitmap(tablet->tablet_id());
std::map<std::string, int64_t> rowset_to_versions;

int32_t old_write_version = config::delete_bitmap_store_write_version;
config::delete_bitmap_store_write_version = 1;
auto* sp = SyncPoint::get_instance();
sp->clear_all_call_backs();
sp->enable_processing();
Defer cleanup {[&] {
config::delete_bitmap_store_write_version = old_write_version;
sp->disable_processing();
sp->clear_all_call_backs();
}};

auto capture_request = [&](const CloudTablet::PreRowsetDeleteBitmapStats* stats) {
bool called = false;
UpdateDeleteBitmapRequest captured_req;
SyncPoint::CallbackGuard guard;
sp->set_call_back(
"CloudMetaMgr::cloud_update_delete_bitmap_without_lock.before_rpc",
[&](auto&& args) {
auto* req = try_any_cast<UpdateDeleteBitmapRequest*>(args[0]);
captured_req.CopyFrom(*req);
called = true;
auto* ret = try_any_cast<std::pair<Status, bool>*>(args.back());
ret->first = Status::OK();
ret->second = true;
},
&guard);
auto status = meta_mgr.cloud_update_delete_bitmap_without_lock(
*tablet, &delete_bitmap, rowset_to_versions, stats, tablet->table_id(), 1, 2);
EXPECT_TRUE(status.ok()) << status;
EXPECT_TRUE(called);
return captured_req;
};

auto config_disabled_req = capture_request(nullptr);
EXPECT_EQ(config_disabled_req.pre_rowset_delete_bitmap_stats_size(), 0);

CloudTablet::PreRowsetDeleteBitmapStats empty_stats;
empty_stats.emplace(
"rowset_without_delete_bitmap",
std::vector<std::tuple<DeleteBitmap::SegmentId, DeleteBitmap::Version, size_t>> {});
auto config_enabled_req = capture_request(&empty_stats);
ASSERT_EQ(config_enabled_req.pre_rowset_delete_bitmap_stats_size(), 1);
EXPECT_EQ(config_enabled_req.pre_rowset_delete_bitmap_stats(0).rowset_id(),
"rowset_without_delete_bitmap");
EXPECT_EQ(config_enabled_req.pre_rowset_delete_bitmap_stats(0).delete_bitmap_stats_size(), 0);

using DeleteBitmapStat = std::tuple<DeleteBitmap::SegmentId, DeleteBitmap::Version, size_t>;
CloudTablet::PreRowsetDeleteBitmapStats populated_stats;
populated_stats.emplace("rowset_with_delete_bitmap",
std::vector<DeleteBitmapStat> {{3, 7, 1024}, {8, 9, 2048}});
auto populated_stats_req = capture_request(&populated_stats);
ASSERT_EQ(populated_stats_req.pre_rowset_delete_bitmap_stats_size(), 1);
const auto& rowset_stats = populated_stats_req.pre_rowset_delete_bitmap_stats(0);
EXPECT_EQ(rowset_stats.rowset_id(), "rowset_with_delete_bitmap");
ASSERT_EQ(rowset_stats.delete_bitmap_stats_size(), 2);
EXPECT_EQ(rowset_stats.delete_bitmap_stats(0).segment_id(), 3);
EXPECT_EQ(rowset_stats.delete_bitmap_stats(0).version(), 7);
EXPECT_EQ(rowset_stats.delete_bitmap_stats(0).delete_bitmap_size(), 1024);
EXPECT_EQ(rowset_stats.delete_bitmap_stats(1).segment_id(), 8);
EXPECT_EQ(rowset_stats.delete_bitmap_stats(1).version(), 9);
EXPECT_EQ(rowset_stats.delete_bitmap_stats(1).delete_bitmap_size(), 2048);
}

static AbortTxnRequest get_abort_txn_request(CloudMetaMgr* meta_mgr, const StreamLoadContext& ctx) {
auto* sp = SyncPoint::get_instance();
sp->clear_all_call_backs();
Expand Down
73 changes: 73 additions & 0 deletions be/test/cloud/cloud_tablet_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,79 @@ class CloudTabletWarmUpStateTest : public testing::Test {
CloudStorageEngine _engine;
};

class CloudTabletDeleteBitmapTest : public CloudTabletWarmUpStateTest {};

TEST_F(CloudTabletDeleteBitmapTest, AggDeleteBitmapForCompactionReturnsPreRowsetStats) {
auto rowset1 = create_rowset(Version(1, 1), 2);
auto rowset2 = create_rowset(Version(2, 2));
auto rowset_without_delete_bitmap = create_rowset(Version(3, 3));
ASSERT_NE(rowset1, nullptr);
ASSERT_NE(rowset2, nullptr);
ASSERT_NE(rowset_without_delete_bitmap, nullptr);

roaring::Roaring before_range;
before_range.add(1);
roaring::Roaring at_start;
at_start.add(2);
at_start.add(3);
roaring::Roaring within_range;
within_range.add(4);
roaring::Roaring at_end;
at_end.add(5);
roaring::Roaring second_segment;
second_segment.add(6);
second_segment.add(7);
second_segment.add(8);
roaring::Roaring second_rowset;
second_rowset.add(9);

auto& delete_bitmap = _tablet->tablet_meta()->delete_bitmap();
delete_bitmap.set({rowset1->rowset_id(), 0, 4}, before_range);
delete_bitmap.set({rowset1->rowset_id(), 0, 5}, at_start);
delete_bitmap.set({rowset1->rowset_id(), 0, 6}, within_range);
delete_bitmap.set({rowset1->rowset_id(), 0, 7}, at_end);
delete_bitmap.set({rowset1->rowset_id(), 1, 6}, second_segment);
delete_bitmap.set({rowset2->rowset_id(), 0, 5}, second_rowset);

auto aggregated_delete_bitmap = std::make_shared<DeleteBitmap>(_tablet->tablet_id());
std::map<std::string, int64_t> pre_rowset_to_versions;
CloudTablet::PreRowsetDeleteBitmapStats pre_rowset_delete_bitmap_stats;
_tablet->agg_delete_bitmap_for_compaction(
5, 7, {rowset1, rowset2, rowset_without_delete_bitmap}, aggregated_delete_bitmap,
pre_rowset_to_versions, &pre_rowset_delete_bitmap_stats);

using DeleteBitmapStat = std::tuple<DeleteBitmap::SegmentId, DeleteBitmap::Version, size_t>;
EXPECT_EQ(pre_rowset_delete_bitmap_stats.at(rowset1->rowset_id().to_string()),
(std::vector<DeleteBitmapStat> {
{0, 5, at_start.getSizeInBytes()},
{0, 6, within_range.getSizeInBytes()},
{1, 6, second_segment.getSizeInBytes()},
}));
EXPECT_EQ(pre_rowset_delete_bitmap_stats.at(rowset2->rowset_id().to_string()),
(std::vector<DeleteBitmapStat> {{0, 5, second_rowset.getSizeInBytes()}}));
EXPECT_TRUE(
pre_rowset_delete_bitmap_stats.at(rowset_without_delete_bitmap->rowset_id().to_string())
.empty());

roaring::Roaring aggregated;
ASSERT_EQ(aggregated_delete_bitmap->get({rowset1->rowset_id(), 0, 7}, &aggregated), 0);
EXPECT_EQ(aggregated.cardinality(), 4);
ASSERT_EQ(aggregated_delete_bitmap->get({rowset1->rowset_id(), 1, 7}, &aggregated), 0);
EXPECT_EQ(aggregated.cardinality(), 3);
ASSERT_EQ(aggregated_delete_bitmap->get({rowset2->rowset_id(), 0, 7}, &aggregated), 0);
EXPECT_EQ(aggregated.cardinality(), 1);
EXPECT_EQ(pre_rowset_to_versions.at(rowset1->rowset_id().to_string()), 1);
EXPECT_EQ(pre_rowset_to_versions.at(rowset2->rowset_id().to_string()), 2);

auto aggregated_without_stats = std::make_shared<DeleteBitmap>(_tablet->tablet_id());
std::map<std::string, int64_t> rowset_versions_without_stats;
_tablet->agg_delete_bitmap_for_compaction(
5, 7, {rowset1, rowset2, rowset_without_delete_bitmap}, aggregated_without_stats,
rowset_versions_without_stats, nullptr);
EXPECT_EQ(aggregated_without_stats->delete_bitmap.size(), 3);
EXPECT_EQ(rowset_versions_without_stats.size(), 2);
}

// Test get_rowset_warmup_state for non-existent rowset
TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) {
auto rowset = create_rowset(Version(1, 1));
Expand Down
Loading
Loading