diff --git a/be/benchmark/parquet/README.md b/be/benchmark/parquet/README.md index b4761855f5ceae..064af98ab933c9 100644 --- a/be/benchmark/parquet/README.md +++ b/be/benchmark/parquet/README.md @@ -34,8 +34,8 @@ timed region. It covers PLAIN, dictionary, byte-stream-split, and DELTA encoding supported fixed-width and binary physical types. Sparse selections are provided as both one clustered range and many alternating ranges. -The decoder selection axis includes 0%, 1%, 10%, 50%, 90%, and 100% so boundary and -high-selectivity behavior are visible. +The decoder selection axis includes 0%, 1%, 5%, 10%, 50%, 90%, and 100% so Q28-shaped sparse, +boundary, and high-selectivity behavior are visible. ```shell be/output/lib/benchmark_test \ diff --git a/be/benchmark/parquet/benchmark_parquet_decoder.hpp b/be/benchmark/parquet/benchmark_parquet_decoder.hpp index c1c2e3cb33220c..75fc65a27fd029 100644 --- a/be/benchmark/parquet/benchmark_parquet_decoder.hpp +++ b/be/benchmark/parquet/benchmark_parquet_decoder.hpp @@ -516,8 +516,7 @@ inline DecoderDigest expected_decoder_digest(const DecoderScenario& scenario, inline Status verify_decoder_output(format::parquet::native::Decoder* decoder, Slice* encoded, const DecoderScenario& scenario, bool binary, - const ParquetSelection& selection, - const SelectionPlan& plan) { + const ParquetSelection& selection, const SelectionPlan& plan) { RETURN_IF_ERROR(decoder->set_data(encoded)); DecoderDigest actual; if (scenario.encoding == Encoding::DICTIONARY) { @@ -626,7 +625,7 @@ inline void run_decoder(benchmark::State& state, DecoderScenario scenario, int s inline bool register_decoder_benchmarks() { for (const auto& scenario : decoder_scenarios()) { - for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const int selectivity : {0, 1, 5, 10, 50, 90, 100}) { for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { const std::string name = "ParquetDecoder/" + to_string(scenario.encoding) + "/" + to_string(scenario.value_type) + "/sel_" + diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 093550f20d8c68..0a15a6281c2fed 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1271,6 +1271,8 @@ DEFINE_mInt64(file_cache_background_block_lru_update_qps_limit, "1000"); DEFINE_mInt64(file_cache_background_block_lru_update_queue_max_size, "500000"); DEFINE_mBool(enable_file_cache_async_touch_on_get_or_set, "false"); DEFINE_mBool(enable_reader_dryrun_when_download_file_cache, "true"); +DEFINE_mBool(enable_file_scanner_v2_reader_local_cache, "true"); +DEFINE_mInt64(file_scanner_v2_reader_local_cache_size, "67108864"); // 64MB per scanner DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "180000"); DEFINE_mInt64(file_cache_background_ttl_info_update_interval_ms, "180000"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 335ce7b72317ce..7030ec16e06174 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1310,6 +1310,11 @@ DECLARE_mInt64(file_cache_background_block_lru_update_qps_limit); DECLARE_mInt64(file_cache_background_block_lru_update_queue_max_size); DECLARE_mBool(enable_file_cache_async_touch_on_get_or_set); DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); +// Cache File Scanner V2 file-cache blocks in reader-local memory. File Scanner V1 and internal +// table readers never opt in to this cache. +DECLARE_mBool(enable_file_scanner_v2_reader_local_cache); +// Maximum reader-local cache bytes managed by one File Scanner V2 scanner. +DECLARE_mInt64(file_scanner_v2_reader_local_cache_size); DECLARE_mInt64(file_cache_background_monitor_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms); DECLARE_mInt64(file_cache_background_ttl_info_update_interval_ms); diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index be17a9920e6ac6..188bb67eae88fa 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -67,6 +67,7 @@ #include "format_v2/table_reader.h" #include "format_v2/wal/wal_table_reader.h" #include "io/cache/block_file_cache_profile.h" +#include "io/cache/cached_remote_file_reader.h" #include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/descriptors.h" @@ -281,6 +282,14 @@ Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { } // namespace +int64_t FileScannerV2::_cumulative_profile_delta(int64_t current, int64_t* reported) { + DORIS_CHECK(reported != nullptr); + DORIS_CHECK(current >= *reported); + const int64_t delta = current - *reported; + *reported = current; + return delta; +} + #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) @@ -956,6 +965,12 @@ Status FileScannerV2::_to_file_format(TFileFormatType::type format_type, Status FileScannerV2::_init_io_ctx() { _io_ctx = create_file_scan_io_context(_state); + if (config::enable_file_scanner_v2_reader_local_cache) { + const size_t capacity = cast_set( + std::max(0, config::file_scanner_v2_reader_local_cache_size)); + _io_ctx->reader_local_cache = std::make_shared( + capacity, _state->query_mem_tracker()); + } return Status::OK(); } @@ -1095,7 +1110,10 @@ void FileScannerV2::update_realtime_counters() { _state->get_query_ctx()->resource_ctx()->io_context()->update_scan_bytes_from_remote_storage( deltas.scan_bytes_from_remote_storage); - COUNTER_SET(_file_read_bytes_counter, bytes_read); + // Scanner instances share the profile counter, so publishing an absolute value would erase + // bytes already reported by sibling scanners. + COUNTER_UPDATE(_file_read_bytes_counter, + _cumulative_profile_delta(bytes_read, &_reported_file_read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); @@ -1192,7 +1210,9 @@ void FileScannerV2::_collect_profile_before_close() { _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { - COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); + COUNTER_UPDATE(_file_read_bytes_counter, + _cumulative_profile_delta(cast_set(_file_reader_stats->read_bytes), + &_reported_file_read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); const auto read_time = cast_set(_file_reader_stats->read_time_ns); diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index fbf630731ad889..fb27e0ce70e3f4 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -87,6 +87,9 @@ class FileScannerV2 final : public Scanner { int64_t* last_bytes_read_from_remote); static void TEST_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); + static int64_t TEST_cumulative_profile_delta(int64_t current, int64_t* reported) { + return _cumulative_profile_delta(current, reported); + } static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); static Status TEST_contextualize_output_filter_status(Status status, @@ -98,6 +101,13 @@ class FileScannerV2 final : public Scanner { return _should_run_adaptive_batch_size(predictor_initialized, current_split_uses_metadata_count); } + Status TEST_init_io_ctx() { return _init_io_ctx(); } + bool TEST_has_reader_local_cache() const { + return _io_ctx != nullptr && _io_ctx->reader_local_cache != nullptr; + } + const void* TEST_reader_local_cache() const { + return _io_ctx != nullptr ? _io_ctx->reader_local_cache.get() : nullptr; + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -137,6 +147,7 @@ class FileScannerV2 final : public Scanner { static bool _should_skip_empty(const Status& status, bool stopped); static Status _contextualize_output_filter_status(Status status, TFileFormatType::type format_type); + static int64_t _cumulative_profile_delta(int64_t current, int64_t* reported); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -227,6 +238,7 @@ class FileScannerV2 final : public Scanner { int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; int64_t _reported_io_read_time = 0; + int64_t _reported_file_read_bytes = 0; }; } // namespace doris diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index 1b1f2f284405f9..778f0f448a9c13 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -94,6 +94,12 @@ Status FileReader::init(RuntimeState* state) { ++_reader_statistics.open_file_num; io::FileReaderOptions reader_options = FileFactory::get_reader_options(state->query_options(), *_file_description); + // Parquet currently supplies the planned range reuse that amortizes a promoted block. Other + // V2 formats keep their existing buffering path until they opt in with equivalent semantics. + reader_options.reader_local_cache = _supports_reader_local_cache() && _io_ctx != nullptr + ? _io_ctx->reader_local_cache + : nullptr; + reader_options.enable_reader_local_cache = reader_options.reader_local_cache != nullptr; _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( _profile, *_system_properties, *_file_description, reader_options, io::DelegateReader::AccessMode::RANDOM, _io_ctx)); diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 315d00449bd4ed..a9a977ff885102 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -419,6 +419,7 @@ class FileReader { protected: virtual void _init_profile() {} + virtual bool _supports_reader_local_cache() const { return false; } void _record_scan_rows(int64_t rows) { DORIS_CHECK(rows >= 0); _reader_statistics.read_rows += rows; diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 8ba8cf94662f9a..3b390f43f500c2 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -556,9 +556,18 @@ void ParquetFileContext::prefetch_ranges(const std::vector(reader)) { + reader = tracing_reader->inner_reader(); + } + return dynamic_cast(reader.get()) != nullptr || + reader->get_data_dir_path() == io::FileReader::VIRTUAL_REMOTE_DATA_DIR; +} + bool ParquetFileContext::set_native_random_access_ranges( const std::vector& ranges, size_t avg_io_size, - RuntimeProfile* profile, int64_t merge_read_slice_size) { + RuntimeProfile* profile, int64_t merge_read_slice_size, bool expose_ranges_immediately) { DORIS_CHECK(native_file != nullptr); if (!detail::should_use_merge_range_reader( ranges, avg_io_size, @@ -576,7 +585,9 @@ bool ParquetFileContext::set_native_random_access_ranges( } std::ranges::sort(native_ranges, {}, &io::PrefetchRange::start_offset); native_row_group_file = std::make_shared( - profile, native_file, native_ranges, merge_read_slice_size); + profile, native_file, + expose_ranges_immediately ? native_ranges : std::vector {}, + merge_read_slice_size); return true; } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 38e78438e39b29..c791d806b7eec6 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -164,7 +164,9 @@ struct ParquetFileContext { // sequential projected chunk ranges consumed by MergeRangeFileReader. bool set_native_random_access_ranges(const std::vector& ranges, size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size); + int64_t merge_read_slice_size, + bool expose_ranges_immediately = true); + bool native_file_should_defer_merge_ranges() const; const io::FileReaderSPtr& native_data_file() const { return native_row_group_file != nullptr ? native_row_group_file : native_file; } diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 0a2b3791502736..3391e20c8fa99c 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -86,6 +86,7 @@ class ParquetReader : public format::FileReader { protected: void _init_profile() override; + bool _supports_reader_local_cache() const override { return true; } private: void _sync_page_cache_profile(); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 630e4d8121c970..d6e2da2a5ce09a 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -271,6 +271,11 @@ void materialize_count_star_placeholders(const format::FileScanRequest& request, namespace detail { +std::vector deferred_merge_range_columns( + const format::FileScanRequest& request) { + return request_scan_columns(request); +} + Status build_native_prefetch_ranges( const tparquet::FileMetaData& metadata, const std::vector>& file_schema, @@ -966,6 +971,35 @@ void ParquetScanScheduler::reset_current_row_group() { _current_predicate_prefetched = false; _current_non_predicate_prefetched = false; _current_merge_range_active = false; + _current_merge_range_reader = nullptr; + _current_merge_ranges_by_column.clear(); + _activated_merge_range_columns.clear(); + _current_merge_range_stage = 0; +} + +Status ParquetScanScheduler::activate_merge_ranges_for_columns( + const std::vector& column_ids) { + if (_current_merge_range_reader == nullptr) { + return Status::OK(); + } + std::vector ranges; + for (const auto column_id : column_ids) { + if (!_activated_merge_range_columns.emplace(column_id).second) { + continue; + } + const auto it = _current_merge_ranges_by_column.find(column_id); + if (it == _current_merge_ranges_by_column.end()) { + continue; + } + for (const auto& [start, end] : it->second) { + ranges.emplace_back(start, end); + } + } + if (ranges.empty()) { + return Status::OK(); + } + return _current_merge_range_reader->add_random_access_ranges(ranges, + _current_merge_range_stage++); } void ParquetScanScheduler::flush_current_reader_profiles() { @@ -1177,10 +1211,14 @@ Status ParquetScanScheduler::open_next_row_group( RETURN_IF_ERROR(detail::build_native_prefetch_ranges( thrift_metadata, file_schema, request_scan_columns(request), row_group_idx, file_context.native_file->size(), compat.parquet_816_padding, &native_ranges)); + // Local readers benefit from one eager coalescing plan; splitting their ranges by predicate + // stage only adds small reads. Remote readers can avoid future-stage IO, and exact cache hits + // can bypass their merge path altogether. + const bool defer_merge_ranges = file_context.native_file_should_defer_merge_ranges(); if (request.non_predicate_positions.empty()) { _current_merge_range_active = file_context.set_native_random_access_ranges( native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, - _merge_read_slice_size); + _merge_read_slice_size, !defer_merge_ranges); } else { // Independent predicate/output readers may revisit the same physical leaf at different // cursors. MergeRangeFileReader has one consumptive cache per range, so use the random @@ -1188,6 +1226,23 @@ Status ParquetScanScheduler::open_next_row_group( _current_merge_range_active = file_context.set_native_random_access_ranges( {}, 0, _profile, _merge_read_slice_size); } + if (_current_merge_range_active && defer_merge_ranges) { + _current_merge_range_reader = + typeid_cast(file_context.native_data_file().get()); + DORIS_CHECK(_current_merge_range_reader != nullptr); + for (const auto& column : detail::deferred_merge_range_columns(request)) { + std::vector column_ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + thrift_metadata, file_schema, {column}, row_group_idx, + file_context.native_file->size(), compat.parquet_816_padding, &column_ranges)); + auto& stored_ranges = _current_merge_ranges_by_column[column.column_id()]; + stored_ranges.reserve(column_ranges.size()); + for (const auto& range : detail::valid_prefetch_ranges(column_ranges)) { + stored_ranges.emplace_back(cast_set(range.offset), + cast_set(range.end_offset())); + } + } + } for (const auto& col : request.predicate_columns) { const auto local_id = col.column_id(); @@ -2264,6 +2319,14 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, }; auto read_all_predicate_columns = [&]() -> Status { + if (_current_merge_range_reader != nullptr) { + std::vector stage_columns; + stage_columns.reserve(_current_predicate_columns.size()); + for (const auto& fid : _current_predicate_columns | std::views::keys) { + stage_columns.push_back(fid); + } + RETURN_IF_ERROR(activate_merge_ranges_for_columns(stage_columns)); + } for (const auto& [fid, column_reader] : _current_predicate_columns) { auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); @@ -2310,6 +2373,9 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; const auto fid = col.column_id(); + if (_current_merge_range_reader != nullptr) { + RETURN_IF_ERROR(activate_merge_ranges_for_columns({fid})); + } auto reader_it = _current_predicate_columns.find(fid); DORIS_CHECK(reader_it != _current_predicate_columns.end()); auto position_it = request.local_positions.find(col.column_id()); @@ -2370,6 +2436,19 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, }; auto materialize_predicate_positions = [&](const std::vector& positions) -> Status { + if (_current_merge_range_reader != nullptr) { + std::vector stage_columns; + stage_columns.reserve(positions.size()); + for (const size_t position : positions) { + if (materialized_positions.contains(position)) { + continue; + } + const auto index_it = _predicate_indices_by_position_scratch.find(position); + DORIS_CHECK(index_it != _predicate_indices_by_position_scratch.end()); + stage_columns.push_back(request.predicate_columns[index_it->second].column_id()); + } + RETURN_IF_ERROR(activate_merge_ranges_for_columns(stage_columns)); + } for (const size_t position : positions) { if (materialized_positions.contains(position)) { continue; @@ -2605,6 +2684,17 @@ Status ParquetScanScheduler::read_current_row_group_batch( physical_non_predicate_columns(request), &_current_non_predicate_prefetched)); } + if (_current_merge_range_reader != nullptr && selected_rows > 0) { + std::vector lazy_columns; + lazy_columns.reserve(_current_non_predicate_columns.size()); + for (const auto& fid : _current_non_predicate_columns | std::views::keys) { + lazy_columns.push_back(fid); + } + // Deferred remote ranges are intentionally absent while predicates reject rows. Activate + // every surviving output column at the exact lazy-materialization boundary so its page + // reads do not silently fall back to many direct remote requests. + RETURN_IF_ERROR(activate_merge_ranges_for_columns(lazy_columns)); + } if (selected_rows > _batch_size) { DORIS_CHECK(_pending_predicate_selection.empty()); @@ -2697,6 +2787,14 @@ Status ParquetScanScheduler::materialize_pending_predicate_batch( file_block->replace_by_position( block_position, column->cut(_pending_predicate_selected_offset, output_rows)); } + if (_current_merge_range_reader != nullptr) { + std::vector lazy_columns; + lazy_columns.reserve(_current_non_predicate_columns.size()); + for (const auto& fid : _current_non_predicate_columns | std::views::keys) { + lazy_columns.push_back(fid); + } + RETURN_IF_ERROR(activate_merge_ranges_for_columns(lazy_columns)); + } { SCOPED_TIMER(_scan_profile.column_read_time); RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 474963ed5d9aa3..bcde143c571d5a 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,10 @@ namespace doris { class Block; class RuntimeState; +namespace io { +class MergeRangeFileReader; +} + namespace format { struct FileScanRequest; } // namespace format @@ -92,6 +97,8 @@ Status build_native_prefetch_ranges( const std::vector>& file_schema, const std::vector& scan_columns, int row_group_idx, size_t file_size, bool parquet_816_padding, std::vector* ranges); +std::vector deferred_merge_range_columns( + const format::FileScanRequest& request); Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, const ParquetScanRange& scan_range, std::vector* row_group_first_rows, @@ -226,6 +233,7 @@ class ParquetScanScheduler { Status skip_current_row_group_rows(int64_t rows); Status flush_pending_non_predicate_skip_rows(); + Status activate_merge_ranges_for_columns(const std::vector& column_ids); Status read_filter_columns(int64_t batch_rows, const format::FileScanRequest& request, Block* file_block, SelectionVector* selection, @@ -296,6 +304,11 @@ class ParquetScanScheduler { bool _current_predicate_prefetched = false; bool _current_non_predicate_prefetched = false; bool _current_merge_range_active = false; + io::MergeRangeFileReader* _current_merge_range_reader = nullptr; + std::map>> + _current_merge_ranges_by_column; + std::set _activated_merge_range_columns; + uint32_t _current_merge_range_stage = 0; ParquetPageSkipProfile _page_skip_profile; ParquetScanProfile _scan_profile; const ParquetProfile* _parquet_profile = nullptr; diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index 628a34ac97675e..a665c88e64337a 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -37,6 +37,7 @@ #include "core/custom_allocator.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" +#include "util/cpu_info.h" #include "util/rle_encoding.h" #include "util/slice.h" @@ -260,9 +261,24 @@ class BaseDictDecoder : public Decoder { constexpr size_t MIN_FRAGMENTED_RANGES = 8; constexpr size_t MAX_AVERAGE_RANGE_VALUES = 4; constexpr size_t MAX_DECODE_EXPANSION = 8; - return selection.ranges.size() >= MIN_FRAGMENTED_RANGES && selection.selected_values != 0 && - selection.total_values / selection.selected_values <= MAX_DECODE_EXPANSION && - selection.selected_values <= selection.ranges.size() * MAX_AVERAGE_RANGE_VALUES; + constexpr size_t RANGE_TRANSITION_EQUIVALENT_VALUES = 24; + if (selection.ranges.size() < MIN_FRAGMENTED_RANGES || selection.selected_values == 0 || + selection.selected_values > selection.ranges.size() * MAX_AVERAGE_RANGE_VALUES) { + return false; + } + if (selection.total_values / selection.selected_values <= MAX_DECODE_EXPANSION) { + return true; + } + const size_t transition_threshold = + selection.total_values / RANGE_TRANSITION_EQUIVALENT_VALUES + + (selection.total_values % RANGE_TRANSITION_EQUIVALENT_VALUES != 0); + // Keep very sparse batches on range decode; a cache-resident full index batch wins only + // after repeated decoder transitions cost more than one sequential pass. + if (selection.ranges.size() < transition_threshold) { + return false; + } + const auto l2_cache_size = static_cast(CpuInfo::get_l2_cache_size()); + return selection.total_values <= l2_cache_size / sizeof(uint32_t); } Status _decode_fragmented_selection(const ParquetSelection& selection, diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 6b3334330fc629..8fdaacdc97be08 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -99,6 +99,25 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(lock_wait_timer); SUBTRACT_FIELD(get_timer); SUBTRACT_FIELD(set_timer); + SUBTRACT_FIELD(num_reader_local_cache_total); + SUBTRACT_FIELD(num_reader_local_cache_hit); + SUBTRACT_FIELD(num_reader_local_cache_miss); + SUBTRACT_FIELD(num_reader_local_cache_fill); + SUBTRACT_FIELD(num_reader_local_cache_evict); + SUBTRACT_FIELD(num_reader_local_cache_wait); + SUBTRACT_FIELD(num_reader_local_cache_admission_reject); + SUBTRACT_FIELD(num_reader_local_cache_partial_miss); + SUBTRACT_FIELD(num_reader_local_cache_disk_lru_touch); + SUBTRACT_FIELD(bytes_reader_local_cache_request); + SUBTRACT_FIELD(bytes_read_from_reader_local_cache); + SUBTRACT_FIELD(bytes_read_into_reader_local_cache); + SUBTRACT_FIELD(reader_local_cache_fill_timer); + SUBTRACT_FIELD(reader_local_cache_wait_timer); + SUBTRACT_FIELD(reader_local_cache_probe_timer); + SUBTRACT_FIELD(num_exact_cache_probe); + SUBTRACT_FIELD(num_exact_cache_probe_hit); + SUBTRACT_FIELD(num_exact_cache_probe_miss); + SUBTRACT_FIELD(exact_cache_probe_timer); SUBTRACT_FIELD(inverted_index_num_local_io_total); SUBTRACT_FIELD(inverted_index_num_remote_io_total); @@ -169,6 +188,44 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, TUnit::UNIT, cache_profile, 1); remote_only_on_miss_threshold_bytes = profile->AddHighWaterMarkCounter( "RemoteOnlyOnMissThresholdBytes", TUnit::BYTES, cache_profile, 1); + num_reader_local_cache_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderLocalCacheRequests", + TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_hit = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderLocalCacheHits", + TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_miss = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderLocalCacheMisses", + TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_fill = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderLocalCacheFills", + TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_evict = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCacheEvictions", TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_wait = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderLocalCacheWaits", + TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_admission_reject = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCacheAdmissionRejects", TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_partial_miss = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCachePartialMisses", TUnit::UNIT, cache_profile, 1); + num_reader_local_cache_disk_lru_touch = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCacheDiskLRUTouches", TUnit::UNIT, cache_profile, 1); + bytes_reader_local_cache_request = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCacheRequestBytes", TUnit::BYTES, cache_profile, 1); + bytes_read_from_reader_local_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCacheHitBytes", TUnit::BYTES, cache_profile, 1); + bytes_read_into_reader_local_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ReaderLocalCacheFillBytes", TUnit::BYTES, cache_profile, 1); + reader_local_cache_fill_timer = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "ReaderLocalCacheFillTimer", cache_profile, 1); + reader_local_cache_wait_timer = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "ReaderLocalCacheWaitTimer", cache_profile, 1); + reader_local_cache_probe_timer = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "ReaderLocalCacheProbeTimer", cache_profile, 1); + num_exact_cache_probe = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExactCacheProbes", TUnit::UNIT, + cache_profile, 1); + num_exact_cache_probe_hit = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExactCacheProbeHits", + TUnit::UNIT, cache_profile, 1); + num_exact_cache_probe_miss = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExactCacheProbeMisses", + TUnit::UNIT, cache_profile, 1); + exact_cache_probe_timer = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "ExactCacheProbeTimer", cache_profile, 1); inverted_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "InvertedIndexNumLocalIOTotal", TUnit::UNIT, cache_profile, 1); @@ -267,6 +324,30 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con COUNTER_UPDATE(set_timer, statistics->set_timer); remote_only_on_miss_triggered->set(statistics->remote_only_on_miss_triggered); remote_only_on_miss_threshold_bytes->set(statistics->remote_only_on_miss_threshold_bytes); + COUNTER_UPDATE(num_reader_local_cache_total, statistics->num_reader_local_cache_total); + COUNTER_UPDATE(num_reader_local_cache_hit, statistics->num_reader_local_cache_hit); + COUNTER_UPDATE(num_reader_local_cache_miss, statistics->num_reader_local_cache_miss); + COUNTER_UPDATE(num_reader_local_cache_fill, statistics->num_reader_local_cache_fill); + COUNTER_UPDATE(num_reader_local_cache_evict, statistics->num_reader_local_cache_evict); + COUNTER_UPDATE(num_reader_local_cache_wait, statistics->num_reader_local_cache_wait); + COUNTER_UPDATE(num_reader_local_cache_admission_reject, + statistics->num_reader_local_cache_admission_reject); + COUNTER_UPDATE(num_reader_local_cache_partial_miss, + statistics->num_reader_local_cache_partial_miss); + COUNTER_UPDATE(num_reader_local_cache_disk_lru_touch, + statistics->num_reader_local_cache_disk_lru_touch); + COUNTER_UPDATE(bytes_reader_local_cache_request, statistics->bytes_reader_local_cache_request); + COUNTER_UPDATE(bytes_read_from_reader_local_cache, + statistics->bytes_read_from_reader_local_cache); + COUNTER_UPDATE(bytes_read_into_reader_local_cache, + statistics->bytes_read_into_reader_local_cache); + COUNTER_UPDATE(reader_local_cache_fill_timer, statistics->reader_local_cache_fill_timer); + COUNTER_UPDATE(reader_local_cache_wait_timer, statistics->reader_local_cache_wait_timer); + COUNTER_UPDATE(reader_local_cache_probe_timer, statistics->reader_local_cache_probe_timer); + COUNTER_UPDATE(num_exact_cache_probe, statistics->num_exact_cache_probe); + COUNTER_UPDATE(num_exact_cache_probe_hit, statistics->num_exact_cache_probe_hit); + COUNTER_UPDATE(num_exact_cache_probe_miss, statistics->num_exact_cache_probe_miss); + COUNTER_UPDATE(exact_cache_probe_timer, statistics->exact_cache_probe_timer); COUNTER_UPDATE(inverted_index_num_local_io_total, statistics->inverted_index_num_local_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 74bf994de7be1c..c9b21b0fffb0c9 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -92,6 +92,25 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* set_timer = nullptr; RuntimeProfile::HighWaterMarkCounter* remote_only_on_miss_triggered = nullptr; RuntimeProfile::HighWaterMarkCounter* remote_only_on_miss_threshold_bytes = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_total = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_hit = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_miss = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_fill = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_evict = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_wait = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_admission_reject = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_partial_miss = nullptr; + RuntimeProfile::Counter* num_reader_local_cache_disk_lru_touch = nullptr; + RuntimeProfile::Counter* bytes_reader_local_cache_request = nullptr; + RuntimeProfile::Counter* bytes_read_from_reader_local_cache = nullptr; + RuntimeProfile::Counter* bytes_read_into_reader_local_cache = nullptr; + RuntimeProfile::Counter* reader_local_cache_fill_timer = nullptr; + RuntimeProfile::Counter* reader_local_cache_wait_timer = nullptr; + RuntimeProfile::Counter* reader_local_cache_probe_timer = nullptr; + RuntimeProfile::Counter* num_exact_cache_probe = nullptr; + RuntimeProfile::Counter* num_exact_cache_probe_hit = nullptr; + RuntimeProfile::Counter* num_exact_cache_probe_miss = nullptr; + RuntimeProfile::Counter* exact_cache_probe_timer = nullptr; RuntimeProfile::Counter* inverted_index_num_local_io_total = nullptr; RuntimeProfile::Counter* inverted_index_num_remote_io_total = nullptr; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 47fe02ee2c3e92..d7c1e4026cbf92 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -34,12 +35,14 @@ #include #include #include +#include #include #include #include "cloud/cloud_cluster_info.h" #include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" +#include "common/cast_set.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/metrics/doris_metrics.h" @@ -54,6 +57,9 @@ #include "io/fs/local_file_system.h" #include "io/io_common.h" #include "runtime/exec_env.h" +#include "runtime/memory/global_memory_arbitrator.h" +#include "runtime/memory/mem_tracker.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -107,6 +113,428 @@ bvar::Adder g_peer_cross_compute_group_read("peer_cross_compute_group_ bvar::Adder g_peer_same_compute_group_read("peer_same_compute_group_read"); bvar::Adder g_peer_lazy_fetch_triggered("peer_lazy_fetch_triggered"); +FileScannerV2ReaderLocalCache::FileScannerV2ReaderLocalCache( + size_t capacity, std::shared_ptr query_mem_tracker) + : _capacity(capacity), + _query_mem_tracker(std::move(query_mem_tracker)), + _memory_tracker(std::make_shared("FileScannerV2ReaderLocalCache")) {} + +FileScannerV2ReaderLocalCache::~FileScannerV2ReaderLocalCache() { + auto files = _file_caches(); + for (const auto& file : files) { + file->_drain(this); + } + std::lock_guard lock(_budget_mutex); + DORIS_CHECK(_memory_bytes == 0); + DORIS_CHECK(_reserved_bytes == 0); +} + +std::shared_ptr +FileScannerV2ReaderLocalCache::create_file_cache() { + if (_capacity == 0) { + return nullptr; + } + std::shared_ptr file_cache; + try { + file_cache = std::shared_ptr( + new FileScannerV2ReaderLocalFileCache(shared_from_this())); + } catch (const doris::Exception&) { + return nullptr; + } catch (const std::bad_alloc&) { + return nullptr; + } + { + std::lock_guard lock(_registry_mutex); + try { + std::erase_if(_files, [](const auto& file) { return file.expired(); }); + _files.emplace_back(file_cache); + } catch (const doris::Exception&) { + return nullptr; + } catch (const std::bad_alloc&) { + return nullptr; + } + } + return file_cache; +} + +bool FileScannerV2ReaderLocalCache::_try_reserve(size_t bytes) { + std::lock_guard lock(_budget_mutex); + if (bytes > _capacity) { + return false; + } + if (_memory_bytes + _reserved_bytes + bytes > _capacity) { + return false; + } + if (_query_mem_tracker != nullptr && _query_mem_tracker->limit() >= 0 && + _query_mem_tracker->consumption() + cast_set(_reserved_bytes + bytes) > + _query_mem_tracker->limit()) { + return false; + } + if (GlobalMemoryArbitrator::is_exceed_soft_mem_limit( + cast_set(_reserved_bytes + bytes))) { + return false; + } + _reserved_bytes += bytes; + return true; +} + +bool FileScannerV2ReaderLocalCache::_reserve(size_t bytes, + FileScannerV2ReaderLocalFileCache* requester, + size_t* evicted) { + if (_try_reserve(bytes)) { + return true; + } + // A stream may recycle its own cold blocks, but it must never evict another stream's hot + // block map. StarRocks gets the same isolation from CacheInputStream::_block_map ownership. + while (requester->_evict_one()) { + ++*evicted; + if (_try_reserve(bytes)) { + return true; + } + } + return false; +} + +void FileScannerV2ReaderLocalCache::_commit(size_t bytes) { + { + std::lock_guard lock(_budget_mutex); + DORIS_CHECK(_reserved_bytes >= bytes); + _reserved_bytes -= bytes; + _memory_bytes += bytes; + } + _memory_tracker->consume(cast_set(bytes)); +} + +void FileScannerV2ReaderLocalCache::_cancel_reservation(size_t bytes) { + std::lock_guard lock(_budget_mutex); + DORIS_CHECK(_reserved_bytes >= bytes); + _reserved_bytes -= bytes; +} + +void FileScannerV2ReaderLocalCache::_release(size_t bytes) { + { + std::lock_guard lock(_budget_mutex); + DORIS_CHECK(_memory_bytes >= bytes); + _memory_bytes -= bytes; + } + _memory_tracker->release(cast_set(bytes)); +} + +std::vector> +FileScannerV2ReaderLocalCache::_file_caches() const { + std::vector> files; + std::lock_guard lock(_registry_mutex); + files.reserve(_files.size()); + for (const auto& file : _files) { + if (auto live_file = file.lock(); live_file != nullptr) { + files.push_back(std::move(live_file)); + } + } + return files; +} + +size_t FileScannerV2ReaderLocalCache::entry_count() const { + size_t count = 0; + for (const auto& file : _file_caches()) { + count += file->entry_count(); + } + return count; +} + +size_t FileScannerV2ReaderLocalCache::memory_usage() const { + std::lock_guard lock(_budget_mutex); + return _memory_bytes; +} + +int64_t FileScannerV2ReaderLocalCache::tracked_memory() const { + return _memory_tracker->consumption(); +} + +FileScannerV2ReaderLocalFileCache::FileScannerV2ReaderLocalFileCache( + std::shared_ptr owner) + : _owner(std::move(owner)) {} + +FileScannerV2ReaderLocalFileCache::~FileScannerV2ReaderLocalFileCache() { + if (auto owner = _owner.lock(); owner != nullptr) { + _drain(owner.get()); + } +} + +void FileScannerV2ReaderLocalFileCache::_drain(FileScannerV2ReaderLocalCache* owner) { + DORIS_CHECK(owner != nullptr); + size_t memory_bytes = 0; + size_t reserved_bytes = 0; + auto clear_entries = [&]() { + std::unique_lock lock(_mutex); + for (auto& [_, entry] : _entries) { + if (entry->data != nullptr) { + memory_bytes += entry->data->size(); + } + reserved_bytes += entry->reserved_bytes; + } + _entries.clear(); + _lru.clear(); + }; + try { + std::optional switch_query_tracker; + if (owner->_query_mem_tracker != nullptr) { + switch_query_tracker.emplace(owner->_query_mem_tracker); + } + clear_entries(); + } catch (...) { + // Destructors cannot propagate memory-tracker setup failures during query cancellation. + clear_entries(); + } + if (memory_bytes > 0) { + owner->_release(memory_bytes); + } + if (reserved_bytes > 0) { + owner->_cancel_reservation(reserved_bytes); + } +} + +void FileScannerV2ReaderLocalFileCache::_touch_locked(const std::shared_ptr& entry) { + if (entry->in_lru) { + _lru.splice(_lru.begin(), _lru, entry->lru_position); + } +} + +bool FileScannerV2ReaderLocalFileCache::_evict_one() { + const auto owner = _owner.lock(); + if (owner == nullptr) { + return false; + } + std::shared_ptr> data; + { + std::lock_guard lock(_mutex); + size_t candidates = _lru.size(); + while (candidates-- > 0 && !_lru.empty()) { + const size_t victim_offset = _lru.back(); + const auto victim = _entries.find(victim_offset); + DORIS_CHECK(victim != _entries.end()); + DORIS_CHECK(!victim->second->loading); + if (victim->second->data.use_count() > 1) { + // Keep pinned blocks discoverable so another reader cannot start a duplicate fill. + _lru.splice(_lru.begin(), _lru, victim->second->lru_position); + continue; + } + data = std::move(victim->second->data); + _lru.pop_back(); + _entries.erase(victim); + break; + } + } + if (data == nullptr) { + return false; + } + const size_t bytes = data->size(); + // Eviction and destruction are noexcept cleanup paths. A stack guard avoids a second heap + // allocation while releasing memory under pressure; if tracker switching itself fails, the + // block is still released and the explicit cache budget remains consistent. + try { + std::optional switch_query_tracker; + if (owner->_query_mem_tracker != nullptr) { + switch_query_tracker.emplace(owner->_query_mem_tracker); + } + data.reset(); + } catch (...) { + data.reset(); + } + owner->_release(bytes); + return true; +} + +void FileScannerV2ReaderLocalFileCache::_abort_load(size_t block_offset, + const std::shared_ptr& entry) { + size_t reserved_bytes = 0; + { + std::unique_lock lock(_mutex); + reserved_bytes = entry->reserved_bytes; + entry->reserved_bytes = 0; + entry->loading = false; + const auto it = _entries.find(block_offset); + if (it != _entries.end() && it->second == entry) { + _entries.erase(it); + } + } + if (reserved_bytes != 0) { + if (const auto owner = _owner.lock(); owner != nullptr) { + owner->_cancel_reservation(reserved_bytes); + } + } + // A loader must publish every exit, including allocation and tracker exceptions, otherwise a + // same-block waiter can remain asleep after the scan has already fallen back to FileCache. + entry->ready.notify_all(); +} + +bool FileScannerV2ReaderLocalFileCache::pin_if_present(size_t block_offset, size_t read_offset, + size_t read_size, LookupResult* lookup) { + DORIS_CHECK(lookup != nullptr); + *lookup = {}; + std::shared_ptr entry; + bool touch_lru = false; + { + std::shared_lock lock(_mutex); + const auto it = _entries.find(block_offset); + if (it == _entries.end() || it->second->loading || !it->second->load_status.ok() || + it->second->data == nullptr || read_offset < block_offset || + read_offset - block_offset > it->second->data->size() || + read_size > it->second->data->size() - (read_offset - block_offset)) { + return false; + } + entry = it->second; + lookup->data = entry->data; + lookup->admitted = true; + lookup->hit = true; + touch_lru = + entry->hit_count.fetch_add(1, std::memory_order_relaxed) % LRU_TOUCH_INTERVAL == 0; + if (touch_lru) { + lookup->file_block_to_touch = entry->source_file_block.lock(); + } + } + if (touch_lru) { + std::unique_lock lock(_mutex); + const auto it = _entries.find(block_offset); + if (it != _entries.end() && it->second == entry) { + _touch_locked(entry); + } + } + return true; +} + +bool FileScannerV2ReaderLocalFileCache::read_if_present(size_t block_offset, size_t read_offset, + Slice result, LookupResult* lookup) { + if (!pin_if_present(block_offset, read_offset, result.size, lookup)) { + return false; + } + memcpy(result.data, lookup->data->data() + read_offset - block_offset, result.size); + return true; +} + +Status FileScannerV2ReaderLocalFileCache::get_or_load(size_t block_offset, size_t block_size, + const FileBlockSPtr& file_block, + size_t file_block_offset, + LookupResult* lookup) { + DORIS_CHECK(lookup != nullptr); + *lookup = {}; + const auto owner = _owner.lock(); + if (owner == nullptr) { + return Status::OK(); + } + std::shared_ptr entry; + bool load = false; + { + std::unique_lock lock(_mutex); + const auto it = _entries.find(block_offset); + if (it == _entries.end()) { + try { + entry = std::make_shared(); + } catch (const doris::Exception&) { + return Status::OK(); + } catch (const std::bad_alloc&) { + return Status::OK(); + } + try { + _entries.emplace(block_offset, entry); + } catch (const doris::Exception&) { + return Status::OK(); + } catch (const std::bad_alloc&) { + return Status::OK(); + } + load = true; + } else { + entry = it->second; + lookup->admitted = true; + if (entry->loading) { + lookup->waited = true; + TEST_SYNC_POINT("CachedRemoteFileReader::reader_local_cache_before_wait"); + MonotonicStopWatch wait_watch; + wait_watch.start(); + entry->ready.wait(lock, [&entry]() { return !entry->loading; }); + lookup->wait_time = wait_watch.elapsed_time(); + } + RETURN_IF_ERROR(entry->load_status); + if (entry->data == nullptr || entry->data->size() < block_size) { + // FileCache ranges can end at different boundaries for the same aligned offset. + // Never reuse a shorter promotion for a later, wider range. + lookup->admitted = false; + return Status::OK(); + } + lookup->hit = true; + lookup->data = entry->data; + _touch_locked(entry); + } + } + + if (!load) { + return Status::OK(); + } + + std::shared_ptr> data; + std::optional switch_query_tracker; + try { + if (!owner->_reserve(block_size, this, &lookup->evicted)) { + _abort_load(block_offset, entry); + return Status::OK(); + } + entry->reserved_bytes = block_size; + lookup->admitted = true; + + if (owner->_query_mem_tracker != nullptr) { + switch_query_tracker.emplace(owner->_query_mem_tracker); + } + data = std::make_shared>(block_size); + MonotonicStopWatch fill_watch; + fill_watch.start(); + TEST_SYNC_POINT("CachedRemoteFileReader::reader_local_cache_before_fill"); + const Status load_status = + file_block->read(Slice(data->data(), data->size()), file_block_offset); + lookup->fill_time = fill_watch.elapsed_time(); + { + std::unique_lock lock(_mutex); + entry->load_status = load_status; + entry->loading = false; + if (load_status.ok()) { + entry->data = data; + entry->source_file_block = file_block; + try { + _lru.push_front(block_offset); + } catch (...) { + owner->_cancel_reservation(entry->reserved_bytes); + entry->reserved_bytes = 0; + entry->data.reset(); + data.reset(); + _entries.erase(block_offset); + entry->ready.notify_all(); + lookup->admitted = false; + return Status::OK(); + } + entry->lru_position = _lru.begin(); + entry->in_lru = true; + owner->_commit(entry->reserved_bytes); + entry->reserved_bytes = 0; + lookup->data = std::move(data); + } else { + owner->_cancel_reservation(entry->reserved_bytes); + entry->reserved_bytes = 0; + _entries.erase(block_offset); + } + entry->ready.notify_all(); + } + return load_status; + } catch (...) { + data.reset(); + _abort_load(block_offset, entry); + lookup->admitted = false; + return Status::OK(); + } +} + +size_t FileScannerV2ReaderLocalFileCache::entry_count() const { + std::lock_guard lock(_mutex); + return _entries.size(); +} + static bool use_remote_only_on_cache_miss(const IOContext* io_ctx) { if (io_ctx->file_cache_miss_policy == FileCacheMissPolicy::REMOTE_ONLY_ON_MISS) { return true; @@ -118,15 +546,23 @@ static bool use_remote_only_on_cache_miss(const IOContext* io_ctx) { CachedRemoteFileReader::CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const FileReaderOptions& opts) : _is_doris_table(opts.is_doris_table), + _enable_reader_local_cache(opts.enable_reader_local_cache && + opts.reader_local_cache != nullptr && !opts.is_doris_table), _tablet_id(opts.tablet_id), _storage_resource_id(opts.storage_resource_id), - _remote_file_reader(std::move(remote_file_reader)) { + _remote_file_reader(std::move(remote_file_reader)), + _reader_local_cache(opts.reader_local_cache) { DCHECK(!_is_doris_table || _tablet_id > 0); if (_is_doris_table) { _init_doris_table_cache(); } else { _init_external_table_cache(opts); } + if (_enable_reader_local_cache) { + // The block map follows this physical reader rather than surviving in a query registry. + _reader_local_file_cache = _reader_local_cache->create_file_cache(); + _enable_reader_local_cache = _reader_local_file_cache != nullptr; + } } void CachedRemoteFileReader::_init_doris_table_cache() { @@ -771,9 +1207,9 @@ bool CachedRemoteFileReader::_try_read_from_cached_files_directly( g_skip_local_cache_io_sum_bytes << reserve_bytes; } else { SCOPED_RAW_TIMER(&stats.local_read_timer); - if (!iter->second - ->read(Slice(result.data + (current_offset - offset), reserve_bytes), - file_offset) + if (!_read_local_block(iter->second, file_offset, current_offset, + Slice(result.data + (current_offset - offset), reserve_bytes), + stats) .ok()) { // TODO: maybe read failed because block evict, should handle error break; } @@ -800,6 +1236,145 @@ bool CachedRemoteFileReader::_try_read_from_cached_files_directly( return false; } +bool CachedRemoteFileReader::_read_from_memory_block_cache(size_t offset, Slice result, + ReadStatistics* stats) { + if (!_enable_reader_local_cache || _reader_local_file_cache == nullptr || _cache == nullptr) { + return false; + } + size_t current_offset = offset; + const size_t request_end = offset + result.size; + struct PinnedRead { + size_t block_offset; + size_t read_offset; + size_t result_offset; + size_t read_size; + FileScannerV2ReaderLocalFileCache::LookupResult lookup; + }; + // Parquet metadata and page reads normally span very few cache blocks. Keep their pins on the + // stack so the direct-memory hot path does not replace FileCache locking with heap allocation. + constexpr size_t INLINE_PINNED_READS = 4; + std::array inline_pinned_reads {}; + std::vector overflow_pinned_reads; + size_t pinned_read_count = 0; + while (current_offset < request_end) { + const size_t block_offset = + current_offset / READER_LOCAL_CACHE_BLOCK_BYTES * READER_LOCAL_CACHE_BLOCK_BYTES; + const size_t read_end = + std::min(request_end, block_offset + READER_LOCAL_CACHE_BLOCK_BYTES); + const size_t read_size = read_end - current_offset; + PinnedRead pinned {.block_offset = block_offset, + .read_offset = current_offset, + .result_offset = current_offset - offset, + .read_size = read_size, + .lookup = {}}; + if (!_reader_local_file_cache->pin_if_present(block_offset, current_offset, read_size, + &pinned.lookup)) { + if (stats != nullptr && pinned_read_count != 0) { + stats->num_reader_local_cache_partial_miss++; + } + return false; + } + try { + if (pinned_read_count < INLINE_PINNED_READS) { + inline_pinned_reads[pinned_read_count] = std::move(pinned); + } else { + overflow_pinned_reads.push_back(std::move(pinned)); + } + } catch (...) { + // Optional hot-cache bookkeeping must never fail the scan under memory pressure. + return false; + } + ++pinned_read_count; + current_offset = read_end; + } + // Pin the complete request before copying. A partial probe must leave the caller's buffer + // untouched because the FileCache fallback will restart the request from its original offset. + auto copy_pinned_read = [&](const PinnedRead& pinned) { + memcpy(result.data + pinned.result_offset, + pinned.lookup.data->data() + pinned.read_offset - pinned.block_offset, + pinned.read_size); + if (pinned.lookup.file_block_to_touch != nullptr) { + _cache->add_need_update_lru_block(pinned.lookup.file_block_to_touch); + if (stats != nullptr) { + stats->num_reader_local_cache_disk_lru_touch++; + } + } + }; + for (size_t i = 0; i < std::min(pinned_read_count, INLINE_PINNED_READS); ++i) { + copy_pinned_read(inline_pinned_reads[i]); + } + for (const auto& pinned : overflow_pinned_reads) { + copy_pinned_read(pinned); + } + if (stats != nullptr) { + stats->num_reader_local_cache_total += cast_set(pinned_read_count); + stats->num_reader_local_cache_hit += cast_set(pinned_read_count); + stats->bytes_reader_local_cache_request += cast_set(result.size); + stats->bytes_read_from_reader_local_cache += cast_set(result.size); + } + return true; +} + +Status CachedRemoteFileReader::_read_local_block(const FileBlockSPtr& block, size_t file_offset, + size_t absolute_offset, Slice result, + ReadStatistics& stats, + bool bypass_reader_local_cache) { + if (!_enable_reader_local_cache || _reader_local_file_cache == nullptr || + bypass_reader_local_cache) { + return block->read(result, file_offset); + } + + size_t current_offset = absolute_offset; + const size_t request_end = absolute_offset + result.size; + while (current_offset < request_end) { + const size_t aligned_offset = + current_offset / READER_LOCAL_CACHE_BLOCK_BYTES * READER_LOCAL_CACHE_BLOCK_BYTES; + const size_t buffer_offset = std::max(aligned_offset, block->range().left); + const size_t buffer_end = std::min({aligned_offset + READER_LOCAL_CACHE_BLOCK_BYTES, + block->range().right + 1, size()}); + const size_t copy_end = std::min(request_end, buffer_end); + const size_t copy_size = copy_end - current_offset; + const size_t result_offset = current_offset - absolute_offset; + + stats.num_reader_local_cache_total++; + stats.bytes_reader_local_cache_request += cast_set(copy_size); + + const size_t buffer_size = buffer_end - buffer_offset; + FileScannerV2ReaderLocalFileCache::LookupResult lookup; + RETURN_IF_ERROR(_reader_local_file_cache->get_or_load( + buffer_offset, buffer_size, block, buffer_offset - block->range().left, &lookup)); + stats.num_reader_local_cache_evict += cast_set(lookup.evicted); + stats.num_reader_local_cache_admission_reject += lookup.admission_rejected ? 1 : 0; + stats.reader_local_cache_fill_timer += lookup.fill_time; + if (lookup.waited) { + stats.num_reader_local_cache_wait++; + stats.reader_local_cache_wait_timer += lookup.wait_time; + } + if (!lookup.admitted) { + // Cache memory is best-effort: preserve the FileCache hit when query or process memory + // is tight instead of failing the scan for an optional promotion. + RETURN_IF_ERROR(block->read(Slice(result.data + result_offset, copy_size), + current_offset - block->range().left)); + stats.num_reader_local_cache_miss++; + current_offset = copy_end; + continue; + } + if (lookup.hit) { + stats.num_reader_local_cache_hit++; + stats.bytes_read_from_reader_local_cache += cast_set(copy_size); + } else { + stats.num_reader_local_cache_miss++; + stats.num_reader_local_cache_fill++; + stats.bytes_read_into_reader_local_cache += cast_set(buffer_size); + } + + memcpy(result.data + result_offset, lookup.data->data() + current_offset - buffer_offset, + copy_size); + current_offset = copy_end; + } + return Status::OK(); +} + std::vector CachedRemoteFileReader::_collect_remote_read_blocks( const FileBlocksHolder& holder, ReadStatistics& stats) { std::vector empty_blocks; @@ -840,17 +1415,13 @@ Status CachedRemoteFileReader::_read_remote_blocks_into_cache( const std::vector& empty_blocks, size_t offset, size_t bytes_req, size_t already_read, Slice result, bool is_dryrun, ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, const IOContext* io_ctx, - size_t& indirect_read_bytes, size_t& empty_start, size_t& empty_end, - PeerFetchedBlockSet& peer_fetched_blocks) { - empty_start = 0; - empty_end = 0; - peer_fetched_blocks.clear(); + size_t& indirect_read_bytes, PeerFetchedBlockSet& fetched_blocks) { if (empty_blocks.empty()) { return Status::OK(); } - empty_start = empty_blocks.front()->range().left; - empty_end = empty_blocks.back()->range().right; + const size_t empty_start = empty_blocks.front()->range().left; + const size_t empty_end = empty_blocks.back()->range().right; const size_t span_read_size = empty_end - empty_start + 1; const auto peer_fetch_layout = build_peer_fetch_layout(empty_blocks, size()); std::unique_ptr buffer; @@ -862,10 +1433,6 @@ Status CachedRemoteFileReader::_read_remote_blocks_into_cache( std::vector> peer_chunks_by_block; if (stats.from_peer_cache) { // Peer returns sparse payloads; remember the exact sparse blocks that were filled. - peer_fetched_blocks.reserve(empty_blocks.size()); - for (const auto& block : empty_blocks) { - peer_fetched_blocks.insert(block.get()); - } peer_chunks_by_block.resize(empty_blocks.size()); for (const auto& chunk : peer_result.chunks) { DCHECK_LT(chunk.block_index, empty_blocks.size()); @@ -874,8 +1441,10 @@ Status CachedRemoteFileReader::_read_remote_blocks_into_cache( } SCOPED_CONCURRENCY_COUNT(ConcurrencyStatsManager::instance().cached_remote_reader_write_back); + fetched_blocks.reserve(fetched_blocks.size() + empty_blocks.size()); for (size_t idx = 0; idx < empty_blocks.size(); ++idx) { auto& block = empty_blocks[idx]; + fetched_blocks.insert(block.get()); if (block->state() == FileBlock::State::SKIP_CACHE) { continue; } @@ -937,8 +1506,7 @@ Status CachedRemoteFileReader::_read_remote_blocks_into_cache( Status CachedRemoteFileReader::_read_remaining_blocks_from_cache( const FileBlocksHolder& holder, size_t offset, size_t bytes_req, Slice result, - bool is_dryrun, size_t empty_start, size_t empty_end, - const PeerFetchedBlockSet& peer_fetched_blocks, ReadStatistics& stats, + bool is_dryrun, const PeerFetchedBlockSet& fetched_blocks, ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, size_t& indirect_read_bytes, size_t* bytes_read, const IOContext* io_ctx) { size_t current_offset = offset + *bytes_read; @@ -957,14 +1525,7 @@ Status CachedRemoteFileReader::_read_remaining_blocks_from_cache( size_t read_size = end_offset > right ? right - current_offset + 1 : end_offset - current_offset + 1; - if (!peer_fetched_blocks.empty() && contains_file_block(peer_fetched_blocks, block)) { - // For sparse peer reads, skip only blocks fetched from peer. Other blocks inside the - // enclosing span may still come from local cache. - *bytes_read += read_size; - current_offset = right + 1; - continue; - } - if (peer_fetched_blocks.empty() && empty_start <= left && right <= empty_end) { + if (contains_file_block(fetched_blocks, block)) { *bytes_read += read_size; current_offset = right + 1; continue; @@ -1004,8 +1565,10 @@ Status CachedRemoteFileReader::_read_remaining_blocks_from_cache( SCOPED_RAW_TIMER(&stats.local_read_timer); SCOPED_CONCURRENCY_COUNT( ConcurrencyStatsManager::instance().cached_remote_reader_local_read); - st = block->read(Slice(result.data + (current_offset - offset), read_size), - file_offset); + st = _read_local_block(block, file_offset, current_offset, + Slice(result.data + (current_offset - offset), read_size), + stats, + io_ctx != nullptr && io_ctx->bypass_reader_local_cache); indirect_read_bytes += read_size; if (st.ok()) { source_read_breakdown.local_bytes += read_size; @@ -1050,6 +1613,120 @@ Status CachedRemoteFileReader::_read_remaining_blocks_from_cache( return Status::OK(); } +Status CachedRemoteFileReader::read_at_from_cache(size_t offset, Slice result, size_t* bytes_read, + bool* cache_hit, const IOContext* io_ctx) { + IOContext default_io_ctx; + if (io_ctx == nullptr) { + io_ctx = &default_io_ctx; + } + const auto read_type = + io_ctx->is_inverted_index + ? FileCacheReadType::INVERTED_INDEX + : (io_ctx->is_index_data ? FileCacheReadType::SEGMENT_FOOTER_INDEX + : FileCacheReadType::DATA); + auto publish_stats = [&](const ReadStatistics& stats, + const SourceReadBreakdown& source_read_breakdown) { + if (io_ctx->is_dryrun) { + return; + } + if (io_ctx->file_cache_stats != nullptr) { + _update_stats(stats, source_read_breakdown, io_ctx->file_cache_stats, read_type); + } + if (!io_ctx->is_warmup) { + FileCacheStatistics increment; + _update_stats(stats, source_read_breakdown, &increment, read_type); + FileCacheMetrics::instance().update(&increment); + } + }; + *bytes_read = 0; + *cache_hit = false; + const size_t bytes_req = std::min(result.size, size() - std::min(offset, size())); + if (bytes_req == 0) { + *cache_hit = true; + return Status::OK(); + } + ReadStatistics stats; + stats.num_exact_cache_probe = 1; + MonotonicStopWatch exact_cache_probe_watch; + exact_cache_probe_watch.start(); + const bool bypass_reader_local_cache = io_ctx != nullptr && io_ctx->bypass_reader_local_cache; + MonotonicStopWatch reader_local_probe_watch; + reader_local_probe_watch.start(); + const bool reader_local_hit = + !bypass_reader_local_cache && + _read_from_memory_block_cache(offset, Slice(result.data, bytes_req), &stats); + stats.reader_local_cache_probe_timer += reader_local_probe_watch.elapsed_time(); + if (reader_local_hit) { + *bytes_read = bytes_req; + *cache_hit = true; + stats.bytes_read = cast_set(bytes_req); + SourceReadBreakdown source_read_breakdown; + source_read_breakdown.local_bytes = cast_set(bytes_req); + stats.num_exact_cache_probe_hit = 1; + stats.exact_cache_probe_timer = exact_cache_probe_watch.elapsed_time(); + publish_stats(stats, source_read_breakdown); + return Status::OK(); + } + + SourceReadBreakdown source_read_breakdown; + stats.bytes_read = cast_set(bytes_req); + const size_t block_size = cast_set(config::file_cache_each_block_size); + const size_t align_left = offset / block_size * block_size; + const size_t request_end = offset + bytes_req; + const size_t align_end = + std::min((request_end + block_size - 1) / block_size * block_size, size()); + const size_t align_size = align_end - align_left; + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + MonotonicStopWatch sw; + sw.start(); + FileBlocks downloaded_blocks; + bool fully_covered = false; + RETURN_IF_ERROR(_cache->get_downloaded_blocks_if_fully_covered( + _cache_hash, align_left, align_size, cache_context, &downloaded_blocks, + &fully_covered)); + stats.cache_get_or_set_timer += sw.elapsed_time(); + if (!fully_covered) { + stats.num_exact_cache_probe_miss = 1; + stats.exact_cache_probe_timer = exact_cache_probe_watch.elapsed_time(); + publish_stats(stats, source_read_breakdown); + return Status::OK(); + } + FileBlocksHolder holder(std::move(downloaded_blocks)); + + for (const auto& block : holder.file_blocks) { + const size_t read_start = std::max(offset, block->range().left); + const size_t read_end = std::min(request_end, block->range().right + 1); + if (read_start >= read_end) { + continue; + } + const size_t read_size = read_end - read_start; + SCOPED_RAW_TIMER(&stats.local_read_timer); + const Status st = _read_local_block(block, read_start - block->range().left, read_start, + Slice(result.data + read_start - offset, read_size), + stats, bypass_reader_local_cache); + if (!st.ok()) { + // A cache file can be evicted between the state check and the read. Preserve the + // cache-only contract and let the normal path self-heal through remote storage. + *bytes_read = 0; + *cache_hit = false; + _cache->remove_if_cached_async(_cache_hash); + stats.num_exact_cache_probe_miss = 1; + stats.exact_cache_probe_timer = exact_cache_probe_watch.elapsed_time(); + publish_stats(stats, source_read_breakdown); + return Status::OK(); + } + source_read_breakdown.local_bytes += cast_set(read_size); + _cache->add_need_update_lru_block(block); + } + *bytes_read = bytes_req; + *cache_hit = true; + stats.num_exact_cache_probe_hit = 1; + stats.exact_cache_probe_timer = exact_cache_probe_watch.elapsed_time(); + publish_stats(stats, source_read_breakdown); + return Status::OK(); +} + Status CachedRemoteFileReader::_read_from_indirect_cache(size_t offset, Slice result, size_t bytes_req, size_t already_read, bool is_dryrun, size_t* bytes_read, @@ -1071,17 +1748,27 @@ Status CachedRemoteFileReader::_read_from_indirect_cache(size_t offset, Slice re stats.cache_get_or_set_timer += sw.elapsed_time(); auto empty_blocks = _collect_remote_read_blocks(holder, stats); - size_t empty_start = 0; - size_t empty_end = 0; - PeerFetchedBlockSet peer_fetched_blocks; - RETURN_IF_ERROR(_read_remote_blocks_into_cache(empty_blocks, offset, bytes_req, already_read, - result, is_dryrun, stats, source_read_breakdown, - io_ctx, indirect_read_bytes, empty_start, - empty_end, peer_fetched_blocks)); + PeerFetchedBlockSet fetched_blocks; + size_t run_start = 0; + for (size_t index = 1; index <= empty_blocks.size(); ++index) { + const bool end_of_run = + index == empty_blocks.size() || + empty_blocks[index - 1]->range().right + 1 != empty_blocks[index]->range().left; + if (!end_of_run) { + continue; + } + // A cache hit is a hard merge boundary. Reading across it would redownload resident data + // and violate the cache-aware miss coalescing invariant used by StarRocks. + std::vector contiguous_misses(empty_blocks.begin() + run_start, + empty_blocks.begin() + index); + RETURN_IF_ERROR(_read_remote_blocks_into_cache( + contiguous_misses, offset, bytes_req, already_read, result, is_dryrun, stats, + source_read_breakdown, io_ctx, indirect_read_bytes, fetched_blocks)); + run_start = index; + } *bytes_read = already_read; RETURN_IF_ERROR(_read_remaining_blocks_from_cache(holder, offset, bytes_req, result, is_dryrun, - empty_start, empty_end, peer_fetched_blocks, - stats, source_read_breakdown, + fetched_blocks, stats, source_read_breakdown, indirect_read_bytes, bytes_read, io_ctx)); g_read_cache_indirect_bytes << indirect_read_bytes; g_read_cache_indirect_total_bytes << *bytes_read; @@ -1246,6 +1933,21 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* } }}; + const bool bypass_reader_local_cache = io_ctx->bypass_reader_local_cache; + MonotonicStopWatch reader_local_probe_watch; + reader_local_probe_watch.start(); + const bool reader_local_hit = + !is_dryrun && !bypass_reader_local_cache && + _read_from_memory_block_cache(offset, Slice(result.data, bytes_req), &stats); + stats.reader_local_cache_probe_timer += reader_local_probe_watch.elapsed_time(); + if (reader_local_hit) { + // A resident file-local block is authoritative for this immutable file identity; avoid + // taking FileCache metadata locks again on the hot path. + *bytes_read = bytes_req; + source_read_breakdown.local_bytes += cast_set(bytes_req); + return Status::OK(); + } + if (use_remote_only_on_cache_miss(io_ctx)) { read_st = _read_remote_only_on_cache_miss(offset, result, bytes_req, is_dryrun, bytes_read, stats, source_read_breakdown, io_ctx); @@ -1318,6 +2020,8 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, const bool has_source_bytes = source_read_breakdown.local_bytes != 0 || source_read_breakdown.remote_bytes != 0 || source_read_breakdown.peer_bytes != 0; + const bool exact_probe_miss_without_io = + read_stats.num_exact_cache_probe_miss != 0 && !has_source_bytes; if (has_source_bytes) { if (source_read_breakdown.local_bytes != 0) { statis->num_local_io_total++; @@ -1337,6 +2041,9 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->bytes_read_from_remote += source_read_breakdown.remote_bytes; statis->remote_io_timer += read_stats.remote_read_timer; } + } else if (exact_probe_miss_without_io) { + // A cache-only miss is a lookup result, not physical remote IO. MergeRange will account + // the subsequent fallback read independently. } else if (read_stats.hit_cache) { statis->num_local_io_total++; statis->bytes_read_from_local += read_stats.bytes_read; @@ -1360,6 +2067,27 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->lock_wait_timer += read_stats.lock_wait_timer; statis->get_timer += read_stats.get_timer; statis->set_timer += read_stats.set_timer; + statis->num_reader_local_cache_total += read_stats.num_reader_local_cache_total; + statis->num_reader_local_cache_hit += read_stats.num_reader_local_cache_hit; + statis->num_reader_local_cache_miss += read_stats.num_reader_local_cache_miss; + statis->num_reader_local_cache_fill += read_stats.num_reader_local_cache_fill; + statis->num_reader_local_cache_evict += read_stats.num_reader_local_cache_evict; + statis->num_reader_local_cache_wait += read_stats.num_reader_local_cache_wait; + statis->num_reader_local_cache_admission_reject += + read_stats.num_reader_local_cache_admission_reject; + statis->num_reader_local_cache_partial_miss += read_stats.num_reader_local_cache_partial_miss; + statis->num_reader_local_cache_disk_lru_touch += + read_stats.num_reader_local_cache_disk_lru_touch; + statis->bytes_reader_local_cache_request += read_stats.bytes_reader_local_cache_request; + statis->bytes_read_from_reader_local_cache += read_stats.bytes_read_from_reader_local_cache; + statis->bytes_read_into_reader_local_cache += read_stats.bytes_read_into_reader_local_cache; + statis->reader_local_cache_fill_timer += read_stats.reader_local_cache_fill_timer; + statis->reader_local_cache_wait_timer += read_stats.reader_local_cache_wait_timer; + statis->reader_local_cache_probe_timer += read_stats.reader_local_cache_probe_timer; + statis->num_exact_cache_probe += read_stats.num_exact_cache_probe; + statis->num_exact_cache_probe_hit += read_stats.num_exact_cache_probe_hit; + statis->num_exact_cache_probe_miss += read_stats.num_exact_cache_probe_miss; + statis->exact_cache_probe_timer += read_stats.exact_cache_probe_timer; auto update_index_stats = [&](int64_t& num_local_io_total, int64_t& num_remote_io_total, int64_t& num_peer_io_total, int64_t& bytes_read_from_local, diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index 5c562c82513f7b..3ca5379b8b4beb 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -17,9 +17,14 @@ #pragma once +#include +#include #include #include +#include #include +#include +#include #include #include #include @@ -42,8 +47,10 @@ struct PeerFetchResult; } // namespace doris::io namespace doris { +class MemTracker; +class MemTrackerLimiter; struct PeerCandidate; -} +} // namespace doris namespace doris::io { struct SourceReadBreakdown { @@ -53,7 +60,110 @@ struct SourceReadBreakdown { }; using PeerFetchedBlockSet = std::unordered_set; +class FileScannerV2ReaderLocalFileCache; + +// Scanner-scoped memory budget for File Scanner V2 reader-local caches. Every physical reader owns +// its block map, matching the lifetime of the corresponding external-file stream. +class FileScannerV2ReaderLocalCache + : public std::enable_shared_from_this { +public: + struct LookupResult { + std::shared_ptr> data; + bool admitted = false; + bool admission_rejected = false; + bool hit = false; + bool waited = false; + size_t evicted = 0; + int64_t wait_time = 0; + int64_t fill_time = 0; + FileBlockSPtr file_block_to_touch; + }; + + explicit FileScannerV2ReaderLocalCache( + size_t capacity, std::shared_ptr query_mem_tracker = nullptr); + ~FileScannerV2ReaderLocalCache(); + + std::shared_ptr create_file_cache(); + + size_t entry_count() const; + size_t memory_usage() const; + int64_t tracked_memory() const; + +private: + friend class FileScannerV2ReaderLocalFileCache; + + bool _reserve(size_t bytes, FileScannerV2ReaderLocalFileCache* requester, size_t* evicted); + bool _try_reserve(size_t bytes); + void _commit(size_t bytes); + void _cancel_reservation(size_t bytes); + void _release(size_t bytes); + std::vector> _file_caches() const; + + const size_t _capacity; + std::shared_ptr _query_mem_tracker; + std::shared_ptr _memory_tracker; + mutable std::mutex _budget_mutex; + size_t _memory_bytes = 0; + size_t _reserved_bytes = 0; + mutable std::mutex _registry_mutex; + std::vector> _files; +}; + +// File-scoped block map for reader-local data. Different external files never contend on this +// mutex; duplicate reads of the same file still share fills and LRU state. +class FileScannerV2ReaderLocalFileCache { +public: + using LookupResult = FileScannerV2ReaderLocalCache::LookupResult; + + ~FileScannerV2ReaderLocalFileCache(); + + bool read_if_present(size_t block_offset, size_t read_offset, Slice result, + LookupResult* lookup); + bool pin_if_present(size_t block_offset, size_t read_offset, size_t read_size, + LookupResult* lookup); + Status get_or_load(size_t block_offset, size_t block_size, const FileBlockSPtr& file_block, + size_t file_block_offset, LookupResult* lookup); + + size_t entry_count() const; + +private: + friend class FileScannerV2ReaderLocalCache; + + struct Entry { + std::shared_ptr> data; + std::weak_ptr source_file_block; + Status load_status = Status::OK(); + std::condition_variable_any ready; + std::list::iterator lru_position; + std::atomic hit_count {0}; + size_t reserved_bytes = 0; + bool loading = true; + bool in_lru = false; + }; + + explicit FileScannerV2ReaderLocalFileCache( + std::shared_ptr owner); + void _touch_locked(const std::shared_ptr& entry); + void _abort_load(size_t block_offset, const std::shared_ptr& entry); + void _drain(FileScannerV2ReaderLocalCache* owner); + bool _evict_one(); + + std::weak_ptr _owner; + mutable std::shared_mutex _mutex; + std::map> _entries; + std::list _lru; + static constexpr uint32_t LRU_TOUCH_INTERVAL = 64; +}; + +class ExactCacheReader { +public: + virtual ~ExactCacheReader() = default; + virtual Status read_at_from_cache(size_t offset, Slice result, size_t* bytes_read, + bool* cache_hit, const IOContext* io_ctx) = 0; +}; + class CachedRemoteFileReader final : public FileReader, + public ExactCacheReader, public std::enable_shared_from_this { public: /// Construct a cached reader on top of a remote reader. @@ -94,6 +204,8 @@ class CachedRemoteFileReader final : public FileReader, static std::pair s_align_size(size_t offset, size_t size, size_t length); int64_t mtime() const override { return _remote_file_reader->mtime(); } + Status read_at_from_cache(size_t offset, Slice result, size_t* bytes_read, bool* cache_hit, + const IOContext* io_ctx = nullptr) override; // Asynchronously prefetch a range of file cache blocks. // This method triggers read file cache in dryrun mode to warm up the cache @@ -178,17 +290,14 @@ class CachedRemoteFileReader final : public FileReader, /// @param[in,out] stats Read statistics updated for remote and local cache work. /// @param[in] io_ctx IO context passed to peer/S3 reads. /// @param[in,out] indirect_read_bytes Bytes copied into result through the indirect path. - /// @param[out] empty_start Left boundary of the fetched contiguous empty range. - /// @param[out] empty_end Right boundary of the fetched contiguous empty range. - /// @param[out] peer_fetched_blocks Exact blocks fetched by peer in sparse mode; empty for S3. + /// @param[in,out] fetched_blocks Exact blocks fetched from peer or remote storage. /// @return OK on success; otherwise an error from peer/S3 read. Status _read_remote_blocks_into_cache(const std::vector& empty_blocks, size_t offset, size_t bytes_req, size_t already_read, Slice result, bool is_dryrun, ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, const IOContext* io_ctx, size_t& indirect_read_bytes, - size_t& empty_start, size_t& empty_end, - PeerFetchedBlockSet& peer_fetched_blocks); + PeerFetchedBlockSet& fetched_blocks); /// Read cached blocks that were not covered by the remote-fetch range, with remote fallback. /// @param[in] holder Cache blocks covering the aligned request range. @@ -196,17 +305,14 @@ class CachedRemoteFileReader final : public FileReader, /// @param[in] bytes_req Original request size. /// @param[out] result Destination buffer for the original request. /// @param[in] is_dryrun True if local cache IO should be skipped. - /// @param[in] empty_start Left boundary of the range already handled by remote fetch. - /// @param[in] empty_end Right boundary of the range already handled by remote fetch. - /// @param[in] peer_fetched_blocks Exact blocks already filled by peer; empty for S3 path. + /// @param[in] fetched_blocks Exact blocks already handled by remote fetch. /// @param[in,out] stats Read statistics updated for wait, cache, and remote fallback paths. /// @param[in,out] indirect_read_bytes Bytes copied into result through this indirect stage. /// @param[out] bytes_read Total bytes covered for the original request after this stage. /// @return OK on success; otherwise an error from cache read or remote fallback read. Status _read_remaining_blocks_from_cache(const FileBlocksHolder& holder, size_t offset, size_t bytes_req, Slice result, bool is_dryrun, - size_t empty_start, size_t empty_end, - const PeerFetchedBlockSet& peer_fetched_blocks, + const PeerFetchedBlockSet& fetched_blocks, ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, size_t& indirect_read_bytes, size_t* bytes_read, @@ -319,7 +425,16 @@ class CachedRemoteFileReader final : public FileReader, const SourceReadBreakdown& source_read_breakdown, FileCacheStatistics* state, FileCacheReadType read_type) const; + /// Try a fully resident reader-local range before consulting FileCache metadata. + bool _read_from_memory_block_cache(size_t offset, Slice result, ReadStatistics* stats); + + /// Read one FileCache block through 256 KiB reader-local sub-blocks. + Status _read_local_block(const FileBlockSPtr& block, size_t file_offset, size_t absolute_offset, + Slice result, ReadStatistics& stats, + bool bypass_reader_local_cache = false); + bool _is_doris_table = false; + bool _enable_reader_local_cache = false; int64_t _tablet_id = -1; std::string _storage_resource_id; FileReaderSPtr _remote_file_reader; @@ -327,6 +442,11 @@ class CachedRemoteFileReader final : public FileReader, BlockFileCache* _cache = nullptr; std::shared_mutex _mtx; std::map _cache_file_readers; + std::shared_ptr _reader_local_cache; + std::shared_ptr _reader_local_file_cache; + // Keep reader-local fills smaller than the FileCache block so page reuse does not promote an + // entire large block. + static constexpr size_t READER_LOCAL_CACHE_BLOCK_BYTES = 256 * 1024; }; } // namespace doris::io diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 8262162564cc78..9873904f7e63af 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -87,6 +87,25 @@ struct ReadStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + int64_t num_reader_local_cache_total = 0; + int64_t num_reader_local_cache_hit = 0; + int64_t num_reader_local_cache_miss = 0; + int64_t num_reader_local_cache_fill = 0; + int64_t num_reader_local_cache_evict = 0; + int64_t num_reader_local_cache_wait = 0; + int64_t num_reader_local_cache_admission_reject = 0; + int64_t num_reader_local_cache_partial_miss = 0; + int64_t num_reader_local_cache_disk_lru_touch = 0; + int64_t bytes_reader_local_cache_request = 0; + int64_t bytes_read_from_reader_local_cache = 0; + int64_t bytes_read_into_reader_local_cache = 0; + int64_t reader_local_cache_fill_timer = 0; + int64_t reader_local_cache_wait_timer = 0; + int64_t reader_local_cache_probe_timer = 0; + int64_t num_exact_cache_probe = 0; + int64_t num_exact_cache_probe_hit = 0; + int64_t num_exact_cache_probe_miss = 0; + int64_t exact_cache_probe_timer = 0; }; class BlockFileCache; diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 91eafdba4319fe..2f43642427798d 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -31,6 +31,7 @@ #include "common/config.h" #include "common/status.h" #include "core/custom_allocator.h" +#include "io/io_common.h" #include "runtime/exec_env.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" @@ -45,6 +46,67 @@ namespace doris { namespace io { struct IOContext; +Status MergeRangeFileReader::add_random_access_ranges(const std::vector& ranges, + uint32_t stage) { + for (const auto& range : ranges) { + if (range.start_offset >= range.end_offset) { + return Status::InvalidArgument("Invalid merge-read range [{}, {})", range.start_offset, + range.end_offset); + } + auto it = std::lower_bound(_random_access_ranges.begin(), _random_access_ranges.end(), + range.start_offset, + [](const PrefetchRange& lhs, size_t start_offset) { + return lhs.start_offset < start_offset; + }); + if (it != _random_access_ranges.begin() && std::prev(it)->end_offset > range.start_offset) { + --it; + } + const size_t first = static_cast(it - _random_access_ranges.begin()); + size_t last = first; + size_t merged_start = range.start_offset; + size_t merged_end = range.end_offset; + uint32_t merged_stage = stage; + while (last < _random_access_ranges.size() && + _random_access_ranges[last].start_offset < merged_end) { + if (_random_access_ranges[last].end_offset <= merged_start) { + ++last; + continue; + } + merged_start = std::min(merged_start, _random_access_ranges[last].start_offset); + merged_end = std::max(merged_end, _random_access_ranges[last].end_offset); + merged_stage = std::min(merged_stage, _range_stages[last]); + ++last; + } + if (last == first) { + _random_access_ranges.insert(it, range); + _range_cached_data.insert(_range_cached_data.begin() + first, RangeCachedData {}); + _range_stages.insert(_range_stages.begin() + first, stage); + continue; + } + if (last == first + 1 && merged_start == _random_access_ranges[first].start_offset && + merged_end == _random_access_ranges[first].end_offset) { + _range_stages[first] = merged_stage; + continue; + } + // PARQUET-816 padding can make staged column-chunk ranges legitimately overlap. Replace + // the whole connected component so ranges stay non-overlapping, and drop cached boxes + // whose coordinates were defined by the old boundaries before realigning these vectors. + for (size_t index = first; index < last; ++index) { + _clean_cached_data(_range_cached_data[index]); + } + _random_access_ranges.erase(_random_access_ranges.begin() + first, + _random_access_ranges.begin() + last); + _range_cached_data.erase(_range_cached_data.begin() + first, + _range_cached_data.begin() + last); + _range_stages.erase(_range_stages.begin() + first, _range_stages.begin() + last); + _random_access_ranges.insert(_random_access_ranges.begin() + first, + PrefetchRange {merged_start, merged_end}); + _range_cached_data.insert(_range_cached_data.begin() + first, RangeCachedData {}); + _range_stages.insert(_range_stages.begin() + first, merged_stage); + } + return Status::OK(); +} + // add bvar to capture the download bytes per second by buffered reader bvar::Adder g_bytes_downloaded("buffered_reader", "bytes_downloaded"); bvar::PerSecond> g_bytes_downloaded_per_second("buffered_reader", @@ -65,6 +127,7 @@ Status MergeRangeFileReader::read_at_impl(size_t offset, Slice result, size_t* b _statistics.merged_io++; _statistics.request_bytes += *bytes_read; _statistics.merged_bytes += *bytes_read; + _record_merged_read(-1, offset, *bytes_read); return st; } if (offset + result.size > _random_access_ranges[range_index].end_offset) { @@ -93,6 +156,34 @@ Status MergeRangeFileReader::read_at_impl(size_t offset, Slice result, size_t* b } size_t to_read = result.size - has_read; + if (_exact_cache_reader != nullptr) { + size_t cache_bytes_read = 0; + bool cache_hit = false; + const auto cache_read_start = std::chrono::steady_clock::now(); + RETURN_IF_ERROR(_exact_cache_reader->read_at_from_cache( + offset + has_read, Slice(result.data + has_read, to_read), &cache_bytes_read, + &cache_hit, io_ctx)); + if (_exact_cache_file_stats != nullptr) { + _exact_cache_file_stats->read_time_ns += + std::chrono::duration_cast( + std::chrono::steady_clock::now() - cache_read_start) + .count(); + } + if (cache_hit) { + if (cache_bytes_read != to_read) { + return Status::IOError("Short exact cache read: expected {}, got {}", to_read, + cache_bytes_read); + } + *bytes_read = has_read + cache_bytes_read; + _statistics.request_bytes += cache_bytes_read; + _statistics.cache_hit_bytes += cache_bytes_read; + if (_exact_cache_file_stats != nullptr) { + _exact_cache_file_stats->read_calls++; + _exact_cache_file_stats->read_bytes += cache_bytes_read; + } + return Status::OK(); + } + } if (to_read >= SMALL_IO || to_read >= _remaining) { SCOPED_RAW_TIMER(&_statistics.read_time); size_t read_size = 0; @@ -102,6 +193,7 @@ Status MergeRangeFileReader::read_at_impl(size_t offset, Slice result, size_t* b _statistics.merged_io++; _statistics.request_bytes += read_size; _statistics.merged_bytes += read_size; + _record_merged_read(range_index, offset + has_read, read_size); return Status::OK(); } @@ -197,6 +289,7 @@ Status MergeRangeFileReader::read_at_impl(size_t offset, Slice result, size_t* b _statistics.merged_io++; _statistics.request_bytes += read_size; _statistics.merged_bytes += read_size; + _record_merged_read(range_index, offset + has_read, read_size); return Status::OK(); } @@ -244,11 +337,11 @@ int MergeRangeFileReader::_search_read_range(size_t start_offset, size_t end_off void MergeRangeFileReader::_clean_cached_data(RangeCachedData& cached_data) { if (!cached_data.empty()) { - for (int i = 0; i < cached_data.ref_box.size(); ++i) { + for (size_t i = 0; i < cached_data.ref_box.size(); ++i) { DCHECK_GT(cached_data.box_end_offset[i], cached_data.box_start_offset[i]); - int16_t box_index = cached_data.ref_box[i]; + const int16_t box_index = cached_data.ref_box[i]; DCHECK_GT(_box_ref[box_index], 0); - _box_ref[box_index]--; + _dec_box_ref(box_index); } } cached_data.reset(); @@ -322,16 +415,20 @@ Status MergeRangeFileReader::_fill_box(int range_index, size_t start_offset, siz *bytes_read = 0; { SCOPED_RAW_TIMER(&_statistics.read_time); + IOContext merge_io_ctx = io_ctx != nullptr ? *io_ctx : IOContext {}; + // MergeRange retains the merged slice in its boxes; bypassing reader-local promotion keeps + // one owner for these bytes instead of duplicating every cached block in both buffers. + merge_io_ctx.bypass_reader_local_cache = true; RETURN_IF_ERROR(_reader->read_at(start_offset, Slice(_read_slice->data(), to_read), - bytes_read, io_ctx)); + bytes_read, &merge_io_ctx)); _statistics.merged_io++; _statistics.merged_bytes += *bytes_read; } + _record_merged_read(range_index, start_offset, *bytes_read); SCOPED_RAW_TIMER(&_statistics.copy_time); size_t copy_start = start_offset; const size_t copy_end = start_offset + *bytes_read; - // copy data into small boxes // tuple(box_index, box_start_offset, file_start_offset, file_end_offset) std::vector> filled_boxes; @@ -358,7 +455,7 @@ Status MergeRangeFileReader::_fill_box(int range_index, size_t start_offset, siz const PrefetchRange& fill_range = _random_access_ranges[fill_range_index]; if (fill_range.start_offset > copy_start) { // don't copy hollow data - size_t hollow_size = fill_range.start_offset - copy_start; + const size_t hollow_size = fill_range.start_offset - copy_start; DCHECK_GT(copy_end - copy_start, hollow_size); copy_start += hollow_size; } @@ -397,6 +494,39 @@ Status MergeRangeFileReader::_fill_box(int range_index, size_t start_offset, siz return Status::OK(); } +void MergeRangeFileReader::_record_merged_read(int range_index, size_t start_offset, + size_t bytes_read) { + if (bytes_read == 0) { + return; + } + if (range_index < 0) { + _statistics.merged_useful_bytes += bytes_read; + return; + } + const size_t read_end = start_offset + bytes_read; + size_t useful_bytes = 0; + size_t future_predicate_bytes = 0; + for (size_t index = static_cast(range_index); + index < _random_access_ranges.size() && + _random_access_ranges[index].start_offset < read_end; + ++index) { + const auto& range = _random_access_ranges[index]; + const size_t overlap_start = std::max(start_offset, range.start_offset); + const size_t overlap_end = std::min(read_end, range.end_offset); + if (overlap_start >= overlap_end) { + continue; + } + const size_t overlap = overlap_end - overlap_start; + useful_bytes += overlap; + if (_range_stages[index] > _range_stages[range_index]) { + future_predicate_bytes += overlap; + } + } + _statistics.merged_useful_bytes += useful_bytes; + _statistics.merged_gap_bytes += bytes_read - useful_bytes; + _statistics.future_predicate_prefetch_bytes += future_predicate_bytes; +} + // there exists occasions where the buffer is already closed but // some prior tasks are still queued in thread pool, so we have to check whether // the buffer is closed each time the condition variable is notified. diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index 04566bea0951da..b445f8d028d3d4 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -37,6 +37,7 @@ #include "io/fs/file_reader.h" #include "io/fs/path.h" #include "io/fs/s3_file_reader.h" +#include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" @@ -230,6 +231,10 @@ class MergeRangeFileReader : public io::FileReader { int64_t merged_io = 0; int64_t request_bytes = 0; int64_t merged_bytes = 0; + int64_t cache_hit_bytes = 0; + int64_t merged_useful_bytes = 0; + int64_t merged_gap_bytes = 0; + int64_t future_predicate_prefetch_bytes = 0; }; struct RangeCachedData { @@ -288,7 +293,15 @@ class MergeRangeFileReader : public io::FileReader { _reader(std::move(reader)), _random_access_ranges(random_access_ranges) { _range_cached_data.resize(random_access_ranges.size()); + _range_stages.resize(random_access_ranges.size(), 0); _size = _reader->size(); + io::FileReaderSPtr exact_cache_candidate = _reader; + if (auto tracing_reader = + std::dynamic_pointer_cast(exact_cache_candidate)) { + _exact_cache_file_stats = tracing_reader->stats(); + exact_cache_candidate = tracing_reader->inner_reader(); + } + _exact_cache_reader = dynamic_cast(exact_cache_candidate.get()); _remaining = TOTAL_BUFFER_SIZE; _is_oss = typeid_cast(_reader.get()) != nullptr; _max_amplified_ratio = config::max_amplified_read_ratio; @@ -318,6 +331,14 @@ class MergeRangeFileReader : public io::FileReader { random_profile, 1); _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, random_profile, 1); + _cache_hit_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ExactCacheHitBytes", + TUnit::BYTES, random_profile, 1); + _merged_useful_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedUsefulBytes", + TUnit::BYTES, random_profile, 1); + _merged_gap_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedGapBytes", + TUnit::BYTES, random_profile, 1); + _future_predicate_prefetch_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "FuturePredicatePrefetchBytes", TUnit::BYTES, random_profile, 1); } } @@ -350,6 +371,11 @@ class MergeRangeFileReader : public io::FileReader { // for test only const Statistics& statistics() const { return _statistics; } + // Make a predicate stage visible to coalescing only when execution reaches that stage. Ranges + // already consumed by earlier stages keep stable cache state while newly exposed ranges are + // inserted in file-offset order. + Status add_random_access_ranges(const std::vector& ranges, uint32_t stage); + protected: Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, const IOContext* io_ctx) override; @@ -363,6 +389,11 @@ class MergeRangeFileReader : public io::FileReader { COUNTER_UPDATE(_merged_io, _statistics.merged_io); COUNTER_UPDATE(_request_bytes, _statistics.request_bytes); COUNTER_UPDATE(_merged_bytes, _statistics.merged_bytes); + COUNTER_UPDATE(_cache_hit_bytes, _statistics.cache_hit_bytes); + COUNTER_UPDATE(_merged_useful_bytes, _statistics.merged_useful_bytes); + COUNTER_UPDATE(_merged_gap_bytes, _statistics.merged_gap_bytes); + COUNTER_UPDATE(_future_predicate_prefetch_bytes, + _statistics.future_predicate_prefetch_bytes); if (_reader != nullptr) { _reader->collect_profile_before_close(); } @@ -377,6 +408,10 @@ class MergeRangeFileReader : public io::FileReader { RuntimeProfile::Counter* _merged_io = nullptr; RuntimeProfile::Counter* _request_bytes = nullptr; RuntimeProfile::Counter* _merged_bytes = nullptr; + RuntimeProfile::Counter* _cache_hit_bytes = nullptr; + RuntimeProfile::Counter* _merged_useful_bytes = nullptr; + RuntimeProfile::Counter* _merged_gap_bytes = nullptr; + RuntimeProfile::Counter* _future_predicate_prefetch_bytes = nullptr; int _search_read_range(size_t start_offset, size_t end_offset); void _clean_cached_data(RangeCachedData& cached_data); @@ -384,12 +419,14 @@ class MergeRangeFileReader : public io::FileReader { size_t* bytes_read); Status _fill_box(int range_index, size_t start_offset, size_t to_read, size_t* bytes_read, const IOContext* io_ctx); + void _record_merged_read(int range_index, size_t start_offset, size_t bytes_read); void _dec_box_ref(int16_t box_index); RuntimeProfile* _profile = nullptr; io::FileReaderSPtr _reader; - const std::vector _random_access_ranges; + std::vector _random_access_ranges; std::vector _range_cached_data; + std::vector _range_stages; size_t _size; bool _closed = false; size_t _remaining; @@ -403,6 +440,8 @@ class MergeRangeFileReader : public io::FileReader { double _max_amplified_ratio; size_t _equivalent_io_size; int64_t _merged_read_slice_size; + io::ExactCacheReader* _exact_cache_reader = nullptr; + io::FileReaderStats* _exact_cache_file_stats = nullptr; Statistics _statistics; }; diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h index ab00a9823520e8..60f97da3c776ab 100644 --- a/be/src/io/fs/file_reader.h +++ b/be/src/io/fs/file_reader.h @@ -37,6 +37,7 @@ namespace doris { namespace io { class FileSystem; +class FileScannerV2ReaderLocalCache; struct IOContext; enum class FileCachePolicy : uint8_t { @@ -56,6 +57,11 @@ inline FileCachePolicy cache_type_from_string(std::string_view type) { struct FileReaderOptions { FileCachePolicy cache_type {FileCachePolicy::NO_CACHE}; bool is_doris_table = false; + // Keep this opt-in so legacy scanners and internal-table readers retain their existing IO path. + bool enable_reader_local_cache = false; + // File Scanner V2 readers created by one scanner share only the bounded memory owner; each + // physical reader receives its own block map. + std::shared_ptr reader_local_cache {nullptr}; std::string cache_base_path; // Length of the file in bytes, -1 means unset. // If the file length is not set, the file length will be fetched from the file system. diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index c085f93347850f..fd1f99d8e76dc3 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -19,6 +19,7 @@ #include +#include #include #include @@ -38,6 +39,7 @@ enum class ReaderType : uint8_t { namespace io { +class FileScannerV2ReaderLocalCache; class RemoteScanCacheWriteLimiter; enum class FileCacheMissPolicy : uint8_t { @@ -71,6 +73,25 @@ struct FileCacheStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + int64_t num_reader_local_cache_total = 0; + int64_t num_reader_local_cache_hit = 0; + int64_t num_reader_local_cache_miss = 0; + int64_t num_reader_local_cache_fill = 0; + int64_t num_reader_local_cache_evict = 0; + int64_t num_reader_local_cache_wait = 0; + int64_t num_reader_local_cache_admission_reject = 0; + int64_t num_reader_local_cache_partial_miss = 0; + int64_t num_reader_local_cache_disk_lru_touch = 0; + int64_t bytes_reader_local_cache_request = 0; + int64_t bytes_read_from_reader_local_cache = 0; + int64_t bytes_read_into_reader_local_cache = 0; + int64_t reader_local_cache_fill_timer = 0; + int64_t reader_local_cache_wait_timer = 0; + int64_t reader_local_cache_probe_timer = 0; + int64_t num_exact_cache_probe = 0; + int64_t num_exact_cache_probe_hit = 0; + int64_t num_exact_cache_probe_miss = 0; + int64_t exact_cache_probe_timer = 0; int64_t inverted_index_num_local_io_total = 0; int64_t inverted_index_num_remote_io_total = 0; @@ -132,6 +153,25 @@ struct FileCacheStatistics { lock_wait_timer += other.lock_wait_timer; get_timer += other.get_timer; set_timer += other.set_timer; + num_reader_local_cache_total += other.num_reader_local_cache_total; + num_reader_local_cache_hit += other.num_reader_local_cache_hit; + num_reader_local_cache_miss += other.num_reader_local_cache_miss; + num_reader_local_cache_fill += other.num_reader_local_cache_fill; + num_reader_local_cache_evict += other.num_reader_local_cache_evict; + num_reader_local_cache_wait += other.num_reader_local_cache_wait; + num_reader_local_cache_admission_reject += other.num_reader_local_cache_admission_reject; + num_reader_local_cache_partial_miss += other.num_reader_local_cache_partial_miss; + num_reader_local_cache_disk_lru_touch += other.num_reader_local_cache_disk_lru_touch; + bytes_reader_local_cache_request += other.bytes_reader_local_cache_request; + bytes_read_from_reader_local_cache += other.bytes_read_from_reader_local_cache; + bytes_read_into_reader_local_cache += other.bytes_read_into_reader_local_cache; + reader_local_cache_fill_timer += other.reader_local_cache_fill_timer; + reader_local_cache_wait_timer += other.reader_local_cache_wait_timer; + reader_local_cache_probe_timer += other.reader_local_cache_probe_timer; + num_exact_cache_probe += other.num_exact_cache_probe; + num_exact_cache_probe_hit += other.num_exact_cache_probe_hit; + num_exact_cache_probe_miss += other.num_exact_cache_probe_miss; + exact_cache_probe_timer += other.exact_cache_probe_timer; inverted_index_num_local_io_total += other.inverted_index_num_local_io_total; inverted_index_num_remote_io_total += other.inverted_index_num_remote_io_total; @@ -212,6 +252,10 @@ struct IOContext { bool bypass_peer_read {false}; FileCacheMissPolicy file_cache_miss_policy = FileCacheMissPolicy::READ_THROUGH_AND_WRITE_BACK; RemoteScanCacheWriteLimiter* remote_scan_cache_write_limiter = nullptr; // Ref + // MergeRange owns the bytes it buffers, so its fill path must not promote the same bytes into + // the reader-local block map as well. + bool bypass_reader_local_cache {false}; + std::shared_ptr reader_local_cache {nullptr}; }; } // namespace io diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 6d10ab156cfe50..8e4dff88daed9a 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -134,6 +134,28 @@ TEST(FileScannerV2Test, AdaptiveBatchSizeRunsForCountFallbackOnly) { EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(false, false)); } +TEST(FileScannerV2Test, ReaderLocalCacheSwitchDefaultsOnAndCanDisable) { + const bool original = config::enable_file_scanner_v2_reader_local_cache; + Defer restore {[&]() { config::enable_file_scanner_v2_reader_local_cache = original; }}; + + EXPECT_TRUE(original); + MockRuntimeState state; + RuntimeProfile profile("reader_local_cache_switch"); + config::enable_file_scanner_v2_reader_local_cache = true; + FileScannerV2 enabled_scanner(&state, &profile, nullptr); + ASSERT_TRUE(enabled_scanner.TEST_init_io_ctx().ok()); + EXPECT_TRUE(enabled_scanner.TEST_has_reader_local_cache()); + FileScannerV2 another_enabled_scanner(&state, &profile, nullptr); + ASSERT_TRUE(another_enabled_scanner.TEST_init_io_ctx().ok()); + EXPECT_NE(enabled_scanner.TEST_reader_local_cache(), + another_enabled_scanner.TEST_reader_local_cache()); + + config::enable_file_scanner_v2_reader_local_cache = false; + FileScannerV2 disabled_scanner(&state, &profile, nullptr); + ASSERT_TRUE(disabled_scanner.TEST_init_io_ctx().ok()); + EXPECT_FALSE(disabled_scanner.TEST_has_reader_local_cache()); +} + struct RetryableCloseState { int close_calls = 0; }; @@ -720,6 +742,14 @@ TEST(FileScannerV2Test, RealtimeCounterDeltasUseReaderBytesAsRemoteWithoutCacheS EXPECT_EQ(60, deltas.scan_bytes_from_remote_storage); } +TEST(FileScannerV2Test, FileReadBytesProfilePublishesOnlyNewScannerDelta) { + int64_t reported = 0; + EXPECT_EQ(FileScannerV2::TEST_cumulative_profile_delta(100, &reported), 100); + EXPECT_EQ(FileScannerV2::TEST_cumulative_profile_delta(150, &reported), 50); + EXPECT_EQ(FileScannerV2::TEST_cumulative_profile_delta(150, &reported), 0); + EXPECT_EQ(reported, 150); +} + TEST(FileScannerV2Test, RealtimeCounterDeltasUseFileCacheDeltasWhenAvailable) { io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_statistics; diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index af4a17d1b8a418..fba471156b0be3 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -364,6 +364,51 @@ TEST(ParquetV2NativeDecoderTest, FragmentedDictionarySelectionUsesOneConsumerBat EXPECT_EQ(selected_indices, expected); } +TEST(ParquetV2NativeDecoderTest, Q28ShapedDictionarySelectionUsesOneConsumerBatch) { + constexpr size_t DICTIONARY_SIZE = 256; + constexpr size_t VALUE_COUNT = 4096; + std::array dictionary_values {}; + std::iota(dictionary_values.begin(), dictionary_values.end(), 0); + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), DICTIONARY_SIZE).ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 8); + for (uint32_t row = 0; row < VALUE_COUNT; ++row) { + encoder.Put(row % DICTIONARY_SIZE); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 8; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice data(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&data).ok()); + + ParquetSelection selection {.total_values = VALUE_COUNT, .selected_values = 0, .ranges = {}}; + std::vector expected; + for (uint32_t row = 0; row < VALUE_COUNT; ++row) { + if ((row * 37) % 101 >= 5) { + continue; + } + selection.ranges.push_back({.first = row, .count = 1}); + ++selection.selected_values; + expected.push_back(row % DICTIONARY_SIZE); + } + + CaptureDictionaryConsumer consumer; + ASSERT_TRUE(decoder->decode_selected_dictionary_values(selection, consumer).ok()); + + EXPECT_EQ(consumer.consume_calls, 1); + EXPECT_EQ(consumer.indices, expected); +} + TEST(ParquetV2NativeDecoderTest, HighlySparseDictionarySelectionAvoidsFullBatchDecode) { constexpr size_t DICTIONARY_SIZE = 32; constexpr size_t VALUE_COUNT = 256; diff --git a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp index 596d076c5f0bca..7bd40ec849abbf 100644 --- a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp +++ b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp @@ -31,7 +31,7 @@ namespace { TEST(ParquetBenchmarkScenariosTest, DecoderMatrixCoversNativeEncodingAndTypeFamilies) { const auto scenarios = decoder_scenarios(); - EXPECT_EQ(scenarios.size() * 6 * 2, size_t {228}); + EXPECT_EQ(scenarios.size() * 7 * 2, size_t {266}); const std::set> actual = [&] { std::set> values; for (const auto& scenario : scenarios) { diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index cb54cab86b27d8..591ffdfcf9131f 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -21,6 +21,7 @@ #include #include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/parquet_scan.h" #include "io/fs/buffered_reader.h" namespace doris::format::parquet { @@ -101,5 +102,25 @@ TEST(ParquetPageCacheRangeTest, MergeRangeReaderDecisionRejectsEmptyInvalidAndIn valid_ranges, detail::average_prefetch_range_size(valid_ranges), true)); } +TEST(ParquetPageCacheRangeTest, DeferredMergeRangesRetainPredicateAndLazyColumns) { + format::FileScanRequest request; + ASSERT_TRUE(format::FileScanRequestBuilder(&request) + .add_predicate_column(format::LocalColumnId(1)) + .ok()); + ASSERT_TRUE(format::FileScanRequestBuilder(&request) + .add_non_predicate_column(format::LocalColumnId(2)) + .ok()); + ASSERT_TRUE(format::FileScanRequestBuilder(&request) + .add_non_predicate_column(format::LocalColumnId(3)) + .ok()); + request.count_star_placeholder_columns.push_back(format::LocalColumnId(3)); + + const auto columns = detail::deferred_merge_range_columns(request); + + ASSERT_EQ(columns.size(), 2); + EXPECT_EQ(columns[0].column_id(), format::LocalColumnId(1)); + EXPECT_EQ(columns[1].column_id(), format::LocalColumnId(2)); +} + } // namespace } // namespace doris::format::parquet diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index c2aea1cd825253..ebd6c55a229dfd 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -54,6 +54,25 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_io_timer = multiplier * 28; stats.remote_only_on_miss_triggered = multiplier * 29; stats.remote_only_on_miss_threshold_bytes = multiplier * 30; + stats.num_reader_local_cache_total = multiplier * 31; + stats.num_reader_local_cache_hit = multiplier * 32; + stats.num_reader_local_cache_miss = multiplier * 33; + stats.num_reader_local_cache_fill = multiplier * 34; + stats.num_reader_local_cache_evict = multiplier * 35; + stats.num_reader_local_cache_wait = multiplier * 36; + stats.num_reader_local_cache_admission_reject = multiplier * 37; + stats.num_reader_local_cache_partial_miss = multiplier * 38; + stats.num_reader_local_cache_disk_lru_touch = multiplier * 39; + stats.bytes_reader_local_cache_request = multiplier * 40; + stats.bytes_read_from_reader_local_cache = multiplier * 41; + stats.bytes_read_into_reader_local_cache = multiplier * 42; + stats.reader_local_cache_fill_timer = multiplier * 43; + stats.reader_local_cache_wait_timer = multiplier * 44; + stats.reader_local_cache_probe_timer = multiplier * 45; + stats.num_exact_cache_probe = multiplier * 46; + stats.num_exact_cache_probe_hit = multiplier * 47; + stats.num_exact_cache_probe_miss = multiplier * 48; + stats.exact_cache_probe_timer = multiplier * 49; return stats; } @@ -94,6 +113,30 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.remote_only_on_miss_triggered, expected.remote_only_on_miss_triggered); EXPECT_EQ(actual.remote_only_on_miss_threshold_bytes, expected.remote_only_on_miss_threshold_bytes); + EXPECT_EQ(actual.num_reader_local_cache_total, expected.num_reader_local_cache_total); + EXPECT_EQ(actual.num_reader_local_cache_hit, expected.num_reader_local_cache_hit); + EXPECT_EQ(actual.num_reader_local_cache_miss, expected.num_reader_local_cache_miss); + EXPECT_EQ(actual.num_reader_local_cache_fill, expected.num_reader_local_cache_fill); + EXPECT_EQ(actual.num_reader_local_cache_evict, expected.num_reader_local_cache_evict); + EXPECT_EQ(actual.num_reader_local_cache_wait, expected.num_reader_local_cache_wait); + EXPECT_EQ(actual.num_reader_local_cache_admission_reject, + expected.num_reader_local_cache_admission_reject); + EXPECT_EQ(actual.num_reader_local_cache_partial_miss, + expected.num_reader_local_cache_partial_miss); + EXPECT_EQ(actual.num_reader_local_cache_disk_lru_touch, + expected.num_reader_local_cache_disk_lru_touch); + EXPECT_EQ(actual.bytes_reader_local_cache_request, expected.bytes_reader_local_cache_request); + EXPECT_EQ(actual.bytes_read_from_reader_local_cache, + expected.bytes_read_from_reader_local_cache); + EXPECT_EQ(actual.bytes_read_into_reader_local_cache, + expected.bytes_read_into_reader_local_cache); + EXPECT_EQ(actual.reader_local_cache_fill_timer, expected.reader_local_cache_fill_timer); + EXPECT_EQ(actual.reader_local_cache_wait_timer, expected.reader_local_cache_wait_timer); + EXPECT_EQ(actual.reader_local_cache_probe_timer, expected.reader_local_cache_probe_timer); + EXPECT_EQ(actual.num_exact_cache_probe, expected.num_exact_cache_probe); + EXPECT_EQ(actual.num_exact_cache_probe_hit, expected.num_exact_cache_probe_hit); + EXPECT_EQ(actual.num_exact_cache_probe_miss, expected.num_exact_cache_probe_miss); + EXPECT_EQ(actual.exact_cache_probe_timer, expected.exact_cache_probe_timer); } } // namespace @@ -139,6 +182,28 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("ReaderLocalCacheRequests")->value(), + after_second_report.num_reader_local_cache_total); + EXPECT_EQ(profile->get_counter("ReaderLocalCacheHitBytes")->value(), + after_second_report.bytes_read_from_reader_local_cache); + EXPECT_EQ(profile->get_counter("ReaderLocalCacheFillBytes")->value(), + after_second_report.bytes_read_into_reader_local_cache); + EXPECT_EQ(profile->get_counter("ReaderLocalCacheAdmissionRejects")->value(), + after_second_report.num_reader_local_cache_admission_reject); + EXPECT_EQ(profile->get_counter("ReaderLocalCachePartialMisses")->value(), + after_second_report.num_reader_local_cache_partial_miss); + EXPECT_EQ(profile->get_counter("ReaderLocalCacheDiskLRUTouches")->value(), + after_second_report.num_reader_local_cache_disk_lru_touch); + EXPECT_EQ(profile->get_counter("ReaderLocalCacheProbeTimer")->value(), + after_second_report.reader_local_cache_probe_timer); + EXPECT_EQ(profile->get_counter("ExactCacheProbes")->value(), + after_second_report.num_exact_cache_probe); + EXPECT_EQ(profile->get_counter("ExactCacheProbeHits")->value(), + after_second_report.num_exact_cache_probe_hit); + EXPECT_EQ(profile->get_counter("ExactCacheProbeMisses")->value(), + after_second_report.num_exact_cache_probe_miss); + EXPECT_EQ(profile->get_counter("ExactCacheProbeTimer")->value(), + after_second_report.exact_cache_probe_timer); } } // namespace doris diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 4b73dcb65db91b..2fda7c8e081803 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -27,6 +27,7 @@ #include "io/cache/remote_scan_cache_write_limiter.h" #include "io/cache/shard_mem_cache.h" #include "io/fs/buffered_reader.h" +#include "runtime/thread_context.h" #include "storage/olap_define.h" #include "util/debug_points.h" #include "util/defer_op.h" @@ -53,13 +54,14 @@ constexpr size_t kCachedRemoteReaderTinyBlockSize = 4; io::FileCacheSettings create_cached_remote_reader_tiny_settings(size_t block_size, std::string storage = "disk") { io::FileCacheSettings settings; - settings.query_queue_size = 64; + const size_t capacity = std::max(64, block_size * 4); + settings.query_queue_size = capacity; settings.query_queue_elements = 16; - settings.index_queue_size = 64; + settings.index_queue_size = capacity; settings.index_queue_elements = 16; - settings.disposable_queue_size = 64; + settings.disposable_queue_size = capacity; settings.disposable_queue_elements = 16; - settings.capacity = 64; + settings.capacity = capacity; settings.max_file_block_size = block_size; settings.max_query_cache_size = 0; settings.storage = std::move(storage); @@ -335,6 +337,31 @@ class FailAfterOffsetFileReader : public FileReader { bool _fail = false; }; +class RecordingFileReader : public FileReader { +public: + explicit RecordingFileReader(FileReaderSPtr reader) : _reader(std::move(reader)) {} + ~RecordingFileReader() override = default; + + Status close() override { return _reader->close(); } + const Path& path() const override { return _reader->path(); } + size_t size() const override { return _reader->size(); } + bool closed() const override { return _reader->closed(); } + int64_t mtime() const override { return _reader->mtime(); } + + const std::vector& reads() const { return _reads; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + _reads.emplace_back(offset, offset + result.size); + return _reader->read_at(offset, result, bytes_read, io_ctx); + } + +private: + FileReaderSPtr _reader; + std::vector _reads; +}; + void assert_range([[maybe_unused]] size_t assert_n, io::FileBlockSPtr file_block, const io::FileBlock::Range& expected_range, io::FileBlock::State expected_state) { auto range = file_block->range(); @@ -10242,6 +10269,51 @@ TEST_F(BlockFileCacheTest, file_block_appendv_persists_segmented_payload_to_memo EXPECT_EQ(buffer, "abcd"); } +TEST_F(BlockFileCacheTest, cached_remote_reader_merges_only_contiguous_cache_misses) { + const std::string content = "abcdefghijklmnop"; + const fs::path file_path = + create_cached_remote_reader_test_file("contiguous_cache_misses", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "contiguous_cache_misses_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + auto* cache = + create_cached_remote_reader_test_cache(cache_path, kCachedRemoteReaderTinyBlockSize); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + auto recording_reader = std::make_shared(local_reader); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader reader(recording_reader, opts); + + ReadStatistics cache_stats; + auto context = create_cached_remote_reader_context(&cache_stats); + FileBlocksHolder cached_holder = cache->get_or_set(reader._cache_hash, 4, 4, context); + auto cached_blocks = fromHolder(cached_holder); + ASSERT_EQ(cached_blocks.size(), 1); + append_cached_remote_reader_block(cached_blocks[0], content); + + std::string buffer(12, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + EXPECT_EQ(buffer, content.substr(0, buffer.size())); + ASSERT_EQ(recording_reader->reads().size(), 2); + EXPECT_EQ(recording_reader->reads()[0], PrefetchRange(0, 4)); + EXPECT_EQ(recording_reader->reads()[1], PrefetchRange(8, 16)); +} + TEST_F(BlockFileCacheTest, cached_remote_file_reader_read_remote_blocks_into_cache_copy_boundaries) { const std::string content = "abcdefghijklmnop"; @@ -10288,20 +10360,17 @@ TEST_F(BlockFileCacheTest, std::string buffer(8, '#'); size_t indirect_read_bytes = 0; - size_t empty_start = std::numeric_limits::max(); - size_t empty_end = std::numeric_limits::max(); PeerFetchedBlockSet peer_fetched_blocks; ReadStatistics read_stats; SourceReadBreakdown source_read_breakdown; IOContext io_ctx; - ASSERT_TRUE(reader._read_remote_blocks_into_cache( - empty_blocks, 2, 8, 2, Slice(buffer.data(), buffer.size()), false, - read_stats, source_read_breakdown, &io_ctx, indirect_read_bytes, - empty_start, empty_end, peer_fetched_blocks) + ASSERT_TRUE(reader._read_remote_blocks_into_cache(empty_blocks, 2, 8, 2, + Slice(buffer.data(), buffer.size()), false, + read_stats, source_read_breakdown, &io_ctx, + indirect_read_bytes, peer_fetched_blocks) .ok()); - EXPECT_EQ(empty_start, 4); - EXPECT_EQ(empty_end, 11); + EXPECT_EQ(peer_fetched_blocks.size(), empty_blocks.size()); EXPECT_EQ(indirect_read_bytes, 6); EXPECT_EQ(buffer, "##efghij"); EXPECT_EQ(read_stats.bytes_write_into_file_cache, 8); @@ -10357,20 +10426,17 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_read_remote_blocks_into_cac std::string buffer(4, '#'); size_t indirect_read_bytes = 0; - size_t empty_start = 99; - size_t empty_end = 99; PeerFetchedBlockSet peer_fetched_blocks; ReadStatistics read_stats; SourceReadBreakdown source_read_breakdown; IOContext io_ctx; - ASSERT_TRUE(reader._read_remote_blocks_into_cache( - empty_blocks, 1, 3, 0, Slice(buffer.data(), buffer.size()), true, - read_stats, source_read_breakdown, &io_ctx, indirect_read_bytes, - empty_start, empty_end, peer_fetched_blocks) + ASSERT_TRUE(reader._read_remote_blocks_into_cache(empty_blocks, 1, 3, 0, + Slice(buffer.data(), buffer.size()), true, + read_stats, source_read_breakdown, &io_ctx, + indirect_read_bytes, peer_fetched_blocks) .ok()); - EXPECT_EQ(empty_start, 0); - EXPECT_EQ(empty_end, 3); + EXPECT_EQ(peer_fetched_blocks.size(), empty_blocks.size()); EXPECT_EQ(indirect_read_bytes, 0); EXPECT_EQ(buffer, "####"); EXPECT_EQ(read_stats.bytes_write_into_file_cache, 4); @@ -10422,10 +10488,11 @@ TEST_F(BlockFileCacheTest, size_t indirect_read_bytes = 0; size_t bytes_read = 0; PeerFetchedBlockSet peer_fetched_blocks; + peer_fetched_blocks.insert(blocks[1].get()); ReadStatistics read_stats; SourceReadBreakdown source_read_breakdown; ASSERT_TRUE(reader._read_remaining_blocks_from_cache( - holder, 1, 10, Slice(buffer.data(), buffer.size()), false, 4, 7, + holder, 1, 10, Slice(buffer.data(), buffer.size()), false, peer_fetched_blocks, read_stats, source_read_breakdown, indirect_read_bytes, &bytes_read, &io_ctx) .ok()); @@ -10486,7 +10553,7 @@ TEST_F(BlockFileCacheTest, ReadStatistics read_stats; SourceReadBreakdown source_read_breakdown; ASSERT_TRUE(reader._read_remaining_blocks_from_cache( - holder, 1, 10, Slice(buffer.data(), buffer.size()), false, 0, 0, + holder, 1, 10, Slice(buffer.data(), buffer.size()), false, peer_fetched_blocks, read_stats, source_read_breakdown, indirect_read_bytes, &bytes_read, &io_ctx) .ok()); @@ -10542,7 +10609,7 @@ TEST_F(BlockFileCacheTest, ReadStatistics read_stats; SourceReadBreakdown source_read_breakdown; ASSERT_TRUE(reader._read_remaining_blocks_from_cache( - holder, 1, 2, Slice(buffer.data(), buffer.size()), false, 0, 0, + holder, 1, 2, Slice(buffer.data(), buffer.size()), false, peer_fetched_blocks, read_stats, source_read_breakdown, indirect_read_bytes, &bytes_read, &io_ctx) .ok()); @@ -10554,4 +10621,813 @@ TEST_F(BlockFileCacheTest, EXPECT_FALSE(read_stats.from_peer_cache); } +TEST_F(BlockFileCacheTest, external_reader_buffers_small_cache_block_on_first_read) { + const std::string content = "abcdefghijklmnop"; + const fs::path file_path = + create_cached_remote_reader_test_file("external_reader_memory_reuse", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "external_reader_memory_reuse_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, kCachedRemoteReaderTinyBlockSize); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = std::make_shared(16); + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader reader(local_reader, opts); + + std::string warmup(4, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(2, Slice(warmup.data(), warmup.size()), &bytes_read).ok()); + EXPECT_EQ(warmup, "cdef"); + + std::string first(4, '#'); + bool cache_hit = false; + FileCacheStatistics first_stats; + IOContext first_ctx; + first_ctx.file_cache_stats = &first_stats; + ASSERT_TRUE(reader.read_at_from_cache(2, Slice(first.data(), first.size()), &bytes_read, + &cache_hit, &first_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(first, "cdef"); + ASSERT_NE(reader._reader_local_cache, nullptr); + EXPECT_GT(reader._reader_local_cache->entry_count(), 0); + EXPECT_EQ(first_stats.num_reader_local_cache_total, 2); + EXPECT_EQ(first_stats.num_reader_local_cache_miss, 2); + EXPECT_EQ(first_stats.num_reader_local_cache_fill, 2); + EXPECT_EQ(first_stats.bytes_reader_local_cache_request, 4); + EXPECT_EQ(first_stats.bytes_read_into_reader_local_cache, 8); + + std::string second(4, '#'); + cache_hit = false; + ASSERT_TRUE(reader.read_at_from_cache(2, Slice(second.data(), second.size()), &bytes_read, + &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(second, "cdef"); + ASSERT_GT(reader._reader_local_cache->entry_count(), 0); + + auto [align_left, align_size] = + CachedRemoteFileReader::s_align_size(2, first.size(), reader.size()); + ReadStatistics cache_stats; + auto context = create_cached_remote_reader_context(&cache_stats); + auto holder = reader._cache->get_or_set(reader._cache_hash, align_left, align_size, context); + for (const auto& block : holder.file_blocks) { + std::error_code ignore; + fs::remove(block->get_cache_file(), ignore); + } + + std::string reused(4, '#'); + cache_hit = false; + ASSERT_TRUE(reader.read_at_from_cache(2, Slice(reused.data(), reused.size()), &bytes_read, + &cache_hit) + .ok()); + EXPECT_TRUE(cache_hit); + EXPECT_EQ(reused, "cdef"); + + FileCacheStatistics hot_read_stats; + IOContext hot_read_ctx; + hot_read_ctx.file_cache_stats = &hot_read_stats; + std::string hot_read(2, '#'); + ASSERT_TRUE( + reader.read_at(2, Slice(hot_read.data(), hot_read.size()), &bytes_read, &hot_read_ctx) + .ok()); + EXPECT_EQ(hot_read, "cd"); + EXPECT_EQ(hot_read_stats.num_reader_local_cache_hit, 1); + EXPECT_EQ(hot_read_stats.cache_get_or_set_timer, 0); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_zero_capacity_is_disabled) { + auto cache = std::make_shared(0); + EXPECT_EQ(cache->create_file_cache(), nullptr); + EXPECT_EQ(cache->entry_count(), 0); + EXPECT_EQ(cache->memory_usage(), 0); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_partial_probe_does_not_modify_output) { + constexpr size_t reader_block_size = 256_kb; + const std::string content(2 * reader_block_size, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("reader_local_partial_probe", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "reader_local_partial_probe_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 2 * reader_block_size); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = std::make_shared(reader_block_size); + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader reader(local_reader, opts); + + std::string warmup(reader_block_size, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(warmup.data(), warmup.size()), &bytes_read).ok()); + bool cache_hit = false; + ASSERT_TRUE(reader.read_at_from_cache(0, Slice(warmup.data(), warmup.size()), &bytes_read, + &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + ASSERT_TRUE(reader.read_at_from_cache(0, Slice(warmup.data(), warmup.size()), &bytes_read, + &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + + std::string output(16, '#'); + ReadStatistics stats; + EXPECT_FALSE(reader._read_from_memory_block_cache(reader_block_size - 8, + Slice(output.data(), output.size()), &stats)); + EXPECT_EQ(output, std::string(16, '#')); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_fill_exception_is_best_effort) { + const std::string content(8_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("reader_local_fill_exception", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "reader_local_fill_exception_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 8_kb); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = std::make_shared(8_kb); + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader reader(local_reader, opts); + + std::string buffer(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(1_kb, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + auto* sp = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sp->set_call_back( + "CachedRemoteFileReader::reader_local_cache_before_fill", + [&](auto&&) { throw std::bad_alloc(); }, &guard); + sp->enable_processing(); + Defer clear_sync_points {[&]() { sp->disable_processing(); }}; + + bool cache_hit = false; + EXPECT_NO_THROW({ + const auto status = reader.read_at_from_cache(1_kb, Slice(buffer.data(), buffer.size()), + &bytes_read, &cache_hit); + EXPECT_TRUE(status.ok()); + }); + EXPECT_TRUE(cache_hit); + EXPECT_EQ(buffer, std::string(8, 'x')); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_uses_one_cache_object_per_reader) { + auto cache = std::make_shared(1_mb); + auto first = cache->create_file_cache(); + auto second = cache->create_file_cache(); + + ASSERT_NE(first, nullptr); + EXPECT_NE(first, second); + + std::unique_lock first_file_lock(first->_mutex); + auto second_lookup = std::async(std::launch::async, [&]() { + std::string result(1, '#'); + FileScannerV2ReaderLocalFileCache::LookupResult lookup; + return second->read_if_present(0, 0, Slice(result.data(), result.size()), &lookup); + }); + const auto second_lookup_status = second_lookup.wait_for(std::chrono::seconds(1)); + first_file_lock.unlock(); + EXPECT_EQ(second_lookup_status, std::future_status::ready); + EXPECT_FALSE(second_lookup.get()); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_is_opt_in_and_excludes_doris_tables) { + const std::string content = "abcdefghijklmnop"; + const fs::path file_path = + create_cached_remote_reader_test_file("reader_local_cache_scope", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "reader_local_cache_scope_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, kCachedRemoteReaderTinyBlockSize); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader v1_reader(local_reader, opts); + + std::string buffer(4, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(v1_reader.read_at(2, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + for (int i = 0; i < 3; ++i) { + bool cache_hit = false; + ASSERT_TRUE(v1_reader + .read_at_from_cache(2, Slice(buffer.data(), buffer.size()), &bytes_read, + &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + } + EXPECT_EQ(v1_reader._reader_local_cache, nullptr); + + opts.enable_reader_local_cache = true; + opts.is_doris_table = true; + opts.tablet_id = 1; + CachedRemoteFileReader internal_reader(local_reader, opts); + ASSERT_TRUE(internal_reader.read_at(2, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + for (int i = 0; i < 3; ++i) { + bool cache_hit = false; + FileCacheStatistics stats; + IOContext io_ctx; + io_ctx.file_cache_stats = &stats; + ASSERT_TRUE(internal_reader + .read_at_from_cache(2, Slice(buffer.data(), buffer.size()), &bytes_read, + &cache_hit, &io_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(stats.num_reader_local_cache_total, 0); + } + EXPECT_EQ(internal_reader._reader_local_cache, nullptr); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_buffers_full_block_on_first_cache_hit) { + constexpr size_t cache_block_size = 1_mb; + constexpr size_t reader_block_size = 256_kb; + const std::string content(600_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("reader_local_cache_sub_block", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "reader_local_cache_sub_block_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, cache_block_size); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = std::make_shared(16_mb); + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader reader(local_reader, opts); + + std::string buffer(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + + FileCacheStatistics fill_stats; + IOContext fill_ctx; + fill_ctx.file_cache_stats = &fill_stats; + bool cache_hit = false; + ASSERT_TRUE(reader.read_at_from_cache(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read, + &cache_hit, &fill_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + ASSERT_NE(reader._reader_local_cache, nullptr); + EXPECT_EQ(reader._reader_local_cache->entry_count(), 1); + EXPECT_EQ(reader._reader_local_cache->memory_usage(), reader_block_size); + EXPECT_EQ(fill_stats.num_reader_local_cache_miss, 1); + EXPECT_EQ(fill_stats.num_reader_local_cache_fill, 1); + EXPECT_EQ(fill_stats.bytes_reader_local_cache_request, buffer.size()); + EXPECT_EQ(fill_stats.bytes_read_into_reader_local_cache, reader_block_size); + + FileCacheStatistics hit_stats; + IOContext hit_ctx; + hit_ctx.file_cache_stats = &hit_stats; + cache_hit = false; + ASSERT_TRUE(reader.read_at_from_cache(160_kb, Slice(buffer.data(), buffer.size()), &bytes_read, + &cache_hit, &hit_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(hit_stats.num_reader_local_cache_hit, 1); + EXPECT_EQ(hit_stats.bytes_read_from_reader_local_cache, buffer.size()); + EXPECT_EQ(hit_stats.bytes_read_into_reader_local_cache, 0); + + FileCacheStatistics cross_block_stats; + IOContext cross_block_ctx; + cross_block_ctx.file_cache_stats = &cross_block_stats; + std::string cross_block_buffer(8, '#'); + ASSERT_TRUE(reader.read_at(reader_block_size - 4, + Slice(cross_block_buffer.data(), cross_block_buffer.size()), + &bytes_read, &cross_block_ctx) + .ok()); + EXPECT_EQ(cross_block_buffer, std::string(8, 'x')); + // The all-or-none memory probe must leave the output untouched before FileCache serves the + // first part from memory and promotes the missing second block. + EXPECT_EQ(cross_block_stats.num_reader_local_cache_partial_miss, 1); + EXPECT_EQ(cross_block_stats.num_reader_local_cache_hit, 1); + EXPECT_EQ(cross_block_stats.num_reader_local_cache_fill, 1); +} + +TEST_F(BlockFileCacheTest, concurrent_reader_local_cache_fill_is_single_flight) { + const std::string content(8_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("reader_local_cache_single_flight", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "reader_local_cache_single_flight_cache"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 8_kb); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = std::make_shared(8_kb); + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + auto loader_reader = std::make_shared(local_reader, opts); + auto waiter_reader = loader_reader; + + std::string warmup(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE( + loader_reader->read_at(1_kb, Slice(warmup.data(), warmup.size()), &bytes_read).ok()); + std::promise loader_entered; + std::promise waiter_entered; + std::promise release_loader; + auto release_future = release_loader.get_future().share(); + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("CachedRemoteFileReader::reader_local_cache_before_fill", [&](auto&&) { + loader_entered.set_value(); + release_future.wait(); + }); + sp->set_call_back("CachedRemoteFileReader::reader_local_cache_before_wait", + [&](auto&&) { waiter_entered.set_value(); }); + sp->enable_processing(); + Defer clear_sync_points {[&]() { + sp->clear_call_back("CachedRemoteFileReader::reader_local_cache_before_fill"); + sp->clear_call_back("CachedRemoteFileReader::reader_local_cache_before_wait"); + sp->disable_processing(); + }}; + + auto read_cache = [](const std::shared_ptr& reader, + FileCacheStatistics* stats) { + SCOPED_INIT_THREAD_CONTEXT(); + std::string buffer(8, '#'); + size_t read_size = 0; + bool cache_hit = false; + IOContext io_ctx; + io_ctx.file_cache_stats = stats; + Status st = reader->read_at_from_cache(1_kb, Slice(buffer.data(), buffer.size()), + &read_size, &cache_hit, &io_ctx); + return std::tuple(st, cache_hit, buffer); + }; + + FileCacheStatistics loader_stats; + FileCacheStatistics waiter_stats; + auto loader = std::async(std::launch::async, read_cache, loader_reader, &loader_stats); + auto loader_ready = loader_entered.get_future(); + if (loader_ready.wait_for(std::chrono::seconds(5)) != std::future_status::ready) { + release_loader.set_value(); + FAIL() << "the first reader did not start the reader-local fill"; + } + auto waiter = std::async(std::launch::async, read_cache, waiter_reader, &waiter_stats); + auto waiter_ready = waiter_entered.get_future(); + if (waiter_ready.wait_for(std::chrono::seconds(5)) != std::future_status::ready) { + release_loader.set_value(); + FAIL() << "the second reader did not wait for the in-flight fill"; + } + release_loader.set_value(); + + auto [loader_status, loader_hit, loader_buffer] = loader.get(); + auto [waiter_status, waiter_hit, waiter_buffer] = waiter.get(); + ASSERT_TRUE(loader_status.ok()); + ASSERT_TRUE(waiter_status.ok()); + EXPECT_TRUE(loader_hit); + EXPECT_TRUE(waiter_hit); + EXPECT_EQ(loader_buffer, std::string(8, 'x')); + EXPECT_EQ(waiter_buffer, std::string(8, 'x')); + EXPECT_EQ(loader_stats.num_reader_local_cache_miss, 1); + EXPECT_EQ(loader_stats.num_reader_local_cache_fill, 1); + EXPECT_EQ(waiter_stats.num_reader_local_cache_wait, 1); + EXPECT_EQ(waiter_stats.num_reader_local_cache_hit, 1); + EXPECT_EQ(loader_stats.num_reader_local_cache_fill + waiter_stats.num_reader_local_cache_fill, + 1); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_fill_exception_wakes_waiter) { + const std::string content(8_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("reader_local_cache_exception_waiter", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "reader_local_cache_exception_waiter_blocks"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 8_kb); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = std::make_shared(8_kb); + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + auto loader_reader = std::make_shared(local_reader, opts); + auto waiter_reader = loader_reader; + + std::string warmup(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE( + loader_reader->read_at(1_kb, Slice(warmup.data(), warmup.size()), &bytes_read).ok()); + std::promise loader_entered; + std::promise waiter_entered; + std::promise release_loader; + auto release_future = release_loader.get_future().share(); + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("CachedRemoteFileReader::reader_local_cache_before_fill", [&](auto&&) { + loader_entered.set_value(); + release_future.wait(); + throw std::bad_alloc(); + }); + sp->set_call_back("CachedRemoteFileReader::reader_local_cache_before_wait", + [&](auto&&) { waiter_entered.set_value(); }); + sp->enable_processing(); + Defer clear_sync_points {[&]() { + sp->clear_call_back("CachedRemoteFileReader::reader_local_cache_before_fill"); + sp->clear_call_back("CachedRemoteFileReader::reader_local_cache_before_wait"); + sp->disable_processing(); + }}; + + auto read_cache = [](const std::shared_ptr& reader) { + SCOPED_INIT_THREAD_CONTEXT(); + std::string buffer(8, '#'); + size_t read_size = 0; + bool cache_hit = false; + Status st = reader->read_at_from_cache(1_kb, Slice(buffer.data(), buffer.size()), + &read_size, &cache_hit); + return std::tuple(st, cache_hit, buffer); + }; + + auto loader = std::async(std::launch::async, read_cache, loader_reader); + if (loader_entered.get_future().wait_for(std::chrono::seconds(5)) != + std::future_status::ready) { + release_loader.set_value(); + FAIL() << "the loader did not enter the reader-local fill"; + } + auto waiter = std::async(std::launch::async, read_cache, waiter_reader); + if (waiter_entered.get_future().wait_for(std::chrono::seconds(5)) != + std::future_status::ready) { + release_loader.set_value(); + FAIL() << "the waiter did not observe the in-flight reader-local fill"; + } + release_loader.set_value(); + + ASSERT_EQ(loader.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_EQ(waiter.wait_for(std::chrono::seconds(5)), std::future_status::ready); + const auto [loader_status, loader_hit, loader_buffer] = loader.get(); + const auto [waiter_status, waiter_hit, waiter_buffer] = waiter.get(); + EXPECT_TRUE(loader_status.ok()); + EXPECT_TRUE(waiter_status.ok()); + EXPECT_TRUE(loader_hit); + EXPECT_TRUE(waiter_hit); + EXPECT_EQ(loader_buffer, std::string(8, 'x')); + EXPECT_EQ(waiter_buffer, std::string(8, 'x')); + EXPECT_EQ(opts.reader_local_cache->entry_count(), 0); + EXPECT_EQ(opts.reader_local_cache->memory_usage(), 0); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_block_maps_are_scoped_to_physical_readers) { + constexpr size_t reader_block_size = 256_kb; + const std::string content(600_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("shared_reader_local_cache", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "shared_reader_local_cache_blocks"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 1_mb); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + auto shared_cache = std::make_shared(4 * reader_block_size); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = shared_cache; + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader first_reader(local_reader, opts); + CachedRemoteFileReader second_reader(local_reader, opts); + + std::string buffer(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE( + first_reader.read_at(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + + FileCacheStatistics first_stats; + IOContext first_ctx; + first_ctx.file_cache_stats = &first_stats; + bool cache_hit = false; + ASSERT_TRUE(first_reader + .read_at_from_cache(128_kb, Slice(buffer.data(), buffer.size()), + &bytes_read, &cache_hit, &first_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(first_stats.num_reader_local_cache_fill, 1); + + FileCacheStatistics second_stats; + IOContext second_ctx; + second_ctx.file_cache_stats = &second_stats; + cache_hit = false; + ASSERT_TRUE(second_reader + .read_at_from_cache(160_kb, Slice(buffer.data(), buffer.size()), + &bytes_read, &cache_hit, &second_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(second_stats.num_reader_local_cache_hit, 0); + EXPECT_EQ(second_stats.num_reader_local_cache_fill, 1); + EXPECT_EQ(shared_cache->entry_count(), 2); + EXPECT_EQ(shared_cache->memory_usage(), 2 * reader_block_size); + EXPECT_EQ(shared_cache->tracked_memory(), 2 * reader_block_size); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_is_released_with_file_reader_lifetime) { + constexpr size_t reader_block_size = 256_kb; + const std::string content(300_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("sequential_reader_local_cache", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "sequential_reader_local_cache_blocks"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 1_mb); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + auto shared_cache = std::make_shared(reader_block_size); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = shared_cache; + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + + std::string buffer(8, '#'); + size_t bytes_read = 0; + bool cache_hit = false; + { + CachedRemoteFileReader first_reader(local_reader, opts); + ASSERT_TRUE(first_reader.read_at(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read) + .ok()); + ASSERT_TRUE(first_reader + .read_at_from_cache(128_kb, Slice(buffer.data(), buffer.size()), + &bytes_read, &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + ASSERT_TRUE(first_reader + .read_at_from_cache(128_kb, Slice(buffer.data(), buffer.size()), + &bytes_read, &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + ASSERT_EQ(shared_cache->entry_count(), 1); + } + EXPECT_EQ(shared_cache->entry_count(), 0); + EXPECT_EQ(shared_cache->memory_usage(), 0); + + CachedRemoteFileReader second_reader(local_reader, opts); + FileCacheStatistics stats; + IOContext io_ctx; + io_ctx.file_cache_stats = &stats; + ASSERT_TRUE(second_reader + .read_at_from_cache(160_kb, Slice(buffer.data(), buffer.size()), + &bytes_read, &cache_hit, &io_ctx) + .ok()); + EXPECT_TRUE(cache_hit); + EXPECT_EQ(stats.num_reader_local_cache_hit, 0); + EXPECT_EQ(stats.num_reader_local_cache_fill, 1); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_does_not_evict_another_reader) { + constexpr size_t reader_block_size = 256_kb; + const std::string content(600_kb, 'x'); + const fs::path first_file_path = + create_cached_remote_reader_test_file("bounded_shared_reader_local_cache_a", content); + const fs::path second_file_path = + create_cached_remote_reader_test_file("bounded_shared_reader_local_cache_b", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(first_file_path, ignore); + fs::remove(second_file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "bounded_shared_reader_local_cache_blocks"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 1_mb); + + FileReaderSPtr first_local_reader; + FileReaderSPtr second_local_reader; + ASSERT_TRUE(global_local_filesystem() + ->open_file(first_file_path.string(), &first_local_reader) + .ok()); + ASSERT_TRUE(global_local_filesystem() + ->open_file(second_file_path.string(), &second_local_reader) + .ok()); + auto shared_cache = std::make_shared(reader_block_size); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = shared_cache; + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader first_reader(first_local_reader, opts); + CachedRemoteFileReader second_reader(second_local_reader, opts); + + std::string warmup(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE( + first_reader.read_at(128_kb, Slice(warmup.data(), warmup.size()), &bytes_read).ok()); + + bool cache_hit = false; + ASSERT_TRUE(first_reader + .read_at_from_cache(128_kb, Slice(warmup.data(), warmup.size()), + &bytes_read, &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + ASSERT_TRUE(first_reader + .read_at_from_cache(128_kb, Slice(warmup.data(), warmup.size()), + &bytes_read, &cache_hit) + .ok()); + ASSERT_TRUE(cache_hit); + + ASSERT_TRUE( + second_reader.read_at(384_kb, Slice(warmup.data(), warmup.size()), &bytes_read).ok()); + + FileCacheStatistics second_stats; + IOContext second_ctx; + second_ctx.file_cache_stats = &second_stats; + cache_hit = false; + ASSERT_TRUE(second_reader + .read_at_from_cache(384_kb, Slice(warmup.data(), warmup.size()), + &bytes_read, &cache_hit, &second_ctx) + .ok()); + ASSERT_TRUE(cache_hit); + EXPECT_EQ(second_stats.num_reader_local_cache_evict, 0); + EXPECT_EQ(second_stats.num_reader_local_cache_fill, 0); + EXPECT_EQ(second_stats.num_reader_local_cache_miss, 1); + EXPECT_EQ(shared_cache->entry_count(), 1); + EXPECT_LE(shared_cache->memory_usage(), reader_block_size); + EXPECT_EQ(shared_cache->tracked_memory(), shared_cache->memory_usage()); +} + +TEST_F(BlockFileCacheTest, reader_local_cache_bypasses_promotion_at_query_memory_limit) { + constexpr size_t reader_block_size = 256_kb; + const std::string content(600_kb, 'x'); + const fs::path file_path = + create_cached_remote_reader_test_file("query_bounded_reader_local_cache", content); + Defer cleanup_file {[&]() { + std::error_code ignore; + fs::remove(file_path, ignore); + }}; + + const fs::path cache_path = caches_dir / "query_bounded_reader_local_cache_blocks"; + clear_cached_remote_reader_factory(); + Defer cleanup_cache {[&]() { + std::error_code ignore; + fs::remove_all(cache_path, ignore); + clear_cached_remote_reader_factory(); + }}; + create_cached_remote_reader_test_cache(cache_path, 1_mb); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(file_path.string(), &local_reader).ok()); + auto query_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::QUERY, + "reader-local-cache-test", 1); + auto shared_cache = + std::make_shared(reader_block_size, query_tracker); + io::FileReaderOptions opts; + opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = false; + opts.enable_reader_local_cache = true; + opts.reader_local_cache = shared_cache; + opts.cache_base_path = cache_path.string(); + opts.mtime = 1; + CachedRemoteFileReader reader(local_reader, opts); + + std::string buffer(8, '#'); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read).ok()); + + FileCacheStatistics stats; + IOContext io_ctx; + io_ctx.file_cache_stats = &stats; + bool cache_hit = false; + ASSERT_TRUE(reader.read_at_from_cache(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read, + &cache_hit, &io_ctx) + .ok()); + EXPECT_TRUE(cache_hit); + stats = {}; + ASSERT_TRUE(reader.read_at_from_cache(128_kb, Slice(buffer.data(), buffer.size()), &bytes_read, + &cache_hit, &io_ctx) + .ok()); + EXPECT_TRUE(cache_hit); + EXPECT_EQ(stats.num_reader_local_cache_miss, 1); + EXPECT_EQ(stats.num_reader_local_cache_fill, 0); + EXPECT_EQ(shared_cache->entry_count(), 0); + EXPECT_EQ(shared_cache->memory_usage(), 0); +} + } // namespace doris::io diff --git a/be/test/io/fs/buffered_reader_test.cpp b/be/test/io/fs/buffered_reader_test.cpp index 75ed5e351dc0ae..08b43d73d8a6b7 100644 --- a/be/test/io/fs/buffered_reader_test.cpp +++ b/be/test/io/fs/buffered_reader_test.cpp @@ -124,6 +124,47 @@ class MockOffsetFileReader : public io::FileReader { io::Path _path = "/tmp/mock"; }; +class CacheAwareMockFileReader : public MockOffsetFileReader, public io::ExactCacheReader { +public: + CacheAwareMockFileReader(size_t size, bool cache_hit) + : MockOffsetFileReader(size), _cache_hit(cache_hit) {} + + size_t remote_read_calls() const { return _remote_read_calls; } + size_t cache_read_calls() const { return _cache_read_calls; } + size_t last_cache_read_size() const { return _last_cache_read_size; } + bool merged_read_bypassed_reader_local_cache() const { + return _merged_read_bypassed_reader_local_cache; + } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext* io_ctx) override { + ++_remote_read_calls; + _merged_read_bypassed_reader_local_cache = + io_ctx != nullptr && io_ctx->bypass_reader_local_cache; + return MockOffsetFileReader::read_at_impl(offset, result, bytes_read, io_ctx); + } + + Status read_at_from_cache(size_t offset, Slice result, size_t* bytes_read, bool* cache_hit, + const io::IOContext* io_ctx) override { + ++_cache_read_calls; + _last_cache_read_size = result.size; + *cache_hit = _cache_hit; + if (!_cache_hit) { + *bytes_read = 0; + return Status::OK(); + } + return MockOffsetFileReader::read_at_impl(offset, result, bytes_read, io_ctx); + } + +private: + bool _cache_hit; + size_t _remote_read_calls = 0; + size_t _cache_read_calls = 0; + size_t _last_cache_read_size = 0; + bool _merged_read_bypassed_reader_local_cache = false; +}; + class BlockingFileReader : public io::FileReader { public: BlockingFileReader(size_t size, CountDownLatch* read_started, CountDownLatch* continue_read, @@ -430,6 +471,95 @@ TEST_F(BufferedReaderTest, test_read_amplify) { EXPECT_EQ(merge_reader.statistics().merged_bytes, 1024 * kb + 12 * kb); } +TEST_F(BufferedReaderTest, cache_hit_bypasses_merged_read) { + constexpr size_t KB = 1024; + auto inner = std::make_shared(64 * KB, true); + std::vector ranges {{0, KB}, {2 * KB, 3 * KB}, {4 * KB, 5 * KB}}; + io::MergeRangeFileReader reader(nullptr, inner, ranges, 8 * KB); + + std::vector data(256); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(data.data(), data.size()), &bytes_read).ok()); + + EXPECT_EQ(bytes_read, data.size()); + EXPECT_EQ(inner->cache_read_calls(), 1); + EXPECT_EQ(inner->last_cache_read_size(), data.size()); + EXPECT_EQ(inner->remote_read_calls(), 0); + EXPECT_EQ(reader.statistics().cache_hit_bytes, data.size()); + EXPECT_EQ(reader.statistics().merged_bytes, 0); +} + +TEST_F(BufferedReaderTest, cache_miss_keeps_remote_range_merging) { + constexpr size_t KB = 1024; + auto inner = std::make_shared(64 * KB, false); + std::vector ranges {{0, KB}, {2 * KB, 3 * KB}, {4 * KB, 5 * KB}}; + io::MergeRangeFileReader reader(nullptr, inner, ranges, 8 * KB); + + std::vector data(256); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(data.data(), data.size()), &bytes_read).ok()); + + EXPECT_EQ(inner->cache_read_calls(), 1); + EXPECT_EQ(inner->remote_read_calls(), 1); + EXPECT_GT(reader.statistics().merged_bytes, data.size()); + EXPECT_GT(reader.statistics().merged_gap_bytes, 0); + EXPECT_EQ(reader.statistics().merged_bytes, + reader.statistics().merged_useful_bytes + reader.statistics().merged_gap_bytes); + EXPECT_TRUE(inner->merged_read_bypassed_reader_local_cache()); +} + +TEST_F(BufferedReaderTest, ranges_are_exposed_incrementally_by_predicate_stage) { + constexpr size_t KB = 1024; + auto inner = std::make_shared(64 * KB, false); + io::MergeRangeFileReader reader(nullptr, inner, {}, 8 * KB); + ASSERT_TRUE(reader.add_random_access_ranges({{0, KB}}, 0).ok()); + + std::vector data(256); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(data.data(), data.size()), &bytes_read).ok()); + EXPECT_EQ(reader.statistics().merged_useful_bytes, KB); + EXPECT_EQ(reader.statistics().future_predicate_prefetch_bytes, 0); + + ASSERT_TRUE(reader.add_random_access_ranges({{2 * KB, 3 * KB}}, 1).ok()); + ASSERT_TRUE(reader.read_at(2 * KB, Slice(data.data(), data.size()), &bytes_read).ok()); + EXPECT_EQ(reader.statistics().future_predicate_prefetch_bytes, 0); +} + +TEST_F(BufferedReaderTest, overlapping_incremental_ranges_are_coalesced) { + constexpr size_t KB = 1024; + auto inner = std::make_shared(64 * KB, false); + io::MergeRangeFileReader reader(nullptr, inner, {}, 8 * KB); + ASSERT_TRUE(reader.add_random_access_ranges({{0, 2 * KB}}, 0).ok()); + + std::vector first_page(256); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(first_page.data(), first_page.size()), &bytes_read).ok()); + ASSERT_EQ(bytes_read, first_page.size()); + + ASSERT_TRUE(reader.add_random_access_ranges({{KB, 3 * KB}}, 1).ok()); + EXPECT_EQ(reader.buffer_remaining(), io::MergeRangeFileReader::TOTAL_BUFFER_SIZE); + std::vector cross_original_boundary(KB); + ASSERT_TRUE( + reader.read_at(1536, + Slice(cross_original_boundary.data(), cross_original_boundary.size()), + &bytes_read) + .ok()); + EXPECT_EQ(bytes_read, cross_original_boundary.size()); +} + +TEST_F(BufferedReaderTest, eager_later_stage_range_is_counted_as_future_prefetch) { + constexpr size_t KB = 1024; + auto inner = std::make_shared(64 * KB, false); + io::MergeRangeFileReader reader(nullptr, inner, {}, 8 * KB); + ASSERT_TRUE(reader.add_random_access_ranges({{0, KB}}, 0).ok()); + ASSERT_TRUE(reader.add_random_access_ranges({{2 * KB, 3 * KB}}, 1).ok()); + + std::vector data(256); + size_t bytes_read = 0; + ASSERT_TRUE(reader.read_at(0, Slice(data.data(), data.size()), &bytes_read).ok()); + EXPECT_EQ(reader.statistics().future_predicate_prefetch_bytes, KB); +} + TEST_F(BufferedReaderTest, test_merged_io) { io::FileReaderSPtr offset_reader = std::make_shared(128 * 1024 * 1024); // 128MB