diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 628e9a4d1c7..10f41d34f1a 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -128,6 +128,7 @@ add_arrow_benchmark(decimal_benchmark) add_arrow_benchmark(hashing_benchmark) add_arrow_benchmark(int_util_benchmark) add_arrow_benchmark(machine_benchmark) +add_arrow_benchmark(nested_bitmap_traversal_benchmark) add_arrow_benchmark(queue_benchmark) add_arrow_benchmark(range_benchmark) add_arrow_benchmark(small_vector_benchmark) diff --git a/cpp/src/arrow/util/bit_block_counter_benchmark.cc b/cpp/src/arrow/util/bit_block_counter_benchmark.cc index c08fcfdb604..7be82b24b86 100644 --- a/cpp/src/arrow/util/bit_block_counter_benchmark.cc +++ b/cpp/src/arrow/util/bit_block_counter_benchmark.cc @@ -23,22 +23,24 @@ #include "arrow/array/array_base.h" #include "arrow/array/array_primitive.h" +#include "arrow/testing/gtest_util.h" #include "arrow/testing/random.h" #include "arrow/util/bit_block_counter.h" +#include "arrow/util/bit_run_reader.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_reader.h" namespace arrow { namespace internal { -struct UnaryBitBlockBenchmark { +struct UnaryBitmapTraversalBenchmark { benchmark::State& state; int64_t offset; int64_t bitmap_length; std::shared_ptr arr; int64_t expected; - explicit UnaryBitBlockBenchmark(benchmark::State& state, int64_t offset = 0) + explicit UnaryBitmapTraversalBenchmark(benchmark::State& state, int64_t offset = 0) : state(state), offset(offset), bitmap_length(1 << 20) { random::RandomArrayGenerator rng(/*seed=*/0); // State parameter is the average number of total values for each null @@ -111,9 +113,57 @@ struct UnaryBitBlockBenchmark { } state.SetItemsProcessed(state.iterations() * bitmap_length); } + + void BenchVisitBitRuns() { + const auto& int8_arr = static_cast(*arr); + const uint8_t* bitmap = arr->null_bitmap_data(); + for (auto _ : state) { + int64_t result = 0; + ABORT_NOT_OK(VisitBitRuns(bitmap, this->offset, bitmap_length - this->offset, + [&](int64_t position, int64_t length, bool set) { + if (set) { + int64_t run_sum = 0; + const int64_t end = position + length; + for (int64_t i = position; i < end; ++i) { + run_sum += int8_arr.Value(this->offset + i); + } + result += run_sum; + } + return Status::OK(); + })); + // Sanity check + if (result != expected) { + std::abort(); + } + } + state.SetItemsProcessed(state.iterations() * bitmap_length); + } + + void BenchVisitSetBitRuns() { + const auto& int8_arr = static_cast(*arr); + const uint8_t* bitmap = arr->null_bitmap_data(); + for (auto _ : state) { + int64_t result = 0; + ABORT_NOT_OK(VisitSetBitRuns(bitmap, this->offset, bitmap_length - this->offset, + [&](int64_t position, int64_t length) { + int64_t run_sum = 0; + const int64_t end = position + length; + for (int64_t i = position; i < end; ++i) { + run_sum += int8_arr.Value(this->offset + i); + } + result += run_sum; + return Status::OK(); + })); + // Sanity check + if (result != expected) { + std::abort(); + } + } + state.SetItemsProcessed(state.iterations() * bitmap_length); + } }; -struct BinaryBitBlockBenchmark { +struct BinaryBitmapTraversalBenchmark { benchmark::State& state; int64_t offset; int64_t bitmap_length; @@ -123,7 +173,7 @@ struct BinaryBitBlockBenchmark { const Int8Array* left_int8; const Int8Array* right_int8; - explicit BinaryBitBlockBenchmark(benchmark::State& state, int64_t offset = 0) + explicit BinaryBitmapTraversalBenchmark(benchmark::State& state, int64_t offset = 0) : state(state), offset(offset), bitmap_length(1 << 20) { random::RandomArrayGenerator rng(/*seed=*/0); @@ -202,52 +252,134 @@ struct BinaryBitBlockBenchmark { } state.SetItemsProcessed(state.iterations() * bitmap_length); } + + void BenchVisitTwoBitRuns() { + const uint8_t* left_bitmap = left->null_bitmap_data(); + const uint8_t* right_bitmap = right->null_bitmap_data(); + for (auto _ : state) { + int64_t result = 0; + VisitTwoBitRunsVoid(left_bitmap, this->offset, right_bitmap, this->offset, + bitmap_length - this->offset, + [&](int64_t position, int64_t length, bool set) { + if (set) { + int64_t run_sum = 0; + const int64_t end = position + length; + for (int64_t i = position; i < end; ++i) { + run_sum += left_int8->Value(this->offset + i) + + right_int8->Value(this->offset + i); + } + result += run_sum; + } + }); + // Sanity check + if (result != expected) { + std::abort(); + } + } + state.SetItemsProcessed(state.iterations() * bitmap_length); + } + + void BenchVisitTwoSetBitRuns() { + const uint8_t* left_bitmap = left->null_bitmap_data(); + const uint8_t* right_bitmap = right->null_bitmap_data(); + for (auto _ : state) { + int64_t result = 0; + VisitTwoSetBitRunsVoid(left_bitmap, this->offset, right_bitmap, this->offset, + bitmap_length - this->offset, + [&](int64_t position, int64_t length) { + int64_t run_sum = 0; + const int64_t end = position + length; + for (int64_t i = position; i < end; ++i) { + run_sum += left_int8->Value(this->offset + i) + + right_int8->Value(this->offset + i); + } + result += run_sum; + }); + // Sanity check + if (result != expected) { + std::abort(); + } + } + state.SetItemsProcessed(state.iterations() * bitmap_length); + } }; static void BitBlockCounterSum(benchmark::State& state) { - UnaryBitBlockBenchmark(state, /*offset=*/0) + UnaryBitmapTraversalBenchmark(state, /*offset=*/0) .BenchBitBlockCounter([](BitBlockCounter* counter) { return counter->NextWord(); }); } static void BitBlockCounterSumWithOffset(benchmark::State& state) { - UnaryBitBlockBenchmark(state, /*offset=*/4) + UnaryBitmapTraversalBenchmark(state, /*offset=*/4) .BenchBitBlockCounter([](BitBlockCounter* counter) { return counter->NextWord(); }); } static void BitBlockCounterFourWordsSum(benchmark::State& state) { - UnaryBitBlockBenchmark(state, /*offset=*/0) + UnaryBitmapTraversalBenchmark(state, /*offset=*/0) .BenchBitBlockCounter( [](BitBlockCounter* counter) { return counter->NextFourWords(); }); } static void BitBlockCounterFourWordsSumWithOffset(benchmark::State& state) { - UnaryBitBlockBenchmark(state, /*offset=*/4) + UnaryBitmapTraversalBenchmark(state, /*offset=*/4) .BenchBitBlockCounter( [](BitBlockCounter* counter) { return counter->NextFourWords(); }); } static void BitmapReaderSum(benchmark::State& state) { - UnaryBitBlockBenchmark(state, /*offset=*/0).BenchBitmapReader(); + UnaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchBitmapReader(); } static void BitmapReaderSumWithOffset(benchmark::State& state) { - UnaryBitBlockBenchmark(state, /*offset=*/4).BenchBitmapReader(); + UnaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchBitmapReader(); +} + +static void VisitBitRunsSum(benchmark::State& state) { + UnaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchVisitBitRuns(); +} + +static void VisitBitRunsSumWithOffset(benchmark::State& state) { + UnaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchVisitBitRuns(); +} + +static void VisitSetBitRunsSum(benchmark::State& state) { + UnaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchVisitSetBitRuns(); +} + +static void VisitSetBitRunsSumWithOffset(benchmark::State& state) { + UnaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchVisitSetBitRuns(); } static void BinaryBitBlockCounterSum(benchmark::State& state) { - BinaryBitBlockBenchmark(state, /*offset=*/0).BenchBitBlockCounter(); + BinaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchBitBlockCounter(); } static void BinaryBitBlockCounterSumWithOffset(benchmark::State& state) { - BinaryBitBlockBenchmark(state, /*offset=*/4).BenchBitBlockCounter(); + BinaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchBitBlockCounter(); } static void BinaryBitmapReaderSum(benchmark::State& state) { - BinaryBitBlockBenchmark(state, /*offset=*/0).BenchBitmapReader(); + BinaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchBitmapReader(); } static void BinaryBitmapReaderSumWithOffset(benchmark::State& state) { - BinaryBitBlockBenchmark(state, /*offset=*/4).BenchBitmapReader(); + BinaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchBitmapReader(); +} + +static void BinaryVisitTwoBitRunsSum(benchmark::State& state) { + BinaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchVisitTwoBitRuns(); +} + +static void BinaryVisitTwoBitRunsSumWithOffset(benchmark::State& state) { + BinaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchVisitTwoBitRuns(); +} + +static void BinaryVisitTwoSetBitRunsSum(benchmark::State& state) { + BinaryBitmapTraversalBenchmark(state, /*offset=*/0).BenchVisitTwoSetBitRuns(); +} + +static void BinaryVisitTwoSetBitRunsSumWithOffset(benchmark::State& state) { + BinaryBitmapTraversalBenchmark(state, /*offset=*/4).BenchVisitTwoSetBitRuns(); } // Range value: average number of total values per null @@ -257,10 +389,18 @@ BENCHMARK(BitBlockCounterFourWordsSum)->Range(2, 1 << 16); BENCHMARK(BitBlockCounterFourWordsSumWithOffset)->Range(2, 1 << 16); BENCHMARK(BitmapReaderSum)->Range(2, 1 << 16); BENCHMARK(BitmapReaderSumWithOffset)->Range(2, 1 << 16); +BENCHMARK(VisitBitRunsSum)->Range(2, 1 << 16); +BENCHMARK(VisitBitRunsSumWithOffset)->Range(2, 1 << 16); +BENCHMARK(VisitSetBitRunsSum)->Range(2, 1 << 16); +BENCHMARK(VisitSetBitRunsSumWithOffset)->Range(2, 1 << 16); BENCHMARK(BinaryBitBlockCounterSum)->Range(2, 1 << 16); BENCHMARK(BinaryBitBlockCounterSumWithOffset)->Range(2, 1 << 16); BENCHMARK(BinaryBitmapReaderSum)->Range(2, 1 << 16); BENCHMARK(BinaryBitmapReaderSumWithOffset)->Range(2, 1 << 16); +BENCHMARK(BinaryVisitTwoBitRunsSum)->Range(2, 1 << 16); +BENCHMARK(BinaryVisitTwoBitRunsSumWithOffset)->Range(2, 1 << 16); +BENCHMARK(BinaryVisitTwoSetBitRunsSum)->Range(2, 1 << 16); +BENCHMARK(BinaryVisitTwoSetBitRunsSumWithOffset)->Range(2, 1 << 16); } // namespace internal } // namespace arrow diff --git a/cpp/src/arrow/util/bit_run_reader.h b/cpp/src/arrow/util/bit_run_reader.h index 1a9638880c5..989794b649c 100644 --- a/cpp/src/arrow/util/bit_run_reader.h +++ b/cpp/src/arrow/util/bit_run_reader.h @@ -17,15 +17,21 @@ #pragma once +#include #include #include #include #include #include +#include +#include "arrow/buffer.h" +#include "arrow/memory_pool.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/bitmap_reader.h" #include "arrow/util/endian.h" +#include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h" @@ -536,5 +542,79 @@ inline void VisitSetBitRunsVoid(const std::shared_ptr& bitmap, int64_t o std::forward(visit)); } +template +inline Status VisitTwoSetBitRuns(const uint8_t* left_bitmap, int64_t left_offset, + const uint8_t* right_bitmap, int64_t right_offset, + int64_t length, Visit&& visit, + MemoryPool* pool = default_memory_pool()) { + if (length == 0) { + return Status::OK(); + } + if (left_bitmap == NULLPTR) { + return VisitSetBitRuns(right_bitmap, right_offset, length, + std::forward(visit)); + } + if (right_bitmap == NULLPTR) { + return VisitSetBitRuns(left_bitmap, left_offset, length, std::forward(visit)); + } + + ARROW_ASSIGN_OR_RAISE(auto bitmap_and, + BitmapAnd(pool, left_bitmap, left_offset, right_bitmap, + right_offset, length, /*out_offset=*/0)); + return VisitSetBitRuns(bitmap_and->data(), /*offset=*/0, length, + std::forward(visit)); +} + +template +inline Status VisitTwoBitRuns(const uint8_t* left_bitmap, int64_t left_offset, + const uint8_t* right_bitmap, int64_t right_offset, + int64_t length, Visit&& visit, + MemoryPool* pool = default_memory_pool()) { + int64_t output_position = 0; + ARROW_RETURN_NOT_OK(VisitTwoSetBitRuns( + left_bitmap, left_offset, right_bitmap, right_offset, length, + [&](int64_t position, int64_t run_length) { + if (output_position < position) { + ARROW_RETURN_NOT_OK(visit(output_position, position - output_position, false)); + } + ARROW_RETURN_NOT_OK(visit(position, run_length, true)); + output_position = position + run_length; + return Status::OK(); + }, + pool)); + if (output_position < length) { + return visit(output_position, length - output_position, false); + } + return Status::OK(); +} + +template +inline void VisitTwoBitRunsVoid(const uint8_t* left_bitmap, int64_t left_offset, + const uint8_t* right_bitmap, int64_t right_offset, + int64_t length, Visit&& visit, + MemoryPool* pool = default_memory_pool()) { + ARROW_IGNORE_EXPR(VisitTwoBitRuns( + left_bitmap, left_offset, right_bitmap, right_offset, length, + [&](int64_t position, int64_t length, bool set) { + visit(position, length, set); + return Status::OK(); + }, + pool)); +} + +template +inline void VisitTwoSetBitRunsVoid(const uint8_t* left_bitmap, int64_t left_offset, + const uint8_t* right_bitmap, int64_t right_offset, + int64_t length, Visit&& visit, + MemoryPool* pool = default_memory_pool()) { + ARROW_IGNORE_EXPR(VisitTwoSetBitRuns( + left_bitmap, left_offset, right_bitmap, right_offset, length, + [&](int64_t position, int64_t length) { + visit(position, length); + return Status::OK(); + }, + pool)); +} + } // namespace internal } // namespace arrow diff --git a/cpp/src/arrow/util/bitmap_test.cc b/cpp/src/arrow/util/bitmap_test.cc index 0a71f4e6038..4ede199e626 100644 --- a/cpp/src/arrow/util/bitmap_test.cc +++ b/cpp/src/arrow/util/bitmap_test.cc @@ -332,6 +332,49 @@ class TestSetBitRunReader : public ::testing::Test { AssertBitRuns(buffer.data(), start_offset, length, expected); } + void AssertVisitTwoBitRuns(const uint8_t* left_data, int64_t left_offset, + const uint8_t* right_data, int64_t right_offset, + int64_t length, + const std::vector& expected_positions, + const std::vector& expected_runs, + const std::vector& expected_set_runs) { + std::vector positions; + std::vector runs; + ASSERT_OK(VisitTwoBitRuns(left_data, left_offset, right_data, right_offset, length, + [&](int64_t position, int64_t run_length, bool set) { + positions.push_back(position); + runs.push_back({run_length, set}); + return Status::OK(); + })); + ASSERT_THAT(positions, ElementsAreArray(expected_positions)); + ASSERT_THAT(runs, ElementsAreArray(expected_runs)); + + positions.clear(); + runs.clear(); + VisitTwoBitRunsVoid(left_data, left_offset, right_data, right_offset, length, + [&](int64_t position, int64_t run_length, bool set) { + positions.push_back(position); + runs.push_back({run_length, set}); + }); + ASSERT_THAT(positions, ElementsAreArray(expected_positions)); + ASSERT_THAT(runs, ElementsAreArray(expected_runs)); + + std::vector set_runs; + ASSERT_OK(VisitTwoSetBitRuns(left_data, left_offset, right_data, right_offset, length, + [&](int64_t position, int64_t run_length) { + set_runs.push_back({position, run_length}); + return Status::OK(); + })); + ASSERT_THAT(set_runs, ElementsAreArray(expected_set_runs)); + + set_runs.clear(); + VisitTwoSetBitRunsVoid(left_data, left_offset, right_data, right_offset, length, + [&](int64_t position, int64_t run_length) { + set_runs.push_back({position, run_length}); + }); + ASSERT_THAT(set_runs, ElementsAreArray(expected_set_runs)); + } + struct Range { int64_t offset; int64_t length; @@ -484,6 +527,86 @@ TEST_F(TestSetBitRunReader, Random) { } } +TEST_F(TestSetBitRunReader, VisitTwoBitRuns) { + auto left = BitmapFromString("11110000 11111100"); + auto right = BitmapFromString("11001100 00111111"); + const int64_t left_offset = 1; + const int64_t right_offset = 2; + const int64_t length = 12; + + // With left_offset = 1 and right_offset = 2: + // position: 0 1 2 3 4 5 6 7 8 9 10 11 + // left[+1]: 1 1 1 0 0 0 0 1 1 1 1 1 + // right[+2]: 0 0 1 1 0 0 0 0 1 1 1 1 + // AND: 0 0 1 0 0 0 0 0 1 1 1 1 + const std::vector expected_positions = {0, 2, 3, 8}; + const std::vector expected_runs = {{.length = 2, .set = false}, + {.length = 1, .set = true}, + {.length = 5, .set = false}, + {.length = 4, .set = true}}; + const std::vector expected_set_runs = {{.position = 2, .length = 1}, + {.position = 8, .length = 4}}; + + AssertVisitTwoBitRuns(left->data(), left_offset, right->data(), right_offset, length, + expected_positions, expected_runs, expected_set_runs); +} + +TEST_F(TestSetBitRunReader, VisitTwoBitRunsWithNullBitmap) { + auto bitmap = BitmapFromString("01101101"); + const int64_t left_offset = 0; + const int64_t right_offset = 1; + const int64_t length = 6; + + // With a null left bitmap and right_offset = 1: + // left all set: 1 1 1 1 1 1 + // right[1..6]: 1 1 0 1 1 0 + // AND: 1 1 0 1 1 0 + const std::vector expected_positions = {0, 2, 3, 5}; + const std::vector expected_runs = {{.length = 2, .set = true}, + {.length = 1, .set = false}, + {.length = 2, .set = true}, + {.length = 1, .set = false}}; + const std::vector expected_set_runs = {{.position = 0, .length = 2}, + {.position = 3, .length = 2}}; + + AssertVisitTwoBitRuns(nullptr, left_offset, bitmap->data(), right_offset, length, + expected_positions, expected_runs, expected_set_runs); +} + +TEST_F(TestSetBitRunReader, VisitTwoBitRunsFalseTail) { + auto left = BitmapFromString("11111111"); + auto right = BitmapFromString("11110000"); + + std::vector positions; + std::vector runs; + ASSERT_OK(VisitTwoBitRuns(left->data(), /*left_offset=*/0, right->data(), + /*right_offset=*/0, /*length=*/8, + [&](int64_t position, int64_t length, bool set) { + positions.push_back(position); + runs.push_back({length, set}); + return Status::OK(); + })); + ASSERT_THAT(positions, ElementsAreArray({0, 4})); + ASSERT_THAT(runs, ElementsAreArray({BitRun{4, true}, BitRun{4, false}})); +} + +TEST_F(TestSetBitRunReader, VisitTwoBitRunsNoOverlap) { + auto left = BitmapFromString("11110000"); + auto right = BitmapFromString("00001111"); + + std::vector positions; + std::vector runs; + ASSERT_OK(VisitTwoBitRuns(left->data(), /*left_offset=*/0, right->data(), + /*right_offset=*/0, /*length=*/8, + [&](int64_t position, int64_t length, bool set) { + positions.push_back(position); + runs.push_back({length, set}); + return Status::OK(); + })); + ASSERT_THAT(positions, ElementsAreArray({0})); + ASSERT_THAT(runs, ElementsAreArray({BitRun{8, false}})); +} + // Tests for BitRunReader. TEST(BitRunReader, ZeroLength) { diff --git a/cpp/src/arrow/util/nested_bitmap_traversal_benchmark.cc b/cpp/src/arrow/util/nested_bitmap_traversal_benchmark.cc new file mode 100644 index 00000000000..65393f79ff4 --- /dev/null +++ b/cpp/src/arrow/util/nested_bitmap_traversal_benchmark.cc @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "benchmark/benchmark.h" + +#include +#include +#include + +#include "arrow/array/array_nested.h" +#include "arrow/array/array_primitive.h" +#include "arrow/buffer.h" +#include "arrow/memory_pool.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" +#include "arrow/util/bit_block_counter.h" +#include "arrow/util/bit_run_reader.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" + +namespace arrow::internal { + +struct NestedBitmapTraversalBenchmark { + benchmark::State& state; + MemoryPool* memory_pool; + int64_t parent_length; + std::shared_ptr list_array; + std::shared_ptr outer_bitmap; + int64_t expected; + + explicit NestedBitmapTraversalBenchmark(benchmark::State& state) + : state(state), memory_pool(default_memory_pool()), parent_length(1 << 20) { + constexpr double kValuesNullProbability = 0.1; + random::RandomArrayGenerator rng(/*seed=*/0); + const auto parent_null_probability = 1. / static_cast(state.range(0)); + const auto avg_list_length = state.range(1); + + auto child_values = + rng.Int8(parent_length * avg_list_length, 0, 100, kValuesNullProbability); + list_array = std::static_pointer_cast( + rng.List(*child_values, parent_length, parent_null_probability, + /*force_empty_nulls=*/false)); + outer_bitmap = rng.NullBitmap(parent_length, parent_null_probability); + const auto& values = static_cast(*list_array->values()); + + expected = 0; + for (int64_t i = 0; i < parent_length; ++i) { + if (bit_util::GetBit(outer_bitmap->data(), i) && list_array->IsValid(i)) { + const int64_t child_start = list_array->value_offset(i); + const int64_t child_end = list_array->value_offset(i + 1); + expected += CountValues(values, child_start, child_end - child_start); + } + } + } + + static int64_t CountValues(const Int8Array& values, int64_t offset, int64_t length) { + const uint8_t* values_bitmap = values.null_bitmap_data(); + return CountSetBits(values_bitmap, values.offset() + offset, length); + } + + void CheckResult(int64_t result) const { + if (result != expected) { + std::abort(); + } + } + + void BenchBitmapAnd() { + const uint8_t* outer_bitmap_data = outer_bitmap->data(); + const uint8_t* list_bitmap = list_array->null_bitmap_data(); + const auto& values = static_cast(*list_array->values()); + for (auto _ : state) { + std::shared_ptr visible_bitmap; + ABORT_NOT_OK(BitmapAnd(memory_pool, outer_bitmap_data, /*left_offset=*/0, + list_bitmap, /*right_offset=*/0, parent_length, + /*out_offset=*/0) + .Value(&visible_bitmap)); + + int64_t result = 0; + VisitSetBitRunsVoid( + visible_bitmap->data(), /*offset=*/0, parent_length, + [&](int64_t position, int64_t length) { + const int64_t child_start = list_array->value_offset(position); + const int64_t child_end = list_array->value_offset(position + length); + result += CountValues(values, child_start, child_end - child_start); + }); + CheckResult(result); + } + state.SetItemsProcessed(state.iterations() * parent_length); + } + + void BenchVisitTwoBitBlocks() { + const uint8_t* outer_bitmap_data = outer_bitmap->data(); + const uint8_t* list_bitmap = list_array->null_bitmap_data(); + const auto& values = static_cast(*list_array->values()); + for (auto _ : state) { + int64_t result = 0; + VisitTwoBitBlocksVoid( + outer_bitmap_data, /*left_offset=*/0, list_bitmap, /*right_offset=*/0, + parent_length, + [&](int64_t position) { + const int64_t child_start = list_array->value_offset(position); + const int64_t child_end = list_array->value_offset(position + 1); + result += CountValues(values, child_start, child_end - child_start); + }, + [] {}); + CheckResult(result); + } + state.SetItemsProcessed(state.iterations() * parent_length); + } + + void BenchVisitTwoBitRuns() { + const uint8_t* outer_bitmap_data = outer_bitmap->data(); + const uint8_t* list_bitmap = list_array->null_bitmap_data(); + const auto& values = static_cast(*list_array->values()); + for (auto _ : state) { + int64_t result = 0; + VisitTwoBitRunsVoid( + outer_bitmap_data, /*left_offset=*/0, list_bitmap, + /*right_offset=*/0, parent_length, + [&](int64_t position, int64_t length, bool set) { + if (set) { + const int64_t child_start = list_array->value_offset(position); + const int64_t child_end = list_array->value_offset(position + length); + result += CountValues(values, child_start, child_end - child_start); + } + }, + memory_pool); + CheckResult(result); + } + state.SetItemsProcessed(state.iterations() * parent_length); + } + + void BenchVisitTwoSetBitRuns() { + const uint8_t* outer_bitmap_data = outer_bitmap->data(); + const uint8_t* list_bitmap = list_array->null_bitmap_data(); + const auto& values = static_cast(*list_array->values()); + for (auto _ : state) { + int64_t result = 0; + VisitTwoSetBitRunsVoid( + outer_bitmap_data, /*left_offset=*/0, list_bitmap, + /*right_offset=*/0, parent_length, + [&](int64_t position, int64_t length) { + const int64_t child_start = list_array->value_offset(position); + const int64_t child_end = list_array->value_offset(position + length); + result += CountValues(values, child_start, child_end - child_start); + }, + memory_pool); + CheckResult(result); + } + state.SetItemsProcessed(state.iterations() * parent_length); + } +}; + +static void NestedBitmapAndCount(benchmark::State& state) { + NestedBitmapTraversalBenchmark(state).BenchBitmapAnd(); +} + +static void NestedVisitTwoBitBlocksCount(benchmark::State& state) { + NestedBitmapTraversalBenchmark(state).BenchVisitTwoBitBlocks(); +} + +static void NestedVisitTwoBitRunsCount(benchmark::State& state) { + NestedBitmapTraversalBenchmark(state).BenchVisitTwoBitRuns(); +} + +static void NestedVisitTwoSetBitRunsCount(benchmark::State& state) { + NestedBitmapTraversalBenchmark(state).BenchVisitTwoSetBitRuns(); +} + +static void SetArgs(benchmark::internal::Benchmark* benchmark) { + benchmark->ArgsProduct({benchmark::CreateRange(2, 1 << 16, 8), {1, 4, 16}}); +} + +BENCHMARK(NestedBitmapAndCount)->Apply(SetArgs); +BENCHMARK(NestedVisitTwoBitBlocksCount)->Apply(SetArgs); +BENCHMARK(NestedVisitTwoBitRunsCount)->Apply(SetArgs); +BENCHMARK(NestedVisitTwoSetBitRunsCount)->Apply(SetArgs); + +} // namespace arrow::internal