Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 1 addition & 30 deletions datafusion/physical-plan/src/spill/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ pub use spill_manager::SpillManager;

use std::fs::File;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
Expand Down Expand Up @@ -245,35 +245,6 @@ impl RecordBatchStream for SpillReaderStream {
}
}

/// Spill the `RecordBatch` to disk as smaller batches
/// split by `batch_size_rows`
#[deprecated(
since = "46.0.0",
note = "This method is deprecated. Use `SpillManager::spill_record_batch_by_size` instead."
)]
#[expect(clippy::needless_pass_by_value)]
pub fn spill_record_batch_by_size(
batch: &RecordBatch,
path: PathBuf,
schema: SchemaRef,
batch_size_rows: usize,
) -> Result<()> {
let mut offset = 0;
let total_rows = batch.num_rows();
let mut writer =
IPCStreamWriter::new(&path, schema.as_ref(), SpillCompression::Uncompressed)?;

while offset < total_rows {
let length = std::cmp::min(total_rows - offset, batch_size_rows);
let batch = batch.slice(offset, length);
offset += batch.num_rows();
writer.write(&batch)?;
}
writer.finish()?;

Ok(())
}

/// Write in Arrow IPC Stream format to a file.
///
/// Stream format is used for spill because it supports dictionary replacement, and the random
Expand Down
8 changes: 8 additions & 0 deletions docs/source/library-user-guide/upgrading/55.0.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@ to the main branch and are awaiting release in this version.
`datafusion_common::config::Dialect::AVAILABLE` has been removed. Use
`Dialect::available()` instead.

### `spill_record_batch_by_size` removed

`datafusion_physical_plan::spill::spill_record_batch_by_size` has been removed.
This function was deprecated in DataFusion `46.0.0`.

Use `datafusion_physical_plan::spill::SpillManager::spill_record_batch_by_size`
instead.

### Decimal scalar formatting uses human-readable values

Decimal scalar literals in `EXPLAIN` output, expression display strings, and
Expand Down
Loading