From 84232d806070f4b177e35148255c501a5f6e740b Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Fri, 21 Feb 2025 00:22:22 +0800 Subject: [PATCH] Add additional info about memory reservation to the doc of MemoryPool (#14789) --- datafusion/execution/src/memory_pool/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion/execution/src/memory_pool/mod.rs b/datafusion/execution/src/memory_pool/mod.rs index 5e49debac7e7..71d40aeab53c 100644 --- a/datafusion/execution/src/memory_pool/mod.rs +++ b/datafusion/execution/src/memory_pool/mod.rs @@ -55,7 +55,10 @@ pub use pool::*; /// "large" amounts of memory (proportional to number of input rows), such as /// `GroupByHashExec`. It does NOT track and limit memory used internally by /// other operators such as `DataSourceExec` or the `RecordBatch`es that flow -/// between operators. +/// between operators. Furthermore, operators should not reserve memory for the +/// batches they produce. Instead, if a parent operator needs to hold batches +/// from its children in memory for an extended period, it is the parent +/// operator's responsibility to reserve the necessary memory for those batches. /// /// In order to avoid allocating memory until the OS or the container system /// kills the process, DataFusion `ExecutionPlan`s (operators) that consume