From 7c30fb264e6a6a959010c5d40a8d9e6c772f295f Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 7 Feb 2024 23:45:03 -0800 Subject: [PATCH] Use prep_null_mask_filter to handle nulls in selection mask --- datafusion/physical-plan/src/joins/sort_merge_join.rs | 9 ++++++++- datafusion/sqllogictest/test_files/sort_merge_join.slt | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/joins/sort_merge_join.rs b/datafusion/physical-plan/src/joins/sort_merge_join.rs index 107fd7dde0f6e..9954ca812cf49 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join.rs @@ -1209,7 +1209,14 @@ impl SMJStream { ) { // The reverse of the selection mask. For the rows not pass join filter above, // we need to join them (left or right) with null rows for outer joins. - let not_mask = compute::not(mask)?; + let not_mask = if mask.null_count() > 0 { + // If the mask contains nulls, we need to use `prep_null_mask_filter` to + // handle the nulls in the mask as false. + compute::not(&compute::prep_null_mask_filter(mask))? + } else { + compute::not(mask)? + }; + let null_joined_batch = compute::filter_record_batch(&output_batch, ¬_mask)?; diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.slt index 426b9a3a52919..6b3eb6c3ff947 100644 --- a/datafusion/sqllogictest/test_files/sort_merge_join.slt +++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt @@ -256,6 +256,7 @@ SELECT * FROM t1 FULL JOIN t2 ON t1_id = t2_id AND t2_int <= t1_int 44 d 4 44 x 3 NULL NULL NULL 11 z 3 NULL NULL NULL 55 w 3 +NULL NULL NULL NULL NULL NULL statement ok DROP TABLE t1;