From 1cb4d093fdd0cbd417e780be2abc5e0cb563ba6d Mon Sep 17 00:00:00 2001 From: Nathan Bezualem Date: Sun, 7 Jun 2026 17:51:36 -0400 Subject: [PATCH 1/3] . --- datafusion/core/src/physical_planner.rs | 5 +++ .../sqllogictest/test_files/subquery.slt | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index dd741ee6ff12e..b6fd67c2a048b 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -1741,6 +1741,11 @@ impl DefaultPhysicalPlanner { } else if session_state.config().target_partitions() > 1 && session_state.config().repartition_joins() && !prefer_hash_join + && !*null_aware + // Null-aware joins (e.g. `NOT IN` with a nullable subquery) must + // use the CollectLeft HashJoin below: SortMergeJoinExec does not + // implement null-aware anti-join semantics and would return wrong + // results when the right side contains a null join key. { // Use SortMergeJoin if hash join is not preferred let join_on_len = join_on.len(); diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 3d6f8027454c7..7249e6aeba6ba 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2437,3 +2437,45 @@ DROP TABLE sq_count_customer; statement ok DROP TABLE sq_count_orders; + +# Regression test: `NOT IN` is a null-aware anti join. When the subquery yields a +# NULL the predicate is never TRUE, so the query must return zero rows. This must +# hold regardless of the chosen physical join operator. Previously, with +# prefer_hash_join = false and multiple partitions, the planner routed the +# null-aware anti join to SortMergeJoin (which is not null-aware) and returned +# wrong results; null-aware anti joins must use the CollectLeft HashJoin. +statement ok +set datafusion.optimizer.prefer_hash_join = false; + +statement ok +CREATE TABLE nia_left(x INT) AS VALUES (1), (2), (3), (4); + +statement ok +CREATE TABLE nia_right_with_null(y INT) AS VALUES (2), (NULL); + +statement ok +CREATE TABLE nia_right_no_null(y INT) AS VALUES (2), (4); + +# Subquery contains a NULL -> NOT IN must return no rows. +query I +SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null) ORDER BY x; +---- + +# Subquery has no NULL -> NOT IN behaves like a normal anti join. +query I +SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_no_null) ORDER BY x; +---- +1 +3 + +statement ok +DROP TABLE nia_left; + +statement ok +DROP TABLE nia_right_with_null; + +statement ok +DROP TABLE nia_right_no_null; + +statement ok +set datafusion.optimizer.prefer_hash_join = true; From a4ccbe4d9de5d3d3c120af4569fd0e7c87c0c82f Mon Sep 17 00:00:00 2001 From: Nathan Bezualem Date: Thu, 25 Jun 2026 20:46:26 -0400 Subject: [PATCH 2/3] . --- .../sqllogictest/test_files/subquery.slt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 7249e6aeba6ba..834eaa70c1f6d 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2444,6 +2444,7 @@ DROP TABLE sq_count_orders; # prefer_hash_join = false and multiple partitions, the planner routed the # null-aware anti join to SortMergeJoin (which is not null-aware) and returned # wrong results; null-aware anti joins must use the CollectLeft HashJoin. + statement ok set datafusion.optimizer.prefer_hash_join = false; @@ -2461,6 +2462,21 @@ query I SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null) ORDER BY x; ---- +# The null-aware anti join must be planned as a CollectLeft HashJoinExec even with +# prefer_hash_join = false: SortMergeJoinExec is not null-aware and must not be used. +query TT +EXPLAIN SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null); +---- +logical_plan +01)LeftAnti Join: nia_left.x = __correlated_sq_1.y +02)--TableScan: nia_left projection=[x] +03)--SubqueryAlias: __correlated_sq_1 +04)----TableScan: nia_right_with_null projection=[y] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(x@0, y@0)] +02)--DataSourceExec: partitions=1, partition_sizes=[1] +03)--DataSourceExec: partitions=1, partition_sizes=[1] + # Subquery has no NULL -> NOT IN behaves like a normal anti join. query I SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_no_null) ORDER BY x; @@ -2478,4 +2494,4 @@ statement ok DROP TABLE nia_right_no_null; statement ok -set datafusion.optimizer.prefer_hash_join = true; +reset datafusion.optimizer.prefer_hash_join; From 72f91cd33e459e893c84d189e1b8b37a19886e61 Mon Sep 17 00:00:00 2001 From: Nathan Bezualem Date: Thu, 2 Jul 2026 13:55:45 -0400 Subject: [PATCH 3/3] . --- datafusion/sqllogictest/test_files/subquery.slt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 834eaa70c1f6d..8d5e31d60babc 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2468,12 +2468,12 @@ query TT EXPLAIN SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null); ---- logical_plan -01)LeftAnti Join: nia_left.x = __correlated_sq_1.y +01)LeftAnti Join: nia_left.x = __correlated_sq_1.y null_aware 02)--TableScan: nia_left projection=[x] 03)--SubqueryAlias: __correlated_sq_1 04)----TableScan: nia_right_with_null projection=[y] physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(x@0, y@0)] +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(x@0, y@0)], null_aware 02)--DataSourceExec: partitions=1, partition_sizes=[1] 03)--DataSourceExec: partitions=1, partition_sizes=[1]