search

apache / datafusion

Apache DataFusion SQL Query Engine
https://datafusion.apache.org/
Apache License 2.0
6.33k stars 1.2k forks source link

When upgrading to DataFusion 43 some queries fail with panic "LeftMark join type does not support swapping" #13425

Closed findepi closed 5 days ago

findepi commented 1 week ago

repro

CREATE TABLE t1(a INTEGER, b INTEGER, c INTEGER, d INTEGER, e INTEGER);
INSERT INTO t1(e,c,b,d,a) VALUES(103,102,100,101,104);
INSERT INTO t1(a,c,d,e,b) VALUES(107,106,108,109,105);
INSERT INTO t1(d,c,e,a,b) VALUES(116,119,117,115,118);
INSERT INTO t1(c,d,b,e,a) VALUES(123,122,124,120,121);
INSERT INTO t1(b,a,e,d,c) VALUES(145,149,146,148,147);
INSERT INTO t1(b,c,a,d,e) VALUES(151,150,153,154,152);
INSERT INTO t1(c,b,a,d,e) VALUES(161,160,163,164,162);
INSERT INTO t1(b,d,a,e,c) VALUES(167,169,168,165,166);
INSERT INTO t1(d,b,c,e,a) VALUES(171,170,172,173,174);
INSERT INTO t1(e,c,a,d,b) VALUES(177,176,179,178,175);
INSERT INTO t1(b,e,a,d,c) VALUES(181,180,182,183,184);
INSERT INTO t1(c,e,a,b,d) VALUES(208,209,205,206,207);
INSERT INTO t1(c,e,a,d,b) VALUES(214,210,213,212,211);
INSERT INTO t1(b,c,a,d,e) VALUES(218,215,216,217,219);
INSERT INTO t1(e,c,b,a,d) VALUES(242,244,240,243,241);
INSERT INTO t1(e,d,c,b,a) VALUES(246,248,247,249,245);

SELECT a+b*2,
       a+b*2+c*3+d*4,
       CASE WHEN a<b-3 THEN 111 WHEN a<=b THEN 222
        WHEN a<b+3 THEN 333 ELSE 444 END,
       b,
       c-d,
       a+b*2+c*3+d*4+e*5
  FROM t1
 WHERE (e>c OR e<d)
    OR EXISTS(SELECT 1 FROM t1 AS x WHERE x.b<t1.b)
 ORDER BY 3,5,2,1,4,6;

stacktrace from the debugger, in case it's useful:

[Inlined] datafusion::physical_optimizer::join_selection::swap_join_type join_selection.rs:137
[Inlined] datafusion::physical_optimizer::join_selection::swap_nl_join join_selection.rs:226
[Inlined] datafusion::physical_optimizer::join_selection::statistical_join_selection_subrule join_selection.rs:498
[Inlined] <datafusion::physical_optimizer::join_selection::JoinSelection as datafusion_physical_optimizer::optimizer::PhysicalOptimizerRule>::optimize::{{closure}} join_selection.rs:337
core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &mut F>::call_once function.rs:305
[Inlined] datafusion_common::tree_node::Transformed<T>::transform_parent tree_node.rs:756
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl::{{closure}} tree_node.rs:258
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect::{{closure}} tree_node.rs:826
[Inlined] core::iter::adapters::map::map_try_fold::{{closure}} map.rs:96
[Inlined] core::iter::adapters::cloned::clone_try_fold::{{closure}} cloned.rs:29
<alloc::vec::into_iter::IntoIter<T,A> as core::iter::traits::iterator::Iterator>::try_fold into_iter.rs:345
[Inlined] <core::iter::adapters::cloned::Cloned<I> as core::iter::traits::iterator::Iterator>::try_fold cloned.rs:54
[Inlined] <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold map.rs:122
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold mod.rs:204
[Inlined] core::iter::traits::iterator::Iterator::try_for_each iterator.rs:2472
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next mod.rs:187
<alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter spec_from_iter_nested.rs:26
[Inlined] alloc::vec::in_place_collect::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter in_place_collect.rs:246
[Inlined] <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter mod.rs:2977
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}} result.rs:1960
core::iter::adapters::try_process mod.rs:173
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter result.rs:1960
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect tree_node.rs:824
<alloc::sync::Arc<T> as datafusion_common::tree_node::TreeNode>::map_children tree_node.rs:953
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl::{{closure}} tree_node.rs:258
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect::{{closure}} tree_node.rs:826
[Inlined] core::iter::adapters::map::map_try_fold::{{closure}} map.rs:96
[Inlined] core::iter::adapters::cloned::clone_try_fold::{{closure}} cloned.rs:29
<alloc::vec::into_iter::IntoIter<T,A> as core::iter::traits::iterator::Iterator>::try_fold into_iter.rs:345
[Inlined] <core::iter::adapters::cloned::Cloned<I> as core::iter::traits::iterator::Iterator>::try_fold cloned.rs:54
[Inlined] <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold map.rs:122
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold mod.rs:204
[Inlined] core::iter::traits::iterator::Iterator::try_for_each iterator.rs:2472
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next mod.rs:187
<alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter spec_from_iter_nested.rs:26
[Inlined] alloc::vec::in_place_collect::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter in_place_collect.rs:246
[Inlined] <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter mod.rs:2977
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}} result.rs:1960
core::iter::adapters::try_process mod.rs:173
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter result.rs:1960
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect tree_node.rs:824
<alloc::sync::Arc<T> as datafusion_common::tree_node::TreeNode>::map_children tree_node.rs:953
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl::{{closure}} tree_node.rs:258
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect::{{closure}} tree_node.rs:826
[Inlined] core::iter::adapters::map::map_try_fold::{{closure}} map.rs:96
[Inlined] core::iter::adapters::cloned::clone_try_fold::{{closure}} cloned.rs:29
<alloc::vec::into_iter::IntoIter<T,A> as core::iter::traits::iterator::Iterator>::try_fold into_iter.rs:345
[Inlined] <core::iter::adapters::cloned::Cloned<I> as core::iter::traits::iterator::Iterator>::try_fold cloned.rs:54
[Inlined] <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold map.rs:122
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold mod.rs:204
[Inlined] core::iter::traits::iterator::Iterator::try_for_each iterator.rs:2472
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next mod.rs:187
<alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter spec_from_iter_nested.rs:26
[Inlined] alloc::vec::in_place_collect::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter in_place_collect.rs:246
[Inlined] <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter mod.rs:2977
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}} result.rs:1960
core::iter::adapters::try_process mod.rs:173
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter result.rs:1960
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect tree_node.rs:824
<alloc::sync::Arc<T> as datafusion_common::tree_node::TreeNode>::map_children tree_node.rs:953
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl::{{closure}} tree_node.rs:258
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect::{{closure}} tree_node.rs:826
[Inlined] core::iter::adapters::map::map_try_fold::{{closure}} map.rs:96
[Inlined] core::iter::adapters::cloned::clone_try_fold::{{closure}} cloned.rs:29
<alloc::vec::into_iter::IntoIter<T,A> as core::iter::traits::iterator::Iterator>::try_fold into_iter.rs:345
[Inlined] <core::iter::adapters::cloned::Cloned<I> as core::iter::traits::iterator::Iterator>::try_fold cloned.rs:54
[Inlined] <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold map.rs:122
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold mod.rs:204
[Inlined] core::iter::traits::iterator::Iterator::try_for_each iterator.rs:2472
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next mod.rs:187
<alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter spec_from_iter_nested.rs:26
[Inlined] alloc::vec::in_place_collect::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter in_place_collect.rs:246
[Inlined] <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter mod.rs:2977
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}} result.rs:1960
core::iter::adapters::try_process mod.rs:173
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter result.rs:1960
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect tree_node.rs:824
<alloc::sync::Arc<T> as datafusion_common::tree_node::TreeNode>::map_children tree_node.rs:953
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl::{{closure}} tree_node.rs:258
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect::{{closure}} tree_node.rs:826
[Inlined] core::iter::adapters::map::map_try_fold::{{closure}} map.rs:96
[Inlined] core::iter::adapters::cloned::clone_try_fold::{{closure}} cloned.rs:29
<alloc::vec::into_iter::IntoIter<T,A> as core::iter::traits::iterator::Iterator>::try_fold into_iter.rs:345
[Inlined] <core::iter::adapters::cloned::Cloned<I> as core::iter::traits::iterator::Iterator>::try_fold cloned.rs:54
[Inlined] <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold map.rs:122
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold mod.rs:204
[Inlined] core::iter::traits::iterator::Iterator::try_for_each iterator.rs:2472
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next mod.rs:187
<alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter spec_from_iter_nested.rs:26
[Inlined] alloc::vec::in_place_collect::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter in_place_collect.rs:246
[Inlined] <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter mod.rs:2977
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}} result.rs:1960
core::iter::adapters::try_process mod.rs:173
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter result.rs:1960
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect tree_node.rs:824
<alloc::sync::Arc<T> as datafusion_common::tree_node::TreeNode>::map_children tree_node.rs:953
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl::{{closure}} tree_node.rs:258
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect::{{closure}} tree_node.rs:826
[Inlined] core::iter::adapters::map::map_try_fold::{{closure}} map.rs:96
[Inlined] core::iter::adapters::cloned::clone_try_fold::{{closure}} cloned.rs:29
<alloc::vec::into_iter::IntoIter<T,A> as core::iter::traits::iterator::Iterator>::try_fold into_iter.rs:345
[Inlined] <core::iter::adapters::cloned::Cloned<I> as core::iter::traits::iterator::Iterator>::try_fold cloned.rs:54
[Inlined] <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold map.rs:122
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold mod.rs:204
[Inlined] core::iter::traits::iterator::Iterator::try_for_each iterator.rs:2472
[Inlined] <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next mod.rs:187
<alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter spec_from_iter_nested.rs:26
[Inlined] alloc::vec::in_place_collect::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter in_place_collect.rs:246
[Inlined] <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter mod.rs:2977
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}} result.rs:1960
core::iter::adapters::try_process mod.rs:173
[Inlined] <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter result.rs:1960
[Inlined] core::iter::traits::iterator::Iterator::collect iterator.rs:2005
[Inlined] <I as datafusion_common::tree_node::TreeNodeIterator>::map_until_stop_and_collect tree_node.rs:824
<alloc::sync::Arc<T> as datafusion_common::tree_node::TreeNode>::map_children tree_node.rs:953
datafusion_common::tree_node::TreeNode::transform_up::transform_up_impl tree_node.rs:258
[Inlined] datafusion_common::tree_node::TreeNode::transform_up tree_node.rs:262
<datafusion::physical_optimizer::join_selection::JoinSelection as datafusion_physical_optimizer::optimizer::PhysicalOptimizerRule>::optimize join_selection.rs:335
datafusion::physical_planner::DefaultPhysicalPlanner::optimize_physical_plan physical_planner.rs:1849
<datafusion::physical_planner::DefaultPhysicalPlanner as datafusion::physical_planner::PhysicalPlanner>::create_physical_plan::{{closure}} physical_planner.rs:184
[Inlined] <core::pin::Pin<P> as core::future::future::Future>::poll future.rs:123
<datafusion::execution::session_state::DefaultQueryPlanner as datafusion::execution::context::QueryPlanner>::create_physical_plan::{{closure}} session_state.rs:1846
datafusion::execution::session_state::SessionState::create_physical_plan::{{closure}} session_state.rs:715
datafusion::dataframe::DataFrame::create_physical_plan::{{closure}} mod.rs:218
datafusion::dataframe::DataFrame::collect_partitioned::{{closure}} mod.rs:1295
datafusion::execution::context::SessionContext::create_memory_table::{{closure}} mod.rs:806
<core::pin::Pin<P> as core::future::future::Future>::poll future.rs:123
datafusion::execution::context::SessionContext::execute_logical_plan::{{closure}} mod.rs:661
findepi commented 1 week ago

take