search

pola-rs / polars

Dataframes powered by a multithreaded, vectorized query engine, written in Rust
https://docs.pola.rs
Other
30.51k stars 1.98k forks source link

err: strptime panics (instead of erroring) for out-of-bounds dates #19928

Open Shoeboxam opened 1 day ago

Shoeboxam commented 1 day ago

Checks

Reproducible example

df!("d" => &["2300-01-01T00:00:00Z".to_string()])?
        .lazy()
        .with_column(col("d").str().strptime(
            DataType::Datetime(TimeUnit::Nanoseconds, None),
            StrptimeOptions {
                format: Some("%Y-%m-%dT%H:%M:%SZ".to_string()),
                strict: false,
                exact: true,
                cache: true,
            },
            lit("null"),
        )).collect();

Log output

thread 'transformations::make_stable_expr::expr_str::expr_strptime::test::test_strptime_mismatch' panicked at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.41.3/src/chunked_array/temporal/conversion.rs:37:39:
called `Option::unwrap()` on a `None` value
stack backtrace:
   0: rust_begin_unwind
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:662:5
   1: core::panicking::panic_fmt
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panicking.rs:74:14
   2: core::panicking::panic
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panicking.rs:148:5
   3: core::option::unwrap_failed
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/option.rs:2015:5
   4: core::option::Option<T>::unwrap
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/option.rs:965:21
   5: polars_core::chunked_array::temporal::conversion::datetime_to_timestamp_ns
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.41.3/src/chunked_array/temporal/conversion.rs:37:5
   6: polars_time::chunkedarray::string::infer::transform_datetime_ns
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-time-0.41.3/src/chunkedarray/string/infer.rs:346:25
   7: polars_time::chunkedarray::string::StringMethods::as_datetime::{{closure}}
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-time-0.41.3/src/chunkedarray/string/mod.rs:320:37
   8: polars_utils::cache::FastCachedFunc<T,R,F>::eval
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-utils-0.41.3/src/cache.rs:36:13
   9: polars_time::chunkedarray::string::StringMethods::as_datetime::{{closure}}
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-time-0.41.3/src/chunkedarray/string/mod.rs:326:49
  10: polars_core::chunked_array::ops::apply::<impl polars_core::chunked_array::ChunkedArray<T>>::apply_generic::{{closure}}::{{closure}}
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.41.3/src/chunked_array/ops/apply.rs:95:54
  11: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &mut F>::call_once
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/ops/function.rs:305:13
  12: core::option::Option<T>::map
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/option.rs:1105:29
  13: <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::next
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:107:26
  14: <polars_arrow::array::primitive::PrimitiveArray<T> as polars_arrow::array::static_array_collect::ArrayFromIter<core::option::Option<T>>>::arr_from_iter
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-arrow-0.41.3/src/array/static_array_collect.rs:224:36
  15: polars_arrow::array::static_array_collect::ArrayCollectIterExt::collect_arr
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-arrow-0.41.3/src/array/static_array_collect.rs:137:9
  16: polars_core::chunked_array::ops::apply::<impl polars_core::chunked_array::ChunkedArray<T>>::apply_generic::{{closure}}
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.41.3/src/chunked_array/ops/apply.rs:95:28
  17: core::iter::adapters::map::map_fold::{{closure}}
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:88:28
  18: core::iter::adapters::map::map_fold::{{closure}}
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:88:21
  19: <core::slice::iter::Iter<T> as core::iter::traits::iterator::Iterator>::fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/slice/iter/macros.rs:232:27
  20: <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:128:9
  21: <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:128:9
  22: <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:128:9
  23: core::iter::traits::iterator::Iterator::for_each
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/traits/iterator.rs:813:9
  24: alloc::vec::Vec<T,A>::extend_trusted
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/mod.rs:3121:17
  25: <alloc::vec::Vec<T,A> as alloc::vec::spec_extend::SpecExtend<T,I>>::spec_extend
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/spec_extend.rs:26:9
  26: <alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/spec_from_iter_nested.rs:60:9
  27: <alloc::vec::Vec<T> as alloc::vec::spec_from_iter::SpecFromIter<T,I>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/spec_from_iter.rs:33:9
  28: <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/mod.rs:2985:9
  29: core::iter::traits::iterator::Iterator::collect
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/traits/iterator.rs:2000:9
  30: polars_core::chunked_array::from::<impl polars_core::chunked_array::ChunkedArray<T>>::from_chunk_iter
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.41.3/src/chunked_array/from.rs:121:22
  31: polars_core::chunked_array::ops::apply::<impl polars_core::chunked_array::ChunkedArray<T>>::apply_generic
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.41.3/src/chunked_array/ops/apply.rs:96:13
  32: polars_time::chunkedarray::string::StringMethods::as_datetime
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-time-0.41.3/src/chunkedarray/string/mod.rs:326:17
  33: polars_plan::dsl::function_expr::strings::to_datetime
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-plan-0.41.3/src/dsl/function_expr/strings.rs:695:9
  34: polars_plan::dsl::function_expr::strings::strptime
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-plan-0.41.3/src/dsl/function_expr/strings.rs:618:13
  35: polars_plan::dsl::function_expr::strings::<impl core::convert::From<polars_plan::dsl::function_expr::strings::StringFunction> for polars_plan::dsl::expr_dyn_fn::SpecialEq<alloc::sync::Arc<dyn polars_plan::dsl::expr_dyn_fn::SeriesUdf>>>::from::{{closure}}
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-plan-0.41.3/src/dsl/function_expr/mod.rs:802:13
  36: <F as polars_plan::dsl::expr_dyn_fn::SeriesUdf>::call_udf
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-plan-0.41.3/src/dsl/expr_dyn_fn.rs:83:9
  37: polars_expr::expressions::apply::ApplyExpr::eval_and_flatten
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-expr-0.41.3/src/expressions/apply.rs:127:28
  38: <polars_expr::expressions::apply::ApplyExpr as polars_expr::expressions::PhysicalExpr>::evaluate
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-expr-0.41.3/src/expressions/apply.rs:335:16
  39: polars_mem_engine::executors::projection_utils::run_exprs_seq::{{closure}}
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-mem-engine-0.41.3/src/executors/projection_utils.rs:218:29
  40: core::iter::adapters::map::map_try_fold::{{closure}}
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:95:28
  41: core::iter::traits::iterator::Iterator::try_fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/traits/iterator.rs:2405:21
  42: <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::try_fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/map.rs:121:9
  43: <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::try_fold
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/mod.rs:191:9
  44: core::iter::traits::iterator::Iterator::try_for_each
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/traits/iterator.rs:2467:9
  45: <core::iter::adapters::GenericShunt<I,R> as core::iter::traits::iterator::Iterator>::next
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/mod.rs:174:14
  46: <alloc::vec::Vec<T> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/spec_from_iter_nested.rs:24:32
  47: <alloc::vec::Vec<T> as alloc::vec::spec_from_iter::SpecFromIter<T,I>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/spec_from_iter.rs:33:9
  48: <alloc::vec::Vec<T> as core::iter::traits::collect::FromIterator<T>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/vec/mod.rs:2985:9
  49: core::iter::traits::iterator::Iterator::collect
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/traits/iterator.rs:2000:9
  50: <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter::{{closure}}
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/result.rs:1958:51
  51: core::iter::adapters::try_process
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/adapters/mod.rs:160:17
  52: <core::result::Result<V,E> as core::iter::traits::collect::FromIterator<core::result::Result<A,E>>>::from_iter
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/result.rs:1958:9
  53: core::iter::traits::iterator::Iterator::collect
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/iter/traits/iterator.rs:2000:9
  54: polars_mem_engine::executors::projection_utils::run_exprs_seq
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-mem-engine-0.41.3/src/executors/projection_utils.rs:218:5
  55: polars_mem_engine::executors::projection_utils::evaluate_physical_expressions
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-mem-engine-0.41.3/src/executors/projection_utils.rs:236:28
  56: polars_mem_engine::executors::stack::StackExec::execute_impl
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-mem-engine-0.41.3/src/executors/stack.rs:60:27
  57: <polars_mem_engine::executors::stack::StackExec as polars_mem_engine::executors::executor::Executor>::execute
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-mem-engine-0.41.3/src/executors/stack.rs:118:13
  58: polars_lazy::frame::LazyFrame::_collect_post_opt
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-lazy-0.41.3/src/frame/mod.rs:692:9
  59: polars_lazy::frame::LazyFrame::collect
             at /Users/michael/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-lazy-0.41.3/src/frame/mod.rs:737:9
  60: opendp::transformations::make_stable_expr::expr_str::expr_strptime::test::test_strptime_mismatch
             at ./src/transformations/make_stable_expr/expr_str/expr_strptime/test.rs:104:22
  61: opendp::transformations::make_stable_expr::expr_str::expr_strptime::test::test_strptime_mismatch::{{closure}}
             at ./src/transformations/make_stable_expr/expr_str/expr_strptime/test.rs:90:32
  62: core::ops::function::FnOnce::call_once
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/ops/function.rs:250:5
  63: core::ops::function::FnOnce::call_once
             at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
test transformations::make_stable_expr::expr_str::expr_strptime::test::test_strptime_mismatch ... FAILED

Issue description

Even when strict is false (expecting null), strptime can panic for some inputs, like above in the year 2300. From what testing I've done, microseconds and milliseconds never panic under any condition, they always return null.

Expected behavior

If strict is false, I'd expect nanosecond datetime parsing to return null upon failure, as in micro and milli parsing.

Thanks!

Installed versions

polars = { version = "=0.41.3", features = [ "lazy", "csv", "dtype-struct", "dtype-array", "dtype-date", "dtype-datetime", "dtype-time", "parquet", "serde-lazy", "streaming", "round_series", "rank", "polars-ops", "abs", "meta", "cutqcut", "strings", ], optional = true }
MarcoGorelli commented 1 day ago

thanks @Shoeboxam for the report - this should both raise a proper error message (strict=True) and return None (strict=False)