Open ponychicken opened 3 days ago
import polars as pl from datetime import datetime, date, timedelta from deltalake import DeltaTable # Define schema schema = { "timestamp": pl.Datetime(time_unit="us", time_zone="UTC"), "date": pl.Date, "lon": pl.Decimal(precision=9, scale=6), "lat": pl.Decimal(precision=9, scale=6), "altitude": pl.Decimal(precision=6, scale=1), "course": pl.Decimal(precision=4, scale=1), "heading": pl.Decimal(precision=4, scale=1), "speed": pl.Decimal(precision=4, scale=1), "name": pl.String, "s3_key": pl.String, } # Create sample data data = { "timestamp": [datetime(2024, 3, 20, 12, 30, 0)], "date": [date(2024, 3, 20)], "lon": [122.123456], "lat": [41.987654], "altitude": [150.5], "course": [45.5], "heading": [90.0], "speed": [12.5], "name": ["NAME"], "s3_key": ["s3"], } # Create DataFrame with sample data and schema df = pl.DataFrame(data, schema=schema) # Write DataFrame as Delta table df.write_delta("B") # Write again, skipping duplicates df.write_delta( "B", mode="merge", delta_merge_options={ "predicate": """ t.timestamp = s.timestamp AND t.lat = s.lat AND t.lon = s.lon """, "source_alias": "s", "target_alias": "t", } ).when_matched_update_all().when_not_matched_insert_all().execute()
Traceback (most recent call last): File "<stdin>", line 13, in <module> File ".venv/lib/python3.12/site-packages/deltalake/table.py", line 1800, in execute metrics = self._table.merge_execute(self._builder) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _internal.DeltaError: Generic DeltaTable error: Unable to convert expression to string
If I modify the predicate to only check the timestamp or check a string, it will succeed
Write should succeed
This is probably a upstream bug: https://github.com/delta-io/delta-rs/issues/3033
It seems we are missing a match arm for decimal to allow round tripping through the log
Checks
Reproducible example
Log output
Issue description
If I modify the predicate to only check the timestamp or check a string, it will succeed
Expected behavior
Write should succeed
Installed versions