Open tekumara opened 1 year ago
dagster._core.errors.DagsterExecutionStepExecutionError: Error occurred while executing op "classified_stargazers_df": File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/execute_plan.py", line 273, in dagster_event_sequence_for_step for step_event in check.generator(step_events): File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/execute_step.py", line 369, in core_dagster_event_sequence_for_step for user_event in check.generator( File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/execute_step.py", line 90, in _step_output_error_checked_user_event_sequence for user_event in user_event_sequence: File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/compute.py", line 192, in execute_core_compute for step_output in _yield_compute_results(step_context, inputs, compute_fn): File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/compute.py", line 161, in _yield_compute_results for event in iterate_with_context( File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_utils/__init__.py", line 443, in iterate_with_context with context_fn(): File "/opt/homebrew/Cellar/python@3.11/3.11.4/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__ self.gen.throw(typ, value, traceback) File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/utils.py", line 84, in op_execution_error_boundary raise error_cls( The above exception was caused by the following exception: TypeError: Cannot compare Timestamp with datetime.date. Use ts == pd.Timestamp(date) or ts.date() == date instead. File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/utils.py", line 54, in op_execution_error_boundary yield File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_utils/__init__.py", line 445, in iterate_with_context next_output = next(iterator) ^^^^^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/compute_generator.py", line 124, in _coerce_op_compute_fn_to_iterator result = invoke_compute_fn( ^^^^^^^^^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/dagster/_core/execution/plan/compute_generator.py", line 118, in invoke_compute_fn return fn(context, **args_to_pass) if context_arg_provided else fn(**args_to_pass) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/fake_star_detector/assets/simpler_model.py", line 186, in classified_stargazers_df stargazers_with_user_info["matches_fake_heuristic"] = stargazers_with_user_info.apply( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/pandas/core/frame.py", line 9423, in apply return op.apply().__finalize__(self, method="apply") ^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/pandas/core/apply.py", line 678, in apply return self.apply_standard() ^^^^^^^^^^^^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/pandas/core/apply.py", line 798, in apply_standard results, res_index = self.apply_series_generator() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/.venv/lib/python3.11/site-packages/pandas/core/apply.py", line 814, in apply_series_generator results[i] = self.f(v) ^^^^^^^^^ File "/Users/tekumara/code3/fake-star-detector/fake_star_detector/assets/simpler_model.py", line 206, in _validate_star and (row["created_at"] > datetime.date(2022, 1, 1)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "pandas/_libs/tslibs/timestamps.pyx", line 373, in pandas._libs.tslibs.timestamps._Timestamp.__richcmp__
ran into the same problem, convert datatime.date to Timestamp helps
diff --git a/fake_star_detector/assets/simpler_model.py b/fake_star_detector/assets/simpler_model.py
index d782ff2..03035f6 100644
--- a/fake_star_detector/assets/simpler_model.py
+++ b/fake_star_detector/assets/simpler_model.py
@@ -203,7 +203,7 @@ def _validate_star(row: pd.DataFrame) -> int:
and (row["following"] < 2)
and (row["public_gists"] == 0)
and (row["public_repos"] < 5)
- and (row["created_at"] > datetime.date(2022, 1, 1))
+ and (row["created_at"] > pd.Timestamp(datetime.date(2022, 1, 1)))
and (row["email"] is None)
and (row["bio"] is None)
and (not row["blog"])
The Timestamp helped. Though, I had to make it time zone aware.
Otherwhise you get the error: TypeError: Cannot compare tz-naive and tz-aware timestamps"
The solution on my side:
(row["created_at"] > pd.Timestamp(2022, 1, 1, tz="UTC"))