CI failures on test_integration_crashpad

[vaind]

I've noticed this test fail in a PR but it didn't make sense, considering the changes so I reran the latest successful a CI run on the master branch and indeed it fails now, with no code changes (note that this is "Attempt 3" while "Attempt 2" was successful):

• https://github.com/getsentry/sentry-native/actions/runs/11556758981/job/32386152175

Details

``` =================================== FAILURES =================================== ______________ test_crashpad_dumping_crash[run_args0-build_args0] ______________ cmake = > httpserver = , run_args = [] build_args = {'CMAKE_RUNTIME_OUTPUT_DIRECTORY': PosixPath('/private/var/folders/g6/rgtlsw6n123b0gt5483s5_cm0000gn/T/pytest-of-runne.../var/folders/g6/rgtlsw6n123b0gt5483s5_cm0000gn/T/pytest-of-runner/pytest-0/cmake4'), 'SENTRY_BACKEND': 'crashpad', ...} @pytest.mark.parametrize( "run_args,build_args", [ # if we crash, we want a dump ([], {"SENTRY_TRANSPORT_COMPRESSION": "Off"}), ([], {"SENTRY_TRANSPORT_COMPRESSION": "On"}), # if we crash and before-send doesn't discard, we want a dump pytest.param( ["before-send"], {}, marks=pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", ), ), # if on_crash() is non-discarding, a discarding before_send() is overruled, so we get a dump pytest.param( ["discarding-before-send", "on-crash"], {}, marks=pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", ), ), ], ) def test_crashpad_dumping_crash(cmake, httpserver, run_args, build_args): build_args.update({"SENTRY_BACKEND": "crashpad"}) tmp_path = cmake(["sentry_example"], build_args) # make sure we are isolated from previous runs shutil.rmtree(tmp_path / ".sentry-native", ignore_errors=True) env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data("OK") httpserver.expect_request("/api/123456/envelope/").respond_with_data("OK") with httpserver.wait(timeout=10) as waiting: child = run( tmp_path, "sentry_example", ["log", "start-session", "attachment", "overflow-breadcrumbs", "crash"] + run_args, env=env, ) assert child.returncode # well, it's a crash after all assert waiting.result # the session crash heuristic on Mac uses timestamps, so make sure we have # a small delay here time.sleep(1) run(tmp_path, "sentry_example", ["log", "no-setup"], check=True, env=env) assert len(httpserver.log) == 2 session, multipart = ( (httpserver.log[0][0], httpserver.log[1][0]) if is_session_envelope(httpserver.log[0][0].get_data()) else (httpserver.log[1][0], httpserver.log[0][0]) ) if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": assert_gzip_file_header(session.get_data()) envelope = Envelope.deserialize(session.get_data()) assert_session(envelope, {"status": "crashed", "errors": 1}) > assert_crashpad_upload(multipart) tests/test_integration_crashpad.py:192: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/assertions.py:330: in assert_crashpad_upload assert_overflowing_breadcrumb(attachments) tests/assertions.py:322: in assert_overflowing_breadcrumb assert_breadcrumb_inner(attachments.breadcrumb1) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ breadcrumbs = [] def assert_breadcrumb_inner(breadcrumbs): expected = { "type": "http", "message": "debug crumb", "category": "example!", "level": "debug", "data": { "url": "https://example.com/api/1.0/users", "method": "GET", "status_code": 200, "reason": "OK", }, } > assert any(matches(b, expected) for b in breadcrumbs) E assert False E + where False = any(. at 0x101fbc130>) tests/assertions.py:182: AssertionError ------------------------------ Captured log call ------------------------------- INFO werkzeug:_internal.py:97 127.0.0.1 - - [01/Nov/2024 12:32:17] "POST /api/123456/minidump/?sentry_client=sentry.native/0.7.11&sentry_key=uiaeosnrtdy&guid=2e4440ae-1a48-498e-ac26-0ccfd22ab1c3 HTTP/1.1" 200 - INFO werkzeug:_internal.py:97 127.0.0.1 - - [01/Nov/2024 12:32:18] "POST /api/123456/envelope/ HTTP/1.1" 200 - ______________ test_crashpad_dumping_crash[run_args1-build_args1] ______________ cmake = > httpserver = , run_args = [] build_args = {'CMAKE_RUNTIME_OUTPUT_DIRECTORY': PosixPath('/private/var/folders/g6/rgtlsw6n123b0gt5483s5_cm0000gn/T/pytest-of-runne.../var/folders/g6/rgtlsw6n123b0gt5483s5_cm0000gn/T/pytest-of-runner/pytest-0/cmake5'), 'SENTRY_BACKEND': 'crashpad', ...} @pytest.mark.parametrize( "run_args,build_args", [ # if we crash, we want a dump ([], {"SENTRY_TRANSPORT_COMPRESSION": "Off"}), ([], {"SENTRY_TRANSPORT_COMPRESSION": "On"}), # if we crash and before-send doesn't discard, we want a dump pytest.param( ["before-send"], {}, marks=pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", ), ), # if on_crash() is non-discarding, a discarding before_send() is overruled, so we get a dump pytest.param( ["discarding-before-send", "on-crash"], {}, marks=pytest.mark.skipif( sys.platform == "darwin", reason="crashpad doesn't provide SetFirstChanceExceptionHandler on macOS", ), ), ], ) def test_crashpad_dumping_crash(cmake, httpserver, run_args, build_args): build_args.update({"SENTRY_BACKEND": "crashpad"}) tmp_path = cmake(["sentry_example"], build_args) # make sure we are isolated from previous runs shutil.rmtree(tmp_path / ".sentry-native", ignore_errors=True) env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data("OK") httpserver.expect_request("/api/123456/envelope/").respond_with_data("OK") with httpserver.wait(timeout=10) as waiting: child = run( tmp_path, "sentry_example", ["log", "start-session", "attachment", "overflow-breadcrumbs", "crash"] + run_args, env=env, ) assert child.returncode # well, it's a crash after all assert waiting.result # the session crash heuristic on Mac uses timestamps, so make sure we have # a small delay here time.sleep(1) run(tmp_path, "sentry_example", ["log", "no-setup"], check=True, env=env) assert len(httpserver.log) == 2 session, multipart = ( (httpserver.log[0][0], httpserver.log[1][0]) if is_session_envelope(httpserver.log[0][0].get_data()) else (httpserver.log[1][0], httpserver.log[0][0]) ) if build_args.get("SENTRY_TRANSPORT_COMPRESSION") == "On": assert_gzip_file_header(session.get_data()) envelope = Envelope.deserialize(session.get_data()) assert_session(envelope, {"status": "crashed", "errors": 1}) > assert_crashpad_upload(multipart) tests/test_integration_crashpad.py:192: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/assertions.py:330: in assert_crashpad_upload assert_overflowing_breadcrumb(attachments) tests/assertions.py:322: in assert_overflowing_breadcrumb assert_breadcrumb_inner(attachments.breadcrumb1) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ breadcrumbs = [] def assert_breadcrumb_inner(breadcrumbs): expected = { "type": "http", "message": "debug crumb", "category": "example!", "level": "debug", "data": { "url": "https://example.com/api/1.0/users", "method": "GET", "status_code": 200, "reason": "OK", }, } > assert any(matches(b, expected) for b in breadcrumbs) E assert False E + where False = any(. at 0x101ecb5b0>) tests/assertions.py:182: AssertionError ------------------------------ Captured log call ------------------------------- INFO werkzeug:_internal.py:97 127.0.0.1 - - [01/Nov/2024 12:33:16] "POST /api/123456/minidump/?sentry_client=sentry.native/0.7.11&sentry_key=uiaeosnrtdy&guid=e8912129-337a-4a25-9484-16a2363a420b HTTP/1.1" 200 - INFO werkzeug:_internal.py:97 127.0.0.1 - - [01/Nov/2024 12:33:17] "POST /api/123456/envelope/ HTTP/1.1" 200 - _____________________ test_crashpad_dumping_stack_overflow _____________________ cmake = > httpserver = def test_crashpad_dumping_stack_overflow(cmake, httpserver): tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) # make sure we are isolated from previous runs shutil.rmtree(tmp_path / ".sentry-native", ignore_errors=True) env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data("OK") httpserver.expect_request("/api/123456/envelope/").respond_with_data("OK") with httpserver.wait(timeout=10) as waiting: child = run( tmp_path, "sentry_example", ["log", "start-session", "attachment", "stack-overflow"], env=env, ) assert child.returncode # well, it's a crash after all assert waiting.result # the session crash heuristic on Mac uses timestamps, so make sure we have # a small delay here time.sleep(1) run(tmp_path, "sentry_example", ["log", "no-setup"], check=True, env=env) assert len(httpserver.log) == 2 session, multipart = ( (httpserver.log[0][0], httpserver.log[1][0]) if is_session_envelope(httpserver.log[0][0].get_data()) else (httpserver.log[1][0], httpserver.log[0][0]) ) envelope = Envelope.deserialize(session.get_data()) assert_session(envelope, {"status": "crashed", "errors": 1}) > assert_crashpad_upload(multipart) tests/test_integration_crashpad.py:231: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/assertions.py:330: in assert_crashpad_upload assert_overflowing_breadcrumb(attachments) tests/assertions.py:322: in assert_overflowing_breadcrumb assert_breadcrumb_inner(attachments.breadcrumb1) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ breadcrumbs = [] def assert_breadcrumb_inner(breadcrumbs): expected = { "type": "http", "message": "debug crumb", "category": "example!", "level": "debug", "data": { "url": "https://example.com/api/1.0/users", "method": "GET", "status_code": 200, "reason": "OK", }, } > assert any(matches(b, expected) for b in breadcrumbs) E assert False E + where False = any(. at 0x101fbc040>) tests/assertions.py:182: AssertionError ------------------------------ Captured log call ------------------------------- INFO werkzeug:_internal.py:97 127.0.0.1 - - [01/Nov/2024 12:33:17] "POST /api/123456/minidump/?sentry_client=sentry.native/0.7.11&sentry_key=uiaeosnrtdy&guid=1fdaf2c4-29cb-495e-a465-2099cfd02a77 HTTP/1.1" 200 - INFO werkzeug:_internal.py:97 127.0.0.1 - - [01/Nov/2024 12:33:19] "POST /api/123456/envelope/ HTTP/1.1" 200 - ______________________ test_crashpad_crash_after_shutdown ______________________ cmake = > httpserver = @pytest.mark.skipif( sys.platform == "linux", reason="linux clears the signal handlers on shutdown" ) def test_crashpad_crash_after_shutdown(cmake, httpserver): tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) # make sure we are isolated from previous runs shutil.rmtree(tmp_path / ".sentry-native", ignore_errors=True) env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data("OK") with httpserver.wait(timeout=10) as waiting: child = run( tmp_path, "sentry_example", ["log", "crash-after-shutdown"], env=env, ) assert child.returncode # well, it's a crash after all assert waiting.result # the session crash heuristic on Mac uses timestamps, so make sure we have # a small delay here time.sleep(1) run(tmp_path, "sentry_example", ["log", "no-setup"], check=True, env=env) assert len(httpserver.log) == 1 > assert_crashpad_upload(httpserver.log[0][0]) tests/test_integration_crashpad.py:318: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/assertions.py:330: in assert_crashpad_upload assert_overflowing_breadcrumb(attachments) tests/assertions.py:322: in assert_overflowing_breadcrumb assert_breadcrumb_inner(attachments.breadcrumb1) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ breadcrumbs = [] def assert_breadcrumb_inner(breadcrumbs): expected = { "type": "http", "message": "debug crumb", "category": "example!", "level": "debug", "data": { "url": "https://example.com/api/1.0/users", "method": "GET", "status_code": 200, "reason": "OK", }, } > assert any(matches(b, expected) for b in breadcrumbs) E assert False E + where False = any(. at 0x101fbcd60>) tests/assertions.py:182: AssertionError ```

[supervacuus]

A werkzeug update from 3.0.6 to 3.1.0 broke our multipart-based assertions.

I can reproduce this locally. As an intermediate fix, you can add a version pin in tests/requirements.txt for werkzeug==3.0.6 if it currently blocks you on the PR. I will try to find out what broke between versions.

[supervacuus]

Okay, it seems this was a bug in 3.1.0, and the fix has already been released as 3.1.1. There is no need to pin.