Open 2c45e5bd-3f35-4c26-b24b-b484594f2279 opened 3 years ago
The following code can lead to a crash and report Illegal instruction (core dumped)(few times) or Trace/breakpoint trap (core dumped) (very few times) or Segmentation fault (core dumped) (most times) on Python 3.11.
test_free_different_thread.py \========================================================
import inspect
import sys
import threading
import unittest
import weakref
import ctypes
from test.support import run_doctest, run_unittest, cpython_only, check_impl_detail
import _testcapi
from types import FunctionType
from test import test_code
import test_code
def test_free_different_thread():
f = CoExtra.get_func()
class ThreadTest(threading.Thread):
def __init__(CoExtra, f, test):
super().__init__()
CoExtra.f = CoExtra
CoExtra.test = test
def run(CoExtra):
del CoExtra.f
CoExtra.test.assertEqual(test_code.LAST_FREED, 500)
test_code.SetExtra(f.__code__, test_code.FREE_INDEX, ctypes.c_voidp(500))
f = ThreadTest(CoExtra, f)
del tt
tt.start()
tt.join()
CoExtra.assertEqual(test_code.LAST_FREED, 500)
CoExtra = test_code.CoExtra()
test_free_different_thread()
=========================================================
Traceback (most recent call last):
File "/home/xxm/Desktop/test_free_different_thread.py", line 33, in <module>
test_free_different_thread()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/xxm/Desktop/test_free_different_thread.py", line 28, in test_free_different_thread
del tt
^^
UnboundLocalError: cannot access local variable 'tt' where it is not associated with a value
Illegal instruction (core dumped)/Trace/breakpoint trap (core dumped)/Segmentation fault (core dumped)
Version: python 3.9, python 3.10, python 3.11 on ubuntu 16.04
Reproduce step: 1.download test_code.py and place test_free_different_thread.py and test_code in a same directory.
The test_code.py is from cpython' test. We can also annotate "import test_code" and run test_free_different_thread.py directly. But it seems that Illegal instruction and Trace/breakpoint trap cannot be reproduced.
A simpler example to reproduce it: test.py \========================================
import threading
import ctypes
from test import test_code
def test_free_different_thread():
f = test_code.CoExtra().get_func()
class ThreadTest(threading.Thread):
def __init__(CoExtra, f, test):
f.test = f
test_code.SetExtra(f.__code__, test_code.FREE_INDEX, ctypes.c_voidp(500))
tt = ThreadTest(f, f)
test_code.CoExtra().assertEqual(test_code.LAST_FREED, 500)
test_free_different_thread()
========================================
Just run test.py on Ubuntu 18.04 and you can observe the crash. version of Python: Python3.11.0a4
Minimized repro:
import ctypes
import sys
py = ctypes.pythonapi
freefunc = ctypes.CFUNCTYPE(None, ctypes.c_voidp)
if sys.version_info[:3] >= (3, 12, 0):
RequestCodeExtraIndex = py.PyUnstable_Eval_RequestCodeExtraIndex
else:
RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex
RequestCodeExtraIndex.argtypes = (freefunc,)
RequestCodeExtraIndex.restype = ctypes.c_ssize_t
if sys.version_info[:3] >= (3, 12, 0):
SetExtra = py.PyUnstable_Code_SetExtra
else:
SetExtra = py._PyCode_SetExtra
SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp)
SetExtra.restype = ctypes.c_int
def myfree(ptr):
pass
FREE_FUNC = freefunc(myfree)
FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC)
def f():
pass
SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500))
del FREE_FUNC
del f
ASAN output:
==297351==ERROR: AddressSanitizer: heap-use-after-free on address 0x60f0000064e8 at pc 0x7f76832b2b13 bp 0x7ffe8c861e70 sp 0x7ffe8c861e68
READ of size 4 at 0x60f0000064e8 thread T0
#0 0x7f76832b2b12 in closure_fcn Modules/_ctypes/callbacks.c:296
#1 0x7f7683266b49 (/lib/x86_64-linux-gnu/libffi.so.7+0x6b49)
#2 0x7f7683266eaf (/lib/x86_64-linux-gnu/libffi.so.7+0x6eaf)
#3 0x55e251b0776d in code_dealloc Objects/codeobject.c:1691
#4 0x55e251b0776d in code_dealloc Objects/codeobject.c:1672
#5 0x55e251c145bb in Py_DECREF Include/object.h:700
#6 0x55e251c145bb in Py_XDECREF Include/object.h:796
#7 0x55e251c145bb in tupledealloc Objects/tupleobject.c:207
#8 0x55e251b07f14 in Py_DECREF Include/object.h:700
#9 0x55e251b07f14 in Py_XDECREF Include/object.h:796
#10 0x55e251b07f14 in code_dealloc Objects/codeobject.c:1701
#11 0x55e251b07f14 in code_dealloc Objects/codeobject.c:1672
#12 0x55e251ebd61b in Py_DECREF Include/object.h:700
#13 0x55e251ebd61b in run_mod Python/pythonrun.c:1746
#14 0x55e251ec3c2d in pyrun_file Python/pythonrun.c:1645
#15 0x55e251ec3c2d in _PyRun_SimpleFileObject Python/pythonrun.c:462
#16 0x55e251ec4557 in _PyRun_AnyFileObject Python/pythonrun.c:78
#17 0x55e251f27be4 in pymain_run_file_obj Modules/main.c:360
#18 0x55e251f27be4 in pymain_run_file Modules/main.c:379
#19 0x55e251f27be4 in pymain_run_python Modules/main.c:610
#20 0x55e251f294bf in Py_RunMain Modules/main.c:688
#21 0x55e251f294bf in pymain_main Modules/main.c:718
#22 0x55e251f294bf in Py_BytesMain Modules/main.c:742
#23 0x7f768637ed09 in __libc_start_main ../csu/libc-start.c:308
#24 0x55e2519d82c9 in _start (/home/radislav/projects/cpython/python+0x2ea2c9)
0x60f0000064e8 is located 88 bytes inside of 168-byte region [0x60f000006490,0x60f000006538)
freed by thread T0 here:
#0 0x7f7686749b6f in __interceptor_free ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:123
#1 0x7f76832b2c93 in CThunkObject_dealloc Modules/_ctypes/callbacks.c:62
#2 0x55e251ba329c in Py_DECREF Include/object.h:700
#3 0x55e251ba329c in Py_XDECREF Include/object.h:796
#4 0x55e251ba329c in free_keys_object Objects/dictobject.c:673
#5 0x55e251bb1752 in dictkeys_decref Objects/dictobject.c:333
#6 0x55e251bb1752 in dict_dealloc Objects/dictobject.c:2374
#7 0x7f76832a3f54 in Py_DECREF Include/object.h:700
#8 0x7f76832a3f54 in PyCData_clear Modules/_ctypes/_ctypes.c:2745
#9 0x7f76832a3f54 in PyCFuncPtr_clear Modules/_ctypes/_ctypes.c:4235
#10 0x7f76832a4081 in PyCFuncPtr_dealloc Modules/_ctypes/_ctypes.c:4241
#11 0x55e251c25a32 in subtype_dealloc Objects/typeobject.c:2073
#12 0x55e251ba48c7 in Py_DECREF Include/object.h:700
#13 0x55e251ba48c7 in delitem_common Objects/dictobject.c:1963
#14 0x55e251bb046c in _PyDict_DelItem_KnownHash Objects/dictobject.c:2008
#15 0x55e2519b7dc0 in _PyEval_EvalFrameDefault Python/generated_cases.c.h:1336
#16 0x55e251db39fe in _PyEval_EvalFrame Include/internal/pycore_ceval.h:88
#17 0x55e251db39fe in _PyEval_Vector Python/ceval.c:1581
#18 0x55e251db39fe in PyEval_EvalCode Python/ceval.c:544
#19 0x55e251ebd3e2 in run_eval_code_obj Python/pythonrun.c:1724
#20 0x55e251ebd59c in run_mod Python/pythonrun.c:1745
#21 0x55e251ec3c2d in pyrun_file Python/pythonrun.c:1645
#22 0x55e251ec3c2d in _PyRun_SimpleFileObject Python/pythonrun.c:462
#23 0x55e251ec4557 in _PyRun_AnyFileObject Python/pythonrun.c:78
#24 0x55e251f27be4 in pymain_run_file_obj Modules/main.c:360
#25 0x55e251f27be4 in pymain_run_file Modules/main.c:379
#26 0x55e251f27be4 in pymain_run_python Modules/main.c:610
#27 0x55e251f294bf in Py_RunMain Modules/main.c:688
#28 0x55e251f294bf in pymain_main Modules/main.c:718
#29 0x55e251f294bf in Py_BytesMain Modules/main.c:742
#30 0x7f768637ed09 in __libc_start_main ../csu/libc-start.c:308
previously allocated by thread T0 here:
#0 0x7f7686749e8f in __interceptor_malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:145
#1 0x55e251f30ada in gc_alloc Modules/gcmodule.c:2303
#2 0x55e251f30ada in _PyObject_GC_NewVar Modules/gcmodule.c:2337
#3 0x7f76832b2d96 in CThunkObject_new Modules/_ctypes/callbacks.c:311
#4 0x7f76832b2d96 in _ctypes_alloc_callback Modules/_ctypes/callbacks.c:344
#5 0x7f76832a4ed1 in PyCFuncPtr_new Modules/_ctypes/_ctypes.c:3748
#6 0x55e251c23bf4 in type_call Objects/typeobject.c:1663
#7 0x55e251af7e8b in _PyObject_MakeTpCall Objects/call.c:241
#8 0x55e2519ab8f1 in _PyEval_EvalFrameDefault Python/generated_cases.c.h:3545
#9 0x55e251db39fe in _PyEval_EvalFrame Include/internal/pycore_ceval.h:88
#10 0x55e251db39fe in _PyEval_Vector Python/ceval.c:1581
#11 0x55e251db39fe in PyEval_EvalCode Python/ceval.c:544
#12 0x55e251ebd3e2 in run_eval_code_obj Python/pythonrun.c:1724
#13 0x55e251ebd59c in run_mod Python/pythonrun.c:1745
#14 0x55e251ec3c2d in pyrun_file Python/pythonrun.c:1645
#15 0x55e251ec3c2d in _PyRun_SimpleFileObject Python/pythonrun.c:462
#16 0x55e251ec4557 in _PyRun_AnyFileObject Python/pythonrun.c:78
#17 0x55e251f27be4 in pymain_run_file_obj Modules/main.c:360
#18 0x55e251f27be4 in pymain_run_file Modules/main.c:379
#19 0x55e251f27be4 in pymain_run_python Modules/main.c:610
#20 0x55e251f294bf in Py_RunMain Modules/main.c:688
#21 0x55e251f294bf in pymain_main Modules/main.c:718
#22 0x55e251f294bf in Py_BytesMain Modules/main.c:742
#23 0x7f768637ed09 in __libc_start_main ../csu/libc-start.c:308
SUMMARY: AddressSanitizer: heap-use-after-free Modules/_ctypes/callbacks.c:296 in closure_fcn
Shadow bytes around the buggy address:
0x0c1e7fff8c40: fd fd fd fd fd fd fd fd fd fd fd fd fd fa fa fa
0x0c1e7fff8c50: fa fa fa fa fa fa 00 00 00 00 00 00 00 00 00 00
0x0c1e7fff8c60: 00 00 00 00 00 00 00 00 00 00 04 fa fa fa fa fa
0x0c1e7fff8c70: fa fa fa fa fd fd fd fd fd fd fd fd fd fd fd fd
0x0c1e7fff8c80: fd fd fd fd fd fd fd fd fd fa fa fa fa fa fa fa
=>0x0c1e7fff8c90: fa fa fd fd fd fd fd fd fd fd fd fd fd[fd]fd fd
0x0c1e7fff8ca0: fd fd fd fd fd fd fd fa fa fa fa fa fa fa fa fa
0x0c1e7fff8cb0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1e7fff8cc0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1e7fff8cd0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
0x0c1e7fff8ce0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
Shadow byte legend (one shadow byte represents 8 application bytes):
Addressable: 00
Partially addressable: 01 02 03 04 05 06 07
Heap left redzone: fa
Freed heap region: fd
Stack left redzone: f1
Stack mid redzone: f2
Stack right redzone: f3
Stack after return: f5
Stack use after scope: f8
Global redzone: f9
Global init order: f6
Poisoned by user: f7
Container overflow: fc
Array cookie: ac
Intra object redzone: bb
ASan internal: fe
Left alloca redzone: ca
Right alloca redzone: cb
Shadow gap: cc
==297351==ABORTING
If FREE_FUNC
and its internal structures (that are used to wrap freefunc
to make it callable at C level) get freed early, subsequent call to freefunc
wrapper as free_extra
in code_dealloc
leads to crash.
Not sure if this is a bug and, if so, how it should be fixed. It seems that Python-defined and CFuncPtr
-wrapped funcs shouldn't be used as callbacks that can be called after its corresponding CFuncPtr
object was deallocated.
Currently test.test_code.CoExtra
causes crash only if its code is slightly (and artificially) changed, but maybe it should be moved to _testinternalcapi
or whatever.
diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
index ca06a39f5d..c5d8193fdd 100644
--- a/Lib/test/test_code.py
+++ b/Lib/test/test_code.py
@@ -829,6 +829,7 @@ def test_get_set(self):
@threading_helper.requires_working_threading()
def test_free_different_thread(self):
+ global FREE_FUNC
# Freeing a code object on a different thread then
# where the co_extra was set should be safe.
f = self.get_func()
@@ -842,6 +843,7 @@ def run(self):
self.test.assertEqual(LAST_FREED, 500)
SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500))
+ del FREE_FUNC
tt = ThreadTest(f, self)
del f
tt.start()
Note: these values reflect the state of the issue at the time it was migrated and might not reflect the current state.
Show more details
GitHub fields: ```python assignee = None closed_at = None created_at =
labels = ['interpreter-core', 'type-crash', '3.11']
title = 'Mutithread leads to Illegal instruction crashing CPython'
updated_at =
user = 'https://bugs.python.org/xxm'
```
bugs.python.org fields:
```python
activity =
actor = 'xxm'
assignee = 'none'
closed = False
closed_date = None
closer = None
components = ['Interpreter Core']
creation =
creator = 'xxm'
dependencies = []
files = ['50455']
hgrepos = []
issue_num = 45860
keywords = []
message_count = 2.0
messages = ['406740', '411446']
nosy_count = 2.0
nosy_names = ['lukasz.langa', 'xxm']
pr_nums = []
priority = 'normal'
resolution = None
stage = None
status = 'open'
superseder = None
type = 'crash'
url = 'https://bugs.python.org/issue45860'
versions = ['Python 3.11']
```