python / cpython

The Python programming language
https://www.python.org
Other
63.89k stars 30.58k forks source link

Mutithread leads to Illegal instruction crashing CPython #90018

Open 2c45e5bd-3f35-4c26-b24b-b484594f2279 opened 3 years ago

2c45e5bd-3f35-4c26-b24b-b484594f2279 commented 3 years ago
BPO 45860
Nosy @ambv
Files
  • test_code.py
  • Note: these values reflect the state of the issue at the time it was migrated and might not reflect the current state.

    Show more details

    GitHub fields: ```python assignee = None closed_at = None created_at = labels = ['interpreter-core', 'type-crash', '3.11'] title = 'Mutithread leads to Illegal instruction crashing CPython' updated_at = user = 'https://bugs.python.org/xxm' ``` bugs.python.org fields: ```python activity = actor = 'xxm' assignee = 'none' closed = False closed_date = None closer = None components = ['Interpreter Core'] creation = creator = 'xxm' dependencies = [] files = ['50455'] hgrepos = [] issue_num = 45860 keywords = [] message_count = 2.0 messages = ['406740', '411446'] nosy_count = 2.0 nosy_names = ['lukasz.langa', 'xxm'] pr_nums = [] priority = 'normal' resolution = None stage = None status = 'open' superseder = None type = 'crash' url = 'https://bugs.python.org/issue45860' versions = ['Python 3.11'] ```

    2c45e5bd-3f35-4c26-b24b-b484594f2279 commented 3 years ago

    The following code can lead to a crash and report Illegal instruction (core dumped)(few times) or Trace/breakpoint trap (core dumped) (very few times) or Segmentation fault (core dumped) (most times) on Python 3.11.

    test_free_different_thread.py \========================================================

    import inspect
    import sys
    import threading
    import unittest
    import weakref
    import ctypes
    from test.support import run_doctest, run_unittest, cpython_only, check_impl_detail
    import _testcapi
    from types import FunctionType
    from test import test_code
    import test_code
    
    def test_free_different_thread():
        f = CoExtra.get_func()
    
        class ThreadTest(threading.Thread):
    
            def __init__(CoExtra, f, test):
                super().__init__()
                CoExtra.f = CoExtra
                CoExtra.test = test
    
            def run(CoExtra):
                del CoExtra.f
                CoExtra.test.assertEqual(test_code.LAST_FREED, 500)
        test_code.SetExtra(f.__code__, test_code.FREE_INDEX, ctypes.c_voidp(500))
        f = ThreadTest(CoExtra, f)
        del tt
        tt.start()
        tt.join()
        CoExtra.assertEqual(test_code.LAST_FREED, 500)
    
    CoExtra = test_code.CoExtra()
    test_free_different_thread()
    =========================================================
    

    Traceback (most recent call last):
      File "/home/xxm/Desktop/test_free_different_thread.py", line 33, in <module>
        test_free_different_thread()
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/home/xxm/Desktop/test_free_different_thread.py", line 28, in test_free_different_thread
        del tt
            ^^
    UnboundLocalError: cannot access local variable 'tt' where it is not associated with a value
    Illegal instruction (core dumped)/Trace/breakpoint trap (core dumped)/Segmentation fault (core dumped)

    Version: python 3.9, python 3.10, python 3.11 on ubuntu 16.04

    Reproduce step: 1.download test_code.py and place test_free_different_thread.py and test_code in a same directory.

    1. run with "python test_free_different_thread.py"

    The test_code.py is from cpython' test. We can also annotate "import test_code" and run test_free_different_thread.py directly. But it seems that Illegal instruction and Trace/breakpoint trap cannot be reproduced.

    2c45e5bd-3f35-4c26-b24b-b484594f2279 commented 2 years ago

    A simpler example to reproduce it: test.py \========================================

    import threading
    import ctypes
    from test import test_code
    
    def test_free_different_thread():
        f = test_code.CoExtra().get_func()
    
        class ThreadTest(threading.Thread):
            def __init__(CoExtra, f, test):
                f.test = f
    
        test_code.SetExtra(f.__code__, test_code.FREE_INDEX, ctypes.c_voidp(500))
        tt = ThreadTest(f, f)
        test_code.CoExtra().assertEqual(test_code.LAST_FREED, 500)
    
    test_free_different_thread()

    ========================================

    Just run test.py on Ubuntu 18.04 and you can observe the crash. version of Python: Python3.11.0a4

    chgnrdv commented 1 year ago

    Minimized repro:

    import ctypes
    import sys
    
    py = ctypes.pythonapi
    freefunc = ctypes.CFUNCTYPE(None, ctypes.c_voidp)
    
    if sys.version_info[:3] >= (3, 12, 0):
        RequestCodeExtraIndex = py.PyUnstable_Eval_RequestCodeExtraIndex
    else:
        RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex
    RequestCodeExtraIndex.argtypes = (freefunc,)
    RequestCodeExtraIndex.restype = ctypes.c_ssize_t
    
    if sys.version_info[:3] >= (3, 12, 0):
        SetExtra = py.PyUnstable_Code_SetExtra
    else:
        SetExtra = py._PyCode_SetExtra
    SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp)
    SetExtra.restype = ctypes.c_int
    
    def myfree(ptr):
        pass
    
    FREE_FUNC = freefunc(myfree)
    FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC)
    
    def f():
        pass
    
    SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500))
    del FREE_FUNC
    del f

    ASAN output:

    ==297351==ERROR: AddressSanitizer: heap-use-after-free on address 0x60f0000064e8 at pc 0x7f76832b2b13 bp 0x7ffe8c861e70 sp 0x7ffe8c861e68
    READ of size 4 at 0x60f0000064e8 thread T0
        #0 0x7f76832b2b12 in closure_fcn Modules/_ctypes/callbacks.c:296
        #1 0x7f7683266b49  (/lib/x86_64-linux-gnu/libffi.so.7+0x6b49)
        #2 0x7f7683266eaf  (/lib/x86_64-linux-gnu/libffi.so.7+0x6eaf)
        #3 0x55e251b0776d in code_dealloc Objects/codeobject.c:1691
        #4 0x55e251b0776d in code_dealloc Objects/codeobject.c:1672
        #5 0x55e251c145bb in Py_DECREF Include/object.h:700
        #6 0x55e251c145bb in Py_XDECREF Include/object.h:796
        #7 0x55e251c145bb in tupledealloc Objects/tupleobject.c:207
        #8 0x55e251b07f14 in Py_DECREF Include/object.h:700
        #9 0x55e251b07f14 in Py_XDECREF Include/object.h:796
        #10 0x55e251b07f14 in code_dealloc Objects/codeobject.c:1701
        #11 0x55e251b07f14 in code_dealloc Objects/codeobject.c:1672
        #12 0x55e251ebd61b in Py_DECREF Include/object.h:700
        #13 0x55e251ebd61b in run_mod Python/pythonrun.c:1746
        #14 0x55e251ec3c2d in pyrun_file Python/pythonrun.c:1645
        #15 0x55e251ec3c2d in _PyRun_SimpleFileObject Python/pythonrun.c:462
        #16 0x55e251ec4557 in _PyRun_AnyFileObject Python/pythonrun.c:78
        #17 0x55e251f27be4 in pymain_run_file_obj Modules/main.c:360
        #18 0x55e251f27be4 in pymain_run_file Modules/main.c:379
        #19 0x55e251f27be4 in pymain_run_python Modules/main.c:610
        #20 0x55e251f294bf in Py_RunMain Modules/main.c:688
        #21 0x55e251f294bf in pymain_main Modules/main.c:718
        #22 0x55e251f294bf in Py_BytesMain Modules/main.c:742
        #23 0x7f768637ed09 in __libc_start_main ../csu/libc-start.c:308
        #24 0x55e2519d82c9 in _start (/home/radislav/projects/cpython/python+0x2ea2c9)
    
    0x60f0000064e8 is located 88 bytes inside of 168-byte region [0x60f000006490,0x60f000006538)
    freed by thread T0 here:
        #0 0x7f7686749b6f in __interceptor_free ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:123
        #1 0x7f76832b2c93 in CThunkObject_dealloc Modules/_ctypes/callbacks.c:62
        #2 0x55e251ba329c in Py_DECREF Include/object.h:700
        #3 0x55e251ba329c in Py_XDECREF Include/object.h:796
        #4 0x55e251ba329c in free_keys_object Objects/dictobject.c:673
        #5 0x55e251bb1752 in dictkeys_decref Objects/dictobject.c:333
        #6 0x55e251bb1752 in dict_dealloc Objects/dictobject.c:2374
        #7 0x7f76832a3f54 in Py_DECREF Include/object.h:700
        #8 0x7f76832a3f54 in PyCData_clear Modules/_ctypes/_ctypes.c:2745
        #9 0x7f76832a3f54 in PyCFuncPtr_clear Modules/_ctypes/_ctypes.c:4235
        #10 0x7f76832a4081 in PyCFuncPtr_dealloc Modules/_ctypes/_ctypes.c:4241
        #11 0x55e251c25a32 in subtype_dealloc Objects/typeobject.c:2073
        #12 0x55e251ba48c7 in Py_DECREF Include/object.h:700
        #13 0x55e251ba48c7 in delitem_common Objects/dictobject.c:1963
        #14 0x55e251bb046c in _PyDict_DelItem_KnownHash Objects/dictobject.c:2008
        #15 0x55e2519b7dc0 in _PyEval_EvalFrameDefault Python/generated_cases.c.h:1336
        #16 0x55e251db39fe in _PyEval_EvalFrame Include/internal/pycore_ceval.h:88
        #17 0x55e251db39fe in _PyEval_Vector Python/ceval.c:1581
        #18 0x55e251db39fe in PyEval_EvalCode Python/ceval.c:544
        #19 0x55e251ebd3e2 in run_eval_code_obj Python/pythonrun.c:1724
        #20 0x55e251ebd59c in run_mod Python/pythonrun.c:1745
        #21 0x55e251ec3c2d in pyrun_file Python/pythonrun.c:1645
        #22 0x55e251ec3c2d in _PyRun_SimpleFileObject Python/pythonrun.c:462
        #23 0x55e251ec4557 in _PyRun_AnyFileObject Python/pythonrun.c:78
        #24 0x55e251f27be4 in pymain_run_file_obj Modules/main.c:360
        #25 0x55e251f27be4 in pymain_run_file Modules/main.c:379
        #26 0x55e251f27be4 in pymain_run_python Modules/main.c:610
        #27 0x55e251f294bf in Py_RunMain Modules/main.c:688
        #28 0x55e251f294bf in pymain_main Modules/main.c:718
        #29 0x55e251f294bf in Py_BytesMain Modules/main.c:742
        #30 0x7f768637ed09 in __libc_start_main ../csu/libc-start.c:308
    
    previously allocated by thread T0 here:
        #0 0x7f7686749e8f in __interceptor_malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:145
        #1 0x55e251f30ada in gc_alloc Modules/gcmodule.c:2303
        #2 0x55e251f30ada in _PyObject_GC_NewVar Modules/gcmodule.c:2337
        #3 0x7f76832b2d96 in CThunkObject_new Modules/_ctypes/callbacks.c:311
        #4 0x7f76832b2d96 in _ctypes_alloc_callback Modules/_ctypes/callbacks.c:344
        #5 0x7f76832a4ed1 in PyCFuncPtr_new Modules/_ctypes/_ctypes.c:3748
        #6 0x55e251c23bf4 in type_call Objects/typeobject.c:1663
        #7 0x55e251af7e8b in _PyObject_MakeTpCall Objects/call.c:241
        #8 0x55e2519ab8f1 in _PyEval_EvalFrameDefault Python/generated_cases.c.h:3545
        #9 0x55e251db39fe in _PyEval_EvalFrame Include/internal/pycore_ceval.h:88
        #10 0x55e251db39fe in _PyEval_Vector Python/ceval.c:1581
        #11 0x55e251db39fe in PyEval_EvalCode Python/ceval.c:544
        #12 0x55e251ebd3e2 in run_eval_code_obj Python/pythonrun.c:1724
        #13 0x55e251ebd59c in run_mod Python/pythonrun.c:1745
        #14 0x55e251ec3c2d in pyrun_file Python/pythonrun.c:1645
        #15 0x55e251ec3c2d in _PyRun_SimpleFileObject Python/pythonrun.c:462
        #16 0x55e251ec4557 in _PyRun_AnyFileObject Python/pythonrun.c:78
        #17 0x55e251f27be4 in pymain_run_file_obj Modules/main.c:360
        #18 0x55e251f27be4 in pymain_run_file Modules/main.c:379
        #19 0x55e251f27be4 in pymain_run_python Modules/main.c:610
        #20 0x55e251f294bf in Py_RunMain Modules/main.c:688
        #21 0x55e251f294bf in pymain_main Modules/main.c:718
        #22 0x55e251f294bf in Py_BytesMain Modules/main.c:742
        #23 0x7f768637ed09 in __libc_start_main ../csu/libc-start.c:308
    
    SUMMARY: AddressSanitizer: heap-use-after-free Modules/_ctypes/callbacks.c:296 in closure_fcn
    Shadow bytes around the buggy address:
      0x0c1e7fff8c40: fd fd fd fd fd fd fd fd fd fd fd fd fd fa fa fa
      0x0c1e7fff8c50: fa fa fa fa fa fa 00 00 00 00 00 00 00 00 00 00
      0x0c1e7fff8c60: 00 00 00 00 00 00 00 00 00 00 04 fa fa fa fa fa
      0x0c1e7fff8c70: fa fa fa fa fd fd fd fd fd fd fd fd fd fd fd fd
      0x0c1e7fff8c80: fd fd fd fd fd fd fd fd fd fa fa fa fa fa fa fa
    =>0x0c1e7fff8c90: fa fa fd fd fd fd fd fd fd fd fd fd fd[fd]fd fd
      0x0c1e7fff8ca0: fd fd fd fd fd fd fd fa fa fa fa fa fa fa fa fa
      0x0c1e7fff8cb0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      0x0c1e7fff8cc0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      0x0c1e7fff8cd0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      0x0c1e7fff8ce0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
    Shadow byte legend (one shadow byte represents 8 application bytes):
      Addressable:           00
      Partially addressable: 01 02 03 04 05 06 07 
      Heap left redzone:       fa
      Freed heap region:       fd
      Stack left redzone:      f1
      Stack mid redzone:       f2
      Stack right redzone:     f3
      Stack after return:      f5
      Stack use after scope:   f8
      Global redzone:          f9
      Global init order:       f6
      Poisoned by user:        f7
      Container overflow:      fc
      Array cookie:            ac
      Intra object redzone:    bb
      ASan internal:           fe
      Left alloca redzone:     ca
      Right alloca redzone:    cb
      Shadow gap:              cc
    ==297351==ABORTING
    

    If FREE_FUNC and its internal structures (that are used to wrap freefunc to make it callable at C level) get freed early, subsequent call to freefunc wrapper as free_extra in code_dealloc leads to crash.

    Not sure if this is a bug and, if so, how it should be fixed. It seems that Python-defined and CFuncPtr-wrapped funcs shouldn't be used as callbacks that can be called after its corresponding CFuncPtr object was deallocated. Currently test.test_code.CoExtra causes crash only if its code is slightly (and artificially) changed, but maybe it should be moved to _testinternalcapi or whatever.

    diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
    index ca06a39f5d..c5d8193fdd 100644
    --- a/Lib/test/test_code.py
    +++ b/Lib/test/test_code.py
    @@ -829,6 +829,7 @@ def test_get_set(self):
    
             @threading_helper.requires_working_threading()
             def test_free_different_thread(self):
    +            global FREE_FUNC
                 # Freeing a code object on a different thread then
                 # where the co_extra was set should be safe.
                 f = self.get_func()
    @@ -842,6 +843,7 @@ def run(self):
                         self.test.assertEqual(LAST_FREED, 500)
    
                 SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500))
    +            del FREE_FUNC
                 tt = ThreadTest(f, self)
                 del f
                 tt.start()

    83539 looks like a related issue.