Open giampaolo opened 5 years ago
import multiprocessing
import multiprocessing.managers
def f(n):
return n * n
def worker(pool):
with pool:
pool.apply_async(f, (10, ))
manager = multiprocessing.managers.SyncManager()
manager.start()
pool = manager.Pool(processes=4)
proc = multiprocessing.Process(target=worker, args=(pool, ))
proc.start()
proc.join()
This is related to BPO-35917 and it fails with:
Process Process-2:
Traceback (most recent call last):
File "/home/giampaolo/cpython/Lib/multiprocessing/process.py", line 302, in _bootstrap
self.run()
File "/home/giampaolo/cpython/Lib/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "foo.py", line 54, in worker
pool.apply_async(f, (10, ))
File "<string>", line 2, in apply_async
File "/home/giampaolo/cpython/Lib/multiprocessing/managers.py", line 802, in _callmethod
proxytype = self._manager._registry[token.typeid][-1]
AttributeError: 'NoneType' object has no attribute '_registry'
import traceback
import multiprocessing.managers
class MyManager(multiprocessing.managers.SyncManager):
pass
class DictList(multiprocessing.managers.BaseProxy):
_method_to_typeid_ = {'__getitem__': 'dict'}
def __getitem__(self, key):
return self._callmethod('__getitem__', (key,))
MyManager.register('DictList', None, DictList)
with MyManager() as manager:
nested = manager.DictList([{'hello': 'world'}])
print(nested[0]['hello']) # world
proxy = manager.list([nested])
try:
print(proxy[0][0]['hello'])
except AttributeError:
traceback.print_exc()
print("""
Bug: AttributeError: ProxyBase._callmethod is None
This error is raised because proxies returned as #RETURN messages
have no manager reference, which is required to resolve typeids
(using BaseManager._registry).
Only proxies returned as #PROXY messages will be fully functional.
This is an undocumented current implementation limitation.
Fix (proposal)
\--------------
Include the manager class (not the instance) as part of the proxy
serialization in BaseProxy.__reduce__, as BaseManager._registry is
a class variable.
Note: #PROXY message protocol can be also replaced by regular proxy
serialization after this fix, resulting on simpler codebase.
""")
has this problem been solved? I am still seeing this in 3.9.18
Note: these values reflect the state of the issue at the time it was migrated and might not reflect the current state.
Show more details
GitHub fields: ```python assignee = None closed_at = None created_at =
labels = ['3.7', '3.8', 'type-bug', 'library', '3.9']
title = 'multiprocessing: shared manager Pool fails with AttributeError'
updated_at =
user = 'https://github.com/giampaolo'
```
bugs.python.org fields:
```python
activity =
actor = 'Felipe A. Hernandez'
assignee = 'none'
closed = False
closed_date = None
closer = None
components = ['Library (Lib)']
creation =
creator = 'giampaolo.rodola'
dependencies = []
files = []
hgrepos = []
issue_num = 35919
keywords = []
message_count = 2.0
messages = ['334962', '373427']
nosy_count = 4.0
nosy_names = ['pitrou', 'giampaolo.rodola', 'davin', 'Felipe A. Hernandez']
pr_nums = []
priority = 'normal'
resolution = None
stage = None
status = 'open'
superseder = None
type = 'behavior'
url = 'https://bugs.python.org/issue35919'
versions = ['Python 3.5', 'Python 3.6', 'Python 3.7', 'Python 3.8', 'Python 3.9']
```