Product: multiprocessing 2.6.2.1
Platform: Solaris 8 / Sparc
Python interpreter: python2.4
Compiler: gcc 2.95.2 (vintage, baby!)
What steps will reproduce the problem?
1. python2.4 setup.py build
2. $ ( cd build/lib.solaris-2.8-sun4u-2.4; /apps/pydev/hjoukl/bin/python2.4 -c
'import multiprocessing' )
Traceback (most recent call last):
File "<string>", line 1, in ?
File "multiprocessing/__init__.py", line 87, in ?
import _multiprocessing
ImportError: ld.so.1: /apps/pydev/hjoukl/bin/python2.4: fatal: relocation
error: file multiprocessing/_multiprocessing.so: symbol CMSG_LEN: referenced
symbol not found
3. Seems like Solaris 8 doesn't define these symbols. Using the recipe found
here
http://www.mail-archive.com/quixote-users@mems-exchange.org/msg01047.html
to patch multiprocessing.h:
$ /apps/pydev/bin/hg diff
diff -r 0f2ab22c260e Modules/_multiprocessing/multiprocessing.h
--- a/Modules/_multiprocessing/multiprocessing.h Thu Sep 02 12:25:35
2010 +0200
+++ b/Modules/_multiprocessing/multiprocessing.h Thu Sep 02 12:30:56
2010 +0200
@@ -52,6 +52,28 @@
# endif
#endif
+/*
+ * Support for Solaris 8 (what about other old Solarises?)
+ */
+#if defined (__SVR4) && defined (__sun)
+# include <sys/uio.h>
+# include <stddef.h>
+# ifndef OSSH_ALIGNBYTES
+# define OSSH_ALIGNBYTES (sizeof(int) - 1)
+# endif
+# ifndef __CMSG_ALIGN
+# define __CMSG_ALIGN(p) (((u_int)(p) + OSSH_ALIGNBYTES) &~ OSSH_ALIGNBYTES)
+# endif
+/* Length of the contents of a control message of length len */
+# ifndef CMSG_LEN
+# define CMSG_LEN(len) (__CMSG_ALIGN(sizeof(struct cmsghdr)) + (len))
+# endif
+/* Length of the space taken up by a padded control message of length len */
+# ifndef CMSG_SPACE
+# define CMSG_SPACE(len) (__CMSG_ALIGN(sizeof(struct cmsghdr)) +
__CMSG_ALIGN(len))
+# endif
+#endif
+
/*
* Make sure Py_ssize_t available
*/
4. Still doesn't work after a rebuild:
$ ( cd build/lib.solaris-2.8-sun4u-2.4; /apps/pydev/hjoukl/bin/python2.4 -c
'import multiprocessing' )
Traceback (most recent call last):
File "<string>", line 1, in ?
File "multiprocessing/__init__.py", line 87, in ?
import _multiprocessing
ImportError: ld.so.1: /apps/pydev/hjoukl/bin/python2.4: fatal: relocation
error: file multiprocessing/_multiprocessing.so: symbol sem_timedwait:
referenced symbol not found
5. Some research indicates that sem_timedwait isn't available on Solaris 8,
therefore patching setup.py:
$ /apps/pydev/bin/hg diff setup.py
diff -r 0f2ab22c260e setup.py
--- a/setup.py Thu Sep 02 12:25:35 2010 +0200
+++ b/setup.py Thu Sep 02 12:34:33 2010 +0200
@@ -65,6 +65,13 @@
HAVE_FD_TRANSFER=1,
)
libraries = []
+elif sys.platform.startswith('sunos5') and os.uname()[2] == '5.8':
+ macros = dict( # Solaris 8 (what about older solarises?)
+ HAVE_SEM_OPEN=1,
+ HAVE_SEM_TIMEDWAIT=0, # Not implemented
+ HAVE_FD_TRANSFER=1,
+ )
+ libraries = ['rt']
else: # Linux and other unices
macros = dict(
HAVE_SEM_OPEN=1,
6. Rebuild - works!
$ ( cd build/lib.solaris-2.8-sun4u-2.4; python2.4 -c 'import multiprocessing'
&& PYTHONPATH=`pwd` python2.4 multiprocessing/tests.py )
test_array (__main__.WithProcessesTestArray) ... ERROR
test_getobj_getlock_obj (__main__.WithProcessesTestArray) ... ERROR
test_rawarray (__main__.WithProcessesTestArray) ... ERROR
test_notify (__main__.WithProcessesTestCondition) ... ok
test_notify_all (__main__.WithProcessesTestCondition) ... ok
test_timeout (__main__.WithProcessesTestCondition) ... ok
test_connection (__main__.WithProcessesTestConnection) ... ok
test_duplex_false (__main__.WithProcessesTestConnection) ... ok
test_sendbytes (__main__.WithProcessesTestConnection) ... ok
test_spawn_close (__main__.WithProcessesTestConnection) ... ok
test_event (__main__.WithProcessesTestEvent) ... ok
test_finalize (__main__.WithProcessesTestFinalize) ... ok
test_heap (__main__.WithProcessesTestHeap) ... ok
test_import (__main__.WithProcessesTestImportStar) ... ERROR
test_listener_client (__main__.WithProcessesTestListenerClient) ... ok
test_lock (__main__.WithProcessesTestLock) ... ok
test_rlock (__main__.WithProcessesTestLock) ... ok
test_enable_logging (__main__.WithProcessesTestLogging) ... ok
test_level (__main__.WithProcessesTestLogging) ... ok
test_apply (__main__.WithProcessesTestPool) ... ok
test_async (__main__.WithProcessesTestPool) ... ok
test_async_timeout (__main__.WithProcessesTestPool) ... ok
test_imap (__main__.WithProcessesTestPool) ... ok
test_imap_unordered (__main__.WithProcessesTestPool) ... ok
test_make_pool (__main__.WithProcessesTestPool) ... ok
test_map (__main__.WithProcessesTestPool) ... ok
test_terminate (__main__.WithProcessesTestPool) ... ok
test_active_children (__main__.WithProcessesTestProcess) ... ok
test_cpu_count (__main__.WithProcessesTestProcess) ... ok
test_current (__main__.WithProcessesTestProcess) ... ok
test_process (__main__.WithProcessesTestProcess) ... ok
test_recursion (__main__.WithProcessesTestProcess) ... ok
test_terminate (__main__.WithProcessesTestProcess) ... ok
test_fork (__main__.WithProcessesTestQueue) ... ok
test_get (__main__.WithProcessesTestQueue) ... ok
test_put (__main__.WithProcessesTestQueue) ... ok
test_qsize (__main__.WithProcessesTestQueue) ... ok
test_task_done (__main__.WithProcessesTestQueue) ... ok
test_bounded_semaphore (__main__.WithProcessesTestSemaphore) ... ok
test_semaphore (__main__.WithProcessesTestSemaphore) ... ok
test_timeout (__main__.WithProcessesTestSemaphore) ... ok
test_copy (__main__.WithProcessesTestSharedCTypes) ... ok
test_sharedctypes (__main__.WithProcessesTestSharedCTypes) ... ok
test_synchronize (__main__.WithProcessesTestSharedCTypes) ... ok
test_subclassing (__main__.WithProcessesTestSubclassingProcess) ... ok
test_getobj_getlock (__main__.WithProcessesTestValue) ... ERROR
test_rawvalue (__main__.WithProcessesTestValue) ... ERROR
test_value (__main__.WithProcessesTestValue) ... ERROR
test_array (__main__.WithThreadsTestArray) ... ok
test_getobj_getlock_obj (__main__.WithThreadsTestArray) ... ok
test_rawarray (__main__.WithThreadsTestArray) ... ok
test_notify (__main__.WithThreadsTestCondition) ... ok
test_notify_all (__main__.WithThreadsTestCondition) ... ok
test_timeout (__main__.WithThreadsTestCondition) ... ok
test_connection (__main__.WithThreadsTestConnection) ... ok
test_duplex_false (__main__.WithThreadsTestConnection) ... ok
test_sendbytes (__main__.WithThreadsTestConnection) ... ok
test_spawn_close (__main__.WithThreadsTestConnection) ... ok
test_event (__main__.WithThreadsTestEvent) ... ok
test_listener_client (__main__.WithThreadsTestListenerClient) ... ok
test_lock (__main__.WithThreadsTestLock) ... ok
test_rlock (__main__.WithThreadsTestLock) ... ok
test_apply (__main__.WithThreadsTestPool) ... ok
test_async (__main__.WithThreadsTestPool) ... ok
test_async_timeout (__main__.WithThreadsTestPool) ... ok
test_imap (__main__.WithThreadsTestPool) ... ok
test_imap_unordered (__main__.WithThreadsTestPool) ... ok
test_make_pool (__main__.WithThreadsTestPool) ... ok
test_map (__main__.WithThreadsTestPool) ... ok
test_terminate (__main__.WithThreadsTestPool) ... ok
test_active_children (__main__.WithThreadsTestProcess) ... ok
test_cpu_count (__main__.WithThreadsTestProcess) ... ok
test_current (__main__.WithThreadsTestProcess) ... ok
test_process (__main__.WithThreadsTestProcess) ... ok
test_recursion (__main__.WithThreadsTestProcess) ... ok
test_terminate (__main__.WithThreadsTestProcess) ... ok
test_fork (__main__.WithThreadsTestQueue) ... ok
test_get (__main__.WithThreadsTestQueue) ... ok
test_put (__main__.WithThreadsTestQueue) ... ok
test_qsize (__main__.WithThreadsTestQueue) ... ok
test_task_done (__main__.WithThreadsTestQueue) ... ok
test_bounded_semaphore (__main__.WithThreadsTestSemaphore) ... ok
test_semaphore (__main__.WithThreadsTestSemaphore) ... ok
test_timeout (__main__.WithThreadsTestSemaphore) ... ok
test_getobj_getlock (__main__.WithThreadsTestValue) ... ok
test_rawvalue (__main__.WithThreadsTestValue) ... ok
test_value (__main__.WithThreadsTestValue) ... ok
test_array (__main__.WithManagerTestArray) ... ok
test_getobj_getlock_obj (__main__.WithManagerTestArray) ... ok
test_rawarray (__main__.WithManagerTestArray) ... ok
test_notify (__main__.WithManagerTestCondition) ... ok
test_notify_all (__main__.WithManagerTestCondition) ... ok
test_timeout (__main__.WithManagerTestCondition) ... ok
test_dict (__main__.WithManagerTestContainers) ... ok
test_list (__main__.WithManagerTestContainers) ... ok
test_namespace (__main__.WithManagerTestContainers) ... ok
test_event (__main__.WithManagerTestEvent) ... ok
test_lock (__main__.WithManagerTestLock) ... ok
test_rlock (__main__.WithManagerTestLock) ... ok
test_mymanager (__main__.WithManagerTestMyManager) ... ok
test_apply (__main__.WithManagerTestPool) ... ok
test_async (__main__.WithManagerTestPool) ... ok
test_async_timeout (__main__.WithManagerTestPool) ... ok
test_imap (__main__.WithManagerTestPool) ... ok
test_imap_unordered (__main__.WithManagerTestPool) ... ok
test_make_pool (__main__.WithManagerTestPool) ... ok
test_map (__main__.WithManagerTestPool) ... ok
test_terminate (__main__.WithManagerTestPool) ... ok
test_fork (__main__.WithManagerTestQueue) ... ok
test_get (__main__.WithManagerTestQueue) ... ok
test_put (__main__.WithManagerTestQueue) ... ok
test_qsize (__main__.WithManagerTestQueue) ... ok
test_task_done (__main__.WithManagerTestQueue) ... ok
test_remote (__main__.WithManagerTestRemoteManager) ... ok
test_bounded_semaphore (__main__.WithManagerTestSemaphore) ... ok
test_semaphore (__main__.WithManagerTestSemaphore) ... ok
test_timeout (__main__.WithManagerTestSemaphore) ... ok
test_getobj_getlock (__main__.WithManagerTestValue) ... ok
test_rawvalue (__main__.WithManagerTestValue) ... ok
test_value (__main__.WithManagerTestValue) ... ok
test_number_of_objects (__main__.WithManagerTestZZZNumberOfObjects) ... ok
test_answer_challenge_auth_failure (__main__.OtherTest) ... ok
test_deliver_challenge_auth_failure (__main__.OtherTest) ... ok
test_invalid_handles (__main__.TestInvalidHandle) ... ok
======================================================================
ERROR: test_array (__main__.WithProcessesTestArray)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 862, in test_array
arr = self.Array('i', seq)
File "multiprocessing/__init__.py", line 260, in Array
from multiprocessing.sharedctypes import Array
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
======================================================================
ERROR: test_getobj_getlock_obj (__main__.WithProcessesTestArray)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 887, in test_getobj_getlock_obj
arr1 = self.Array('i', range(10))
File "multiprocessing/__init__.py", line 260, in Array
from multiprocessing.sharedctypes import Array
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
======================================================================
ERROR: test_rawarray (__main__.WithProcessesTestArray)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 881, in test_rawarray
self.test_array(raw=True)
File "multiprocessing/tests.py", line 860, in test_array
arr = self.RawArray('i', seq)
File "multiprocessing/__init__.py", line 246, in RawArray
from multiprocessing.sharedctypes import RawArray
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
======================================================================
ERROR: test_import (__main__.WithProcessesTestImportStar)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 1641, in test_import
__import__(name)
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
======================================================================
ERROR: test_getobj_getlock (__main__.WithProcessesTestValue)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 823, in test_getobj_getlock
val1 = self.Value('i', 5)
File "multiprocessing/__init__.py", line 253, in Value
from multiprocessing.sharedctypes import Value
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
======================================================================
ERROR: test_rawvalue (__main__.WithProcessesTestValue)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 817, in test_rawvalue
self.test_value(raw=True)
File "multiprocessing/tests.py", line 800, in test_value
values = [self.RawValue(code, value)
File "multiprocessing/__init__.py", line 239, in RawValue
from multiprocessing.sharedctypes import RawValue
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
======================================================================
ERROR: test_value (__main__.WithProcessesTestValue)
----------------------------------------------------------------------
Traceback (most recent call last):
File "multiprocessing/tests.py", line 803, in test_value
values = [self.Value(code, value)
File "multiprocessing/__init__.py", line 253, in Value
from multiprocessing.sharedctypes import Value
File "/ae/data/pydev/DOWNLOADS/multiprocessing-2.6.2.1/build/lib.solaris-2.8-sun4u-2.4/multiprocessing/sharedctypes.py", line 10, in ?
import ctypes
ImportError: No module named ctypes
----------------------------------------------------------------------
Ran 124 tests in 13.675s
FAILED (errors=7)
As far as I can see all the errors seem to stem from my Python 2.4 installation
naturally not having ctypes, which was only added in Python 2.5.
I haven't tried building a more recent Python that includes the multiprocessing
package in the stdlib but I suspect this will show similar problems for Solaris
8 / Sparc.
This issue seems to go beyond the fix for Solaris 8 multiprocessing problems
presented here: http://bugs.python.org/issue3110
Disclaimer: I haven't actually *used* the runnable multiprocessing build for
anything real, apart from running the included test suite.
Patchfile attached.
Best regards,
Holger
Original issue reported on code.google.com by holger.j...@web.de on 2 Sep 2010 at 10:52
Original issue reported on code.google.com by
holger.j...@web.deon 2 Sep 2010 at 10:52Attachments: