95a73db4-c9d6-48dc-ad4a-f3520ef9af03
commented
7 years ago shutil.copytree() always calls copystat() for symlinks. Copying symlink to another volume fails if some attributes cannot be set. And there is no alternative to bypass this as it can be done for files and directories using copy_function. By default copy_function equals copy2() which calls copystat(). There is also copy() available which calls copymode().
From the copytree() source code one can see that for symlink copystat() is called in any case:
if os.path.islink(srcname):
linkto = os.readlink(srcname)
if symlinks:
# We can't just leave it to `copy_function` because legacy
# code with a custom `copy_function` may rely on copytree
# doing the right thing.
os.symlink(linkto, dstname)
copystat(srcname, dstname, follow_symlinks=not symlinks)An example is running inside a docker container with a zfs based volume mounted at /root/.cdist. Root filesytem with /tmp is also on zfs based container root-volume.
And copying files with shutil.move which in this case uses copytree results in the following error:
Traceback (most recent call last):
File "/appenv/lib/python3.5/shutil.py", line 538, in move
os.rename(src, real_dst)
OSError: [Errno 18] Cross-device link: '/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data' -> '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/appenv/bin/cdist", line 94, in <module>
commandline()
File "/appenv/bin/cdist", line 66, in commandline
args.func(args)
File "/appenv/lib/python3.5/site-packages/cdist/config.py", line 157, in commandline
args, parallel=False)
File "/appenv/lib/python3.5/site-packages/cdist/config.py", line 227, in onehost
c.run()
File "/appenv/lib/python3.5/site-packages/cdist/config.py", line 255, in run
self.local.save_cache()
File "/appenv/lib/python3.5/site-packages/cdist/exec/local.py", line 265, in save_cache
shutil.move(self.base_path, destination)
File "/appenv/lib/python3.5/shutil.py", line 549, in move
symlinks=True)
File "/appenv/lib/python3.5/shutil.py", line 353, in copytree
raise Error(errors)
shutil.Error: [('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/network', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/network', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/network'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/os', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/os', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/os'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/machine', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/machine', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/machine'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/init-system', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/init-system', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/init-system'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/interfaces', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/interfaces', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/interfaces'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/os_version', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/os_version', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/os_version'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/cpu_cores', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/cpu_cores', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/cpu_cores'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/lsb_description', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/lsb_description', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/lsb_description'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/machine_type', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/machine_type', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/machine_type'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/efi', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/efi', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/efi'"), ('/tmp/tmpuxb_jr3y/936a8745479046ce91a00ee3013fc9b8/data/conf/explorer/lsb_id', '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/lsb_id', "[Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/explorer/lsb_id'"),
...Using copy_function=copy instead of default copy2 does not solve the problem since for symlinks copytree() always uses copystat().
Concrete example which mimics copytree() code for symlink:
(appenv) ~ # rm /root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/type/__hpc_syslog
(appenv) ~ # python
Python 3.5.2 (default, Dec 20 2016, 17:58:45)
[GCC 5.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> src='/tmp/tmpoiwfrgvk/936a8745479046ce91a00ee3013fc9b8/data/conf/type/__hpc_syslog'
>>> dst='/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/type/__hpc_syslog'
>>> linkto = os.readlink(src)
>>> os.symlink(linkto, dst)
>>> os.stat(dst)
os.stat_result(st_mode=16877, st_ino=955, st_dev=71, st_nlink=4, st_uid=65534, st_gid=65534, st_size=8, st_atime=1486540058, st_mtime=1485953153, st_ctime=1485953153)
>>> import shutil
>>> shutil.copystat(src, dst, follow_symlinks=False)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/appenv/lib/python3.5/shutil.py", line 197, in copystat
lookup("chmod")(dst, mode, follow_symlinks=follow)
OSError: [Errno 95] Not supported: '/root/.cdist/cache/936a8745479046ce91a00ee3013fc9b8/conf/type/__hpc_syslog'
>>>
Note: these values reflect the state of the issue at the time it was migrated and might not reflect the current state.
Show more details
GitHub fields: ```python assignee = None closed_at = None created_at =
labels = ['3.7', 'type-bug', 'library']
title = 'shutil.copytree(symlinks=True) fails when copying symlinks cross-device and there is no alternative'
updated_at =
user = 'https://github.com/darko-poljak'
```
bugs.python.org fields:
```python
activity =
actor = 'serhiy.storchaka'
assignee = 'none'
closed = False
closed_date = None
closer = None
components = ['Library (Lib)']
creation =
creator = 'poljakowski'
dependencies = []
files = []
hgrepos = []
issue_num = 29516
keywords = []
message_count = 1.0
messages = ['287455']
nosy_count = 1.0
nosy_names = ['poljakowski']
pr_nums = []
priority = 'normal'
resolution = None
stage = None
status = 'open'
superseder = None
type = 'behavior'
url = 'https://bugs.python.org/issue29516'
versions = ['Python 3.3', 'Python 3.4', 'Python 3.5', 'Python 3.6', 'Python 3.7']
```