Python3.11 test failures

[Nowa-Ammerlaan]

Testing with python 3.11, I am getting some test failures:

================================================== FAILURES ==================================================
_______________________________________ TestSetupReadline.test_import ________________________________________

self = <test.test_utils.TestSetupReadline testMethod=test_import>

def test_import(self):
s = 'from os.path import a'
assert set(self.complete(s)) == {s + 'ltsep', s + 'bspath'}
assert self.complete('import keyword') == ['import keyword']

import os
s = 'from os import '
goal = {s + el for el in dir(os)}
# There are minor differences, e.g. the dir doesn't include deleted
# items as well as items that are not only available on linux.
difference = set(self.complete(s)).symmetric_difference(goal)
difference = {
x for x in difference
if all(not x.startswith('from os import ' + s)
for s in ['_', 'O_', 'EX_', 'MFD_', 'SF_', 'ST_',
'CLD_', 'POSIX_SPAWN_', 'P_', 'RWF_',
'SCHED_'])
}
# There are quite a few differences, because both Windows and Linux
# (posix and nt) librariesare included.
>       assert len(difference) < 30
E       AssertionError: assert 37 < 30
E        +  where 37 = len({'from os import EFD_CLOEXEC', 'from os import EFD_NONBLOCK', 'from os import EFD_SEMAPHORE', 'from os import GRND_NONBLOCK', 'from os import GRND_RANDOM', 'from os import GenericAlias', ...})

difference = {'from os import EFD_CLOEXEC',
'from os import EFD_NONBLOCK',
'from os import EFD_SEMAPHORE',
'from os import GRND_NONBLOCK',
'from os import GRND_RANDOM',
'from os import GenericAlias',
'from os import Mapping',
'from os import MutableMapping',
'from os import NGROUPS_MAX',
'from os import SPLICE_F_MORE',
'from os import SPLICE_F_MOVE',
'from os import SPLICE_F_NONBLOCK',
'from os import abc',
'from os import add_dll_directory',
'from os import chflags',
'from os import copy_file_range',
'from os import eventfd',
'from os import eventfd_read',
'from os import eventfd_write',
'from os import lchflags',
'from os import lchmod',
'from os import login_tty',
'from os import pidfd_open',
'from os import plock',
'from os import posix_spawn',
'from os import posix_spawnp',
'from os import preadv',
'from os import pwritev',
'from os import sched_param',
'from os import splice',
'from os import st',
'from os import startfile',
'from os import sys',
'from os import times_result',
'from os import uname_result',
'from os import waitid_result',
'from os import waitstatus_to_exitcode'}
goal       = {'from os import CLD_CONTINUED',
'from os import CLD_DUMPED',
'from os import CLD_EXITED',
'from os import CLD_KILLED',
'from os import CLD_STOPPED',
'from os import CLD_TRAPPED',
'from os import DirEntry',
'from os import EFD_CLOEXEC',
'from os import EFD_NONBLOCK',
'from os import EFD_SEMAPHORE',
'from os import EX_CANTCREAT',
'from os import EX_CONFIG',
'from os import EX_DATAERR',
'from os import EX_IOERR',
'from os import EX_NOHOST',
'from os import EX_NOINPUT',
'from os import EX_NOPERM',
'from os import EX_NOUSER',
'from os import EX_OK',
'from os import EX_OSERR',
'from os import EX_OSFILE',
'from os import EX_PROTOCOL',
'from os import EX_SOFTWARE',
'from os import EX_TEMPFAIL',
'from os import EX_UNAVAILABLE',
'from os import EX_USAGE',
'from os import F_LOCK',
'from os import F_OK',
'from os import F_TEST',
'from os import F_TLOCK',
'from os import F_ULOCK',
'from os import GRND_NONBLOCK',
'from os import GRND_RANDOM',
'from os import GenericAlias',
'from os import MFD_ALLOW_SEALING',
'from os import MFD_CLOEXEC',
'from os import MFD_HUGETLB',
'from os import MFD_HUGE_16GB',
'from os import MFD_HUGE_16MB',
'from os import MFD_HUGE_1GB',
'from os import MFD_HUGE_1MB',
'from os import MFD_HUGE_256MB',
'from os import MFD_HUGE_2GB',
'from os import MFD_HUGE_2MB',
'from os import MFD_HUGE_32MB',
'from os import MFD_HUGE_512KB',
'from os import MFD_HUGE_512MB',
'from os import MFD_HUGE_64KB',
'from os import MFD_HUGE_8MB',
'from os import MFD_HUGE_MASK',
'from os import MFD_HUGE_SHIFT',
'from os import Mapping',
'from os import MutableMapping',
'from os import NGROUPS_MAX',
'from os import O_ACCMODE',
'from os import O_APPEND',
'from os import O_ASYNC',
'from os import O_CLOEXEC',
'from os import O_CREAT',
'from os import O_DIRECT',
'from os import O_DIRECTORY',
'from os import O_DSYNC',
'from os import O_EXCL',
'from os import O_FSYNC',
'from os import O_LARGEFILE',
'from os import O_NDELAY',
'from os import O_NOATIME',
'from os import O_NOCTTY',
'from os import O_NOFOLLOW',
'from os import O_NONBLOCK',
'from os import O_PATH',
'from os import O_RDONLY',
'from os import O_RDWR',
'from os import O_RSYNC',
'from os import O_SYNC',
'from os import O_TMPFILE',
'from os import O_TRUNC',
'from os import O_WRONLY',
'from os import POSIX_FADV_DONTNEED',
'from os import POSIX_FADV_NOREUSE',
'from os import POSIX_FADV_NORMAL',
'from os import POSIX_FADV_RANDOM',
'from os import POSIX_FADV_SEQUENTIAL',
'from os import POSIX_FADV_WILLNEED',
'from os import POSIX_SPAWN_CLOSE',
'from os import POSIX_SPAWN_DUP2',
'from os import POSIX_SPAWN_OPEN',
'from os import PRIO_PGRP',
'from os import PRIO_PROCESS',
'from os import PRIO_USER',
'from os import P_ALL',
'from os import P_NOWAIT',
'from os import P_NOWAITO',
'from os import P_PGID',
'from os import P_PID',
'from os import P_PIDFD',
'from os import P_WAIT',
'from os import PathLike',
'from os import RTLD_DEEPBIND',
'from os import RTLD_GLOBAL',
'from os import RTLD_LAZY',
'from os import RTLD_LOCAL',
'from os import RTLD_NODELETE',
'from os import RTLD_NOLOAD',
'from os import RTLD_NOW',
'from os import RWF_APPEND',
'from os import RWF_DSYNC',
'from os import RWF_HIPRI',
'from os import RWF_NOWAIT',
'from os import RWF_SYNC',
'from os import R_OK',
'from os import SCHED_BATCH',
'from os import SCHED_FIFO',
'from os import SCHED_IDLE',
'from os import SCHED_OTHER',
'from os import SCHED_RESET_ON_FORK',
'from os import SCHED_RR',
'from os import SEEK_CUR',
'from os import SEEK_DATA',
'from os import SEEK_END',
'from os import SEEK_HOLE',
'from os import SEEK_SET',
'from os import SPLICE_F_MORE',
'from os import SPLICE_F_MOVE',
'from os import SPLICE_F_NONBLOCK',
'from os import ST_APPEND',
'from os import ST_MANDLOCK',
'from os import ST_NOATIME',
'from os import ST_NODEV',
'from os import ST_NODIRATIME',
'from os import ST_NOEXEC',
'from os import ST_NOSUID',
'from os import ST_RDONLY',
'from os import ST_RELATIME',
'from os import ST_SYNCHRONOUS',
'from os import ST_WRITE',
'from os import TMP_MAX',
'from os import WCONTINUED',
'from os import WCOREDUMP',
'from os import WEXITED',
'from os import WEXITSTATUS',
'from os import WIFCONTINUED',
'from os import WIFEXITED',
'from os import WIFSIGNALED',
'from os import WIFSTOPPED',
'from os import WNOHANG',
'from os import WNOWAIT',
'from os import WSTOPPED',
'from os import WSTOPSIG',
'from os import WTERMSIG',
'from os import WUNTRACED',
'from os import W_OK',
'from os import XATTR_CREATE',
'from os import XATTR_REPLACE',
'from os import XATTR_SIZE_MAX',
'from os import X_OK',
'from os import _Environ',
'from os import __all__',
'from os import __builtins__',
'from os import __doc__',
'from os import __file__',
'from os import __loader__',
'from os import __name__',
'from os import __package__',
'from os import __spec__',
'from os import _check_methods',
'from os import _execvpe',
'from os import _exists',
'from os import _exit',
'from os import _fspath',
'from os import _fwalk',
'from os import _get_exports_list',
'from os import _spawnvef',
'from os import _walk',
'from os import _wrap_close',
'from os import abc',
'from os import abort',
'from os import access',
'from os import altsep',
'from os import chdir',
'from os import chmod',
'from os import chown',
'from os import chroot',
'from os import close',
'from os import closerange',
'from os import confstr',
'from os import confstr_names',
'from os import copy_file_range',
'from os import cpu_count',
'from os import ctermid',
'from os import curdir',
'from os import defpath',
'from os import device_encoding',
'from os import devnull',
'from os import dup',
'from os import dup2',
'from os import environ',
'from os import environb',
'from os import error',
'from os import eventfd',
'from os import eventfd_read',
'from os import eventfd_write',
'from os import execl',
'from os import execle',
'from os import execlp',
'from os import execlpe',
'from os import execv',
'from os import execve',
'from os import execvp',
'from os import execvpe',
'from os import extsep',
'from os import fchdir',
'from os import fchmod',
'from os import fchown',
'from os import fdatasync',
'from os import fdopen',
'from os import fork',
'from os import forkpty',
'from os import fpathconf',
'from os import fsdecode',
'from os import fsencode',
'from os import fspath',
'from os import fstat',
'from os import fstatvfs',
'from os import fsync',
'from os import ftruncate',
'from os import fwalk',
'from os import get_blocking',
'from os import get_exec_path',
'from os import get_inheritable',
'from os import get_terminal_size',
'from os import getcwd',
'from os import getcwdb',
'from os import getegid',
'from os import getenv',
'from os import getenvb',
'from os import geteuid',
'from os import getgid',
'from os import getgrouplist',
'from os import getgroups',
'from os import getloadavg',
'from os import getlogin',
'from os import getpgid',
'from os import getpgrp',
'from os import getpid',
'from os import getppid',
'from os import getpriority',
'from os import getrandom',
'from os import getresgid',
'from os import getresuid',
'from os import getsid',
'from os import getuid',
'from os import getxattr',
'from os import initgroups',
'from os import isatty',
'from os import kill',
'from os import killpg',
'from os import lchown',
'from os import linesep',
'from os import link',
'from os import listdir',
'from os import listxattr',
'from os import lockf',
'from os import login_tty',
'from os import lseek',
'from os import lstat',
'from os import major',
'from os import makedev',
'from os import makedirs',
'from os import memfd_create',
'from os import minor',
'from os import mkdir',
'from os import mkfifo',
'from os import mknod',
'from os import name',
'from os import nice',
'from os import open',
'from os import openpty',
'from os import pardir',
'from os import path',
'from os import pathconf',
'from os import pathconf_names',
'from os import pathsep',
'from os import pidfd_open',
'from os import pipe',
'from os import pipe2',
'from os import popen',
'from os import posix_fadvise',
'from os import posix_fallocate',
'from os import posix_spawn',
'from os import posix_spawnp',
'from os import pread',
'from os import preadv',
'from os import putenv',
'from os import pwrite',
'from os import pwritev',
'from os import read',
'from os import readlink',
'from os import readv',
'from os import register_at_fork',
'from os import remove',
'from os import removedirs',
'from os import removexattr',
'from os import rename',
'from os import renames',
'from os import replace',
'from os import rmdir',
'from os import scandir',
'from os import sched_get_priority_max',
'from os import sched_get_priority_min',
'from os import sched_getaffinity',
'from os import sched_getparam',
'from os import sched_getscheduler',
'from os import sched_param',
'from os import sched_rr_get_interval',
'from os import sched_setaffinity',
'from os import sched_setparam',
'from os import sched_setscheduler',
'from os import sched_yield',
'from os import sendfile',
'from os import sep',
'from os import set_blocking',
'from os import set_inheritable',
'from os import setegid',
'from os import seteuid',
'from os import setgid',
'from os import setgroups',
'from os import setpgid',
'from os import setpgrp',
'from os import setpriority',
'from os import setregid',
'from os import setresgid',
'from os import setresuid',
'from os import setreuid',
'from os import setsid',
'from os import setuid',
'from os import setxattr',
'from os import spawnl',
'from os import spawnle',
'from os import spawnlp',
'from os import spawnlpe',
'from os import spawnv',
'from os import spawnve',
'from os import spawnvp',
'from os import spawnvpe',
'from os import splice',
'from os import st',
'from os import stat',
'from os import stat_result',
'from os import statvfs',
'from os import statvfs_result',
'from os import strerror',
'from os import supports_bytes_environ',
'from os import supports_dir_fd',
'from os import supports_effective_ids',
'from os import supports_fd',
'from os import supports_follow_symlinks',
'from os import symlink',
'from os import sync',
'from os import sys',
'from os import sysconf',
'from os import sysconf_names',
'from os import system',
'from os import tcgetpgrp',
'from os import tcsetpgrp',
'from os import terminal_size',
'from os import times',
'from os import times_result',
'from os import truncate',
'from os import ttyname',
'from os import umask',
'from os import uname',
'from os import uname_result',
'from os import unlink',
'from os import unsetenv',
'from os import urandom',
'from os import utime',
'from os import wait',
'from os import wait3',
'from os import wait4',
'from os import waitid',
'from os import waitid_result',
'from os import waitpid',
'from os import waitstatus_to_exitcode',
'from os import walk',
'from os import write',
'from os import writev'}
os         = <module 'os' (frozen)>
s          = 'from os import '
self       = <test.test_utils.TestSetupReadline testMethod=test_import>

test/test_utils.py:88: AssertionError
________________________________________________ test_import _________________________________________________

get_names = <function get_names.<locals>.<lambda> at 0x7f5aea3a1f80>

def test_import(get_names):
nms = get_names('from json import load', references=True)
assert nms[0].name == 'json'
assert nms[0].type == 'module'
n = nms[0].goto()[0]
assert n.name == 'json'
assert n.type == 'module'

assert nms[1].name == 'load'
assert nms[1].type == 'function'
n = nms[1].goto()[0]
assert n.name == 'load'
assert n.type == 'function'

nms = get_names('import os; os.path', references=True)
assert nms[0].name == 'os'
assert nms[0].type == 'module'
n = nms[0].goto()[0]
assert n.name == 'os'
assert n.type == 'module'

nms = nms[2].goto()
assert nms
assert all(n.type == 'module' for n in nms)
>       assert 'posixpath' in {n.name for n in nms}
E       AssertionError: assert 'posixpath' in {'path'}

get_names  = <function get_names.<locals>.<lambda> at 0x7f5aea3a1f80>
n          = <Name full_name='os', description='module os'>
nms        = [<Name full_name='os.path', description='module path'>]

test/test_api/test_classes.py:471: AssertionError
________________________________________ test_find_module_not_package ________________________________________

def test_find_module_not_package():
file_io, is_package = _find_module('io')
>       assert file_io.path.name == 'io.py'
E       AttributeError: 'NoneType' object has no attribute 'path'

file_io    = None
is_package = False

test/test_inference/test_imports.py:39: AttributeError
______________________________________________ test_goto_stubs _______________________________________________

Script = functools.partial(<class 'jedi.api.Script'>, environment=<SameEnvironment: 3.11.0 in /var/tmp/portage/dev-python/jedi-0.18.1/work/jedi-0.18.1-python3_11/install/usr>)

def test_goto_stubs(Script):
s = Script('import os; os')
os_module, = s.infer()
assert os_module.full_name == 'os'
>       assert os_module.is_stub() is False
E       AssertionError: assert True is False
E        +  where True = <bound method BaseName.is_stub of <Name full_name='os', description='module os'>>()
E        +    where <bound method BaseName.is_stub of <Name full_name='os', description='module os'>> = <Name full_name='os', description='module os'>.is_stub

Script     = functools.partial(<class 'jedi.api.Script'>, environment=<SameEnvironment: 3.11.0 in /var/tmp/portage/dev-python/jedi-0.18.1/work/jedi-0.18.1-python3_11/install/usr>)
os_module  = <Name full_name='os', description='module os'>
s          = <Script: None <SameEnvironment: 3.11.0 in /var/tmp/portage/dev-python/jedi-0.18.1/work/jedi-0.18.1-python3_11/install/usr>>

test/test_inference/test_gradual/test_typeshed.py:162: AssertionError
============================================== warnings summary ==============================================
test/test_utils.py::TestSetupReadline::test_import
test/test_utils.py::TestSetupReadline::test_import
test/test_utils.py::TestSetupReadline::test_modules
test/test_api/test_interpreter.py::test_complete_raw_function
test/test_api/test_interpreter.py::test_complete_raw_function_different_name
test/test_api/test_interpreter.py::test_complete_raw_module
test/test_inference/test_imports.py::test_find_module_basic
test/test_inference/test_imports.py::test_find_module_basic
<frozen importlib._bootstrap_external>:1525: DeprecationWarning: PathFinder.find_module() is deprecated and slated for removal in Python 3.12; use find_spec() instead

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
========================================== short test summary info ===========================================
SKIPPED [1] test/run.py:216: Skipped
FAILED test/test_utils.py::TestSetupReadline::test_import - AssertionError: assert 37 < 30
FAILED test/test_api/test_classes.py::test_import - AssertionError: assert 'posixpath' in {'path'}
FAILED test/test_inference/test_imports.py::test_find_module_not_package - AttributeError: 'NoneType' objec...
FAILED test/test_inference/test_gradual/test_typeshed.py::test_goto_stubs - AssertionError: assert True is ...
================== 4 failed, 2541 passed, 1 skipped, 1276 deselected, 8 warnings in 42.48s ===================

[frenzymadness]

I'm testing jedi with Python 3.11 beta 5 on Fedora 36 and I see these failures:

================================================================================== short test summary info ===================================================================================
FAILED test/test_utils.py::TestSetupReadline::test_import - AssertionError: assert 37 < 30
FAILED test/test_api/test_classes.py::test_import - AssertionError: assert 'posixpath' in {'path'}
FAILED test/test_api/test_interpreter.py::test_string_annotation[annotations10-result10-] - AssertionError: assert ['str'] == []
FAILED test/test_api/test_interpreter.py::test_string_annotation[annotations13-result13-] - AssertionError: assert ['_AnyMeta'] == []
FAILED test/test_api/test_project.py::test_search[implicit_namespace_package.ns1.pkg-full_names20-kwargs20] - AssertionError: assert ['examples.im...kage.ns1.pkg'] == ['examples.im...kage...
FAILED test/test_api/test_project.py::test_search[implicit_namespace_package.-full_names24-kwargs24] - AssertionError: assert ['examples.im..._package.ns2'] == ['examples.im..._package.ns2']
FAILED test/test_inference/test_imports.py::test_find_module_not_package - AttributeError: 'NoneType' object has no attribute 'path'
FAILED test/test_inference/test_sys_path.py::test_venv_and_pths - AssertionError: assert ['/tmp/pytest...py:from_func'] == ['/tmp/pytest...py:from_func']
FAILED test/test_inference/test_gradual/test_stubs.py::test_infer_and_goto[import os; os.walk-os.walk-True-True-options0-kwargs0-direct-goto] - assert True == False
FAILED test/test_inference/test_gradual/test_stubs.py::test_infer_and_goto[import os; os.walk-os.walk-True-True-options0-kwargs0-direct-infer] - assert True == False
FAILED test/test_inference/test_gradual/test_stubs.py::test_infer_and_goto[import os; os.walk-os.walk-True-True-options0-kwargs0-indirect-goto] - assert True == False
FAILED test/test_inference/test_gradual/test_stubs.py::test_infer_and_goto[import os; os.walk-os.walk-True-True-options0-kwargs0-indirect-infer] - assert True == False
FAILED test/test_inference/test_gradual/test_typeshed.py::test_goto_stubs - AssertionError: assert True is False
FAILED test/test_inference/test_gradual/test_typeshed.py::test_goto_stubs_on_itself[import os; os.walk-goto] - AssertionError: assert PosixPath('/home/lbalhar/Software/jedi/jedi/third_par...
FAILED test/test_inference/test_gradual/test_typeshed.py::test_goto_stubs_on_itself[import os; os.walk-infer] - AssertionError: assert PosixPath('/home/lbalhar/Software/jedi/jedi/third_pa...
======================================================= 15 failed, 3770 passed, 56 skipped, 5 xfailed, 9 warnings in 155.16s (0:02:35) =======================================================

[frenzymadness]

I'm not sure whether it's related to the problem or not but it seems that a lot of failed tests use os and os.walk as an example code. But I don't see any significant changes in Python related to the os module.

[Nowa-Ammerlaan]

I've been digging into some of these tests to find what is going on:

test/test_api/test_classes.py::test_import I'm pretty sure just fails because of a change in the description, so we can probably just adapt the test to look for path instead of posixpath: i.e. on python3.10:

[<Name full_name='os.path', description='module posixpath'>, <Name full_name='os.path', description='module ntpath'>]

but on python3.11

[<Name full_name='os.path', description='module path'>]

I'm pretty sure the failure in test/test_inference/test_imports.py::test_find_module_not_package is related to python3.11's "frozen imports": i.e. on python3.10:

>>> import sys
>>> finders = sys.meta_path
>>> print(finders)
[<_distutils_hack.DistutilsMetaFinder object at 0x7fd0ccf21f90>, <class '_frozen_importlib.BuiltinImporter'>, <class '_frozen_importlib.FrozenImporter'>, <class '_frozen_importlib_external.PathFinder'>]
>>> finders[2].find_spec('io', None)
>>> finders[3].find_spec('io', None)
ModuleSpec(name='io', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7fd0ccda3610>, origin='/usr/lib/python3.11/io.py')

but on python3.11

>>> import sys
>>> finders = sys.meta_path
>>> print(finders)
[<_distutils_hack.DistutilsMetaFinder object at 0x7fd0ccf21f90>, <class '_frozen_importlib.BuiltinImporter'>, <class '_frozen_importlib.FrozenImporter'>, <class '_frozen_importlib_external.PathFinder'>]
>>> finders[2].find_spec('io', None)
ModuleSpec(name='io', loader=<class '_frozen_importlib.FrozenImporter'>, origin='frozen')
>>> finders[3].find_spec('io', None)
ModuleSpec(name='io', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7fd0ccda3610>, origin='/usr/lib/python3.11/io.py')

Since on pyhton3.11 _frozen_importlib.FrozenImporter already returns something this is propagated further, but the lack of a path causes the test to fail further on.

I'm pretty sure the root cause for the failing *stubs* tests is similar to the above. In test/test_inference/test_gradual/test_typeshed.py::test_goto_stubs we import the os module which on python3.10 gives us <ModuleName: string_name=os start_pos=(1, 0)> but on python3.11 <StubModuleName: string_name=os start_pos=(1, 0)> which causes the is_stub() function to return True instead of False.

[Nowa-Ammerlaan]

The remaining two tests test/test_api/test_interpreter.py::test_string_annotation fail because of: python3.10:

>>> type(typing.Any)
<class 'typing._SpecialForm'>
>>> typing.Union["str", 1]
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python3.10/typing.py", line 312, in inner
return func(*args, **kwds)
File "/usr/lib/python3.10/typing.py", line 403, in __getitem__
return self._getitem(self, parameters)
File "/usr/lib/python3.10/typing.py", line 515, in Union
parameters = tuple(_type_check(p, msg) for p in parameters)
File "/usr/lib/python3.10/typing.py", line 515, in <genexpr>
parameters = tuple(_type_check(p, msg) for p in parameters)
File "/usr/lib/python3.10/typing.py", line 176, in _type_check
raise TypeError(f"{msg} Got {arg!r:.100}.")
TypeError: Union[arg, ...]: each arg must be a type. Got 1.

but with python3.11:

>>> type(typing.Any)
<class 'typing._AnyMeta'>
>>> typing.Union["str", 1]
typing.Union[ForwardRef('str'), 1]

The root cause is probably related to these changes, though I'm not sure how to fix it yet.