Compatibility with Python 3.12 and unreleased astroid 3.0.0

[hrnciar]

Hello,

I'd like to ask for help, in Fedora, we are rebuilding all Python packages with the upcoming Python 3.12.0. I've tested astroid from the main branch (last commit https://github.com/pylint-dev/astroid/commit/8d57ce2f3e226c2ac3cdd7f6a57dac2dd5ec5a4b at this time) and it works with 3.12, but asttokens fails with following failures. Thank you.

=================================== FAILURES ===================================
___________________ TestAstroid.test_adjacent_joined_strings ___________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a8c00ad0>
data = "_\n(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = "_\n(f')", type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = "_\n(f')\n", type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = "_\n(f')\n", filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           (f')
E            ^
E       SyntaxError: unterminated f-string literal (detected at line 2)

/usr/lib64/python3.12/ast.py:52: SyntaxError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_adjacent_joined_strings>

        def test_adjacent_joined_strings(self):
            source = """
    foo = f'x y z' \\
    f'''a b c''' f"u v w"
    bar = ('x y z'   # comment2
           'a b c'   # comment3
           f'u v w'
          )
    """
>           m = self.create_mark_checker(source)

tests/test_mark_tokens.py:327: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:801: in parse_snippet
    return self.module.parse('_\n(' + text + ')').body[1].value
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a8c00ad0>
data = "_\n(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           unterminated f-string literal (detected at line 2) (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
__________________________ TestAstroid.test_fixture9 ___________________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a80692e0>
data = '_\nclass Aaaa(base):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = '_\nclass Aaaa(base):', type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = '_\nclass Aaaa(base):\n', type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = '_\nclass Aaaa(base):\n', filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           class Aaaa(base):
E                            ^
E       IndentationError: expected an indented block after class definition on line 2

/usr/lib64/python3.12/ast.py:52: IndentationError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_fixture9>

>   def test_fixture9(self): self.verify_fixture_file('astroid/module2.py')

tests/test_mark_tokens.py:181: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:140: in verify_fixture_file
    tested_nodes = m.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:804: in parse_snippet
    return self.module.parse('_\n' + text).body[1]
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a80692e0>
data = '_\nclass Aaaa(base):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           expected an indented block after class definition on line 2 (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
__________________________ TestAstroid.test_fstrings ___________________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a806bbf0>
data = '_\n((f")', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = '_\n((f")', type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = '_\n((f")\n', type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = '_\n((f")\n', filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           ((f")
E             ^
E       SyntaxError: unterminated f-string literal (detected at line 2)

/usr/lib64/python3.12/ast.py:52: SyntaxError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_fstrings>

    def test_fstrings(self):
      for source in (
        '(f"He said his name is {name!r}.",)',
        "f'{function(kwarg=24)}'",
        'a = f"""result: {value:{width}.{precision}}"""',
        """[f"abc {a['x']} def"]""",
        "def t():\n  return f'{function(kwarg=24)}'"):
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:316: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:801: in parse_snippet
    return self.module.parse('_\n(' + text + ')').body[1].value
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a806bbf0>
data = '_\n((f")', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           unterminated f-string literal (detected at line 2) (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
_________________________ TestAstroid.test_sys_modules _________________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a750fcb0>
data = "_\n(_DeadlockError(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = "_\n(_DeadlockError(f')", type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = "_\n(_DeadlockError(f')\n", type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = "_\n(_DeadlockError(f')\n", filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           (_DeadlockError(f')
E                           ^
E       SyntaxError: unterminated f-string literal (detected at line 2)

/usr/lib64/python3.12/ast.py:52: SyntaxError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_sys_modules>

    def test_sys_modules(self):
      """
      Verify all nodes on source files obtained from sys.modules.
      This can take a long time as there are many modules,
      so it only tests all modules if the environment variable
      ASTTOKENS_SLOW_TESTS has been set.
      """
      modules = list(sys.modules.values())
      if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
        modules = modules[:20]

      start = time()
      for module in modules:
        # Don't let this test (which runs twice) take longer than 13 minutes
        # to avoid the travis build time limit of 30 minutes
        if time() - start > 13 * 60:
          break

        try:
          filename = inspect.getsourcefile(module)
        except TypeError:
          continue

        if not filename:
          continue

        filename = os.path.abspath(filename)
        print(filename)
        try:
          with io.open(filename) as f:
            source = f.read()
        except OSError:
          continue

        if self.is_astroid_test and (
            # Astroid fails with a syntax error if a type comment is on its own line
            re.search(r'^\s*# type: ', source, re.MULTILINE)
            # Astroid can fail on this file, specifically raising an exception at this line of code:
            #     lambda node: node.name == "NamedTuple" and node.parent.name == "typing"
            # with the error:
            #     AttributeError: 'If' object has no attribute 'name'
            # See https://github.com/gristlabs/asttokens/runs/7602147792
            # I think the code that causes the problem is:
            #     if sys.version_info >= (3, 11):
            #         NamedTuple = typing.NamedTuple
            or filename.endswith("typing_extensions.py")
        ):
          print('Skipping', filename)
          continue

>       self.create_mark_checker(source)

tests/test_mark_tokens.py:673: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:801: in parse_snippet
    return self.module.parse('_\n(' + text + ')').body[1].value
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a750fcb0>
data = "_\n(_DeadlockError(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           unterminated f-string literal (detected at line 2) (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
----------------------------- Captured stdout call -----------------------------
/usr/lib64/python3.12/importlib/_bootstrap.py
_________________ TestMarkTokens.test_adjacent_joined_strings __________________

self = <tests.test_mark_tokens.TestMarkTokens testMethod=test_adjacent_joined_strings>

        def test_adjacent_joined_strings(self):
            source = """
    foo = f'x y z' \\
    f'''a b c''' f"u v w"
    bar = ('x y z'   # comment2
           'a b c'   # comment3
           f'u v w'
          )
    """
>           m = self.create_mark_checker(source)

tests/test_mark_tokens.py:327: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:84: in verify_all_nodes
    self.check_get_text_tokenless(node, test_case, text)
tests/tools.py:142: in check_get_text_tokenless
    test_case.assertEqual(text, text_tokenless, ast.dump(node))
E   AssertionError: "f'" != 'f\'x y z\' \\\nf\'\'\'a b c\'\'\' f"u v w"'
E   - f'
E   + f'x y z' \
E   + f'''a b c''' f"u v w"
E    : JoinedStr(values=[Constant(value='x y za b cu v w')])
_________________________ TestMarkTokens.test_fstrings _________________________

self = <tests.test_mark_tokens.TestMarkTokens testMethod=test_fstrings>

    def test_fstrings(self):
      for source in (
        '(f"He said his name is {name!r}.",)',
        "f'{function(kwarg=24)}'",
        'a = f"""result: {value:{width}.{precision}}"""',
        """[f"abc {a['x']} def"]""",
        "def t():\n  return f'{function(kwarg=24)}'"):
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:316: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:84: in verify_all_nodes
    self.check_get_text_tokenless(node, test_case, text)
tests/tools.py:142: in check_get_text_tokenless
    test_case.assertEqual(text, text_tokenless, ast.dump(node))
E   AssertionError: '(f"' != '(f"He said his name is {name!r}.",)'
E   - (f"
E   + (f"He said his name is {name!r}.",)
E    : Tuple(elts=[JoinedStr(values=[Constant(value='He said his name is '), FormattedValue(value=Name(id='name', ctx=Load()), conversion=114), Constant(value='.')])], ctx=Load())
_______________________ TestMarkTokens.test_sys_modules ________________________

self = <tests.test_mark_tokens.TestMarkTokens testMethod=test_sys_modules>

    def test_sys_modules(self):
      """
      Verify all nodes on source files obtained from sys.modules.
      This can take a long time as there are many modules,
      so it only tests all modules if the environment variable
      ASTTOKENS_SLOW_TESTS has been set.
      """
      modules = list(sys.modules.values())
      if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
        modules = modules[:20]

      start = time()
      for module in modules:
        # Don't let this test (which runs twice) take longer than 13 minutes
        # to avoid the travis build time limit of 30 minutes
        if time() - start > 13 * 60:
          break

        try:
          filename = inspect.getsourcefile(module)
        except TypeError:
          continue

        if not filename:
          continue

        filename = os.path.abspath(filename)
        print(filename)
        try:
          with io.open(filename) as f:
            source = f.read()
        except OSError:
          continue

        if self.is_astroid_test and (
            # Astroid fails with a syntax error if a type comment is on its own line
            re.search(r'^\s*# type: ', source, re.MULTILINE)
            # Astroid can fail on this file, specifically raising an exception at this line of code:
            #     lambda node: node.name == "NamedTuple" and node.parent.name == "typing"
            # with the error:
            #     AttributeError: 'If' object has no attribute 'name'
            # See https://github.com/gristlabs/asttokens/runs/7602147792
            # I think the code that causes the problem is:
            #     if sys.version_info >= (3, 11):
            #         NamedTuple = typing.NamedTuple
            or filename.endswith("typing_extensions.py")
        ):
          print('Skipping', filename)
          continue

>       self.create_mark_checker(source)

tests/test_mark_tokens.py:673: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:84: in verify_all_nodes
    self.check_get_text_tokenless(node, test_case, text)
tests/tools.py:142: in check_get_text_tokenless
    test_case.assertEqual(text, text_tokenless, ast.dump(node))
E   AssertionError: "_DeadlockError(f'" != "_DeadlockError(f'deadlock detected by {self!r}')"
E   - _DeadlockError(f'
E   + _DeadlockError(f'deadlock detected by {self!r}')
E    : Call(func=Name(id='_DeadlockError', ctx=Load()), args=[JoinedStr(values=[Constant(value='deadlock detected by '), FormattedValue(value=Name(id='self', ctx=Load()), conversion=114)])], keywords=[])
----------------------------- Captured stdout call -----------------------------
/usr/lib64/python3.12/importlib/_bootstrap.py
____________________ TestTokenless.test_get_text_tokenless _____________________

self = <tests.test_tokenless.TestTokenless testMethod=test_get_text_tokenless>

    def test_get_text_tokenless(self):
      atok = ASTText(source)

      for node in ast.walk(atok.tree):
        if not isinstance(node, (ast.arguments, ast.arg)):
>         self.check_node(atok, node)

tests/test_tokenless.py:70: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_tokenless.py:108: in check_node
    self.assertEqual(atok_text, ast_text, node)
E   AssertionError: '' != '{xx + 22}'
E   + {xx + 22}
E    : <ast.FormattedValue object at 0x7f28a95a4ed0>
_____________________________ test_combine_tokens ______________________________

    def test_combine_tokens():
      from tokenize import TokenInfo, generate_tokens, ERRORTOKEN, OP, NUMBER, NAME
      from asttokens.util import combine_tokens, patched_generate_tokens

      text = "℘·2=1"
      original_tokens = list(generate_tokens(io.StringIO(text).readline))[:4]
>     assert original_tokens == [
        TokenInfo(ERRORTOKEN, string='℘', start=(1, 0), end=(1, 1), line='℘·2=1'),
        TokenInfo(ERRORTOKEN, string='·', start=(1, 1), end=(1, 2), line='℘·2=1'),
        TokenInfo(NUMBER, string='2', start=(1, 2), end=(1, 3), line='℘·2=1'),
        TokenInfo(OP, string='=', start=(1, 3), end=(1, 4), line='℘·2=1'),
      ]
E     AssertionError: assert [TokenInfo(ty...line='℘·2=1')] == [TokenInfo(ty...line='℘·2=1')]
E       At index 0 diff: TokenInfo(type=1 (NAME), string='℘·2', start=(1, 0), end=(1, 3), line='℘·2=1') != TokenInfo(type=66 (ERRORTOKEN), string='℘', start=(1, 0), end=(1, 1), line='℘·2=1')
E       Full diff:
E         [
E       -  TokenInfo(type=66 (ERRORTOKEN), string='℘', start=(1, 0), end=(1, 1), line='℘·2=1'),
E       -  TokenInfo(type=66 (ERRORTOKEN), string='·', start=(1, 1), end=(1, 2), line='℘·2=1'),
E       -  TokenInfo(type=2 (NUMBER), string='2', start=(1, 2), end=(1, 3), line='℘·2=1'),
E       ?                 ^   ^ - -                         ^...
E       
E       ...Full output truncated (6 lines hidden), use '-vv' to show

tests/test_util.py:126: AssertionError
=========================== short test summary info ============================
FAILED tests/test_astroid.py::TestAstroid::test_adjacent_joined_strings - ast...
FAILED tests/test_astroid.py::TestAstroid::test_fixture9 - astroid.exceptions...
FAILED tests/test_astroid.py::TestAstroid::test_fstrings - astroid.exceptions...
FAILED tests/test_astroid.py::TestAstroid::test_sys_modules - astroid.excepti...
FAILED tests/test_mark_tokens.py::TestMarkTokens::test_adjacent_joined_strings
FAILED tests/test_mark_tokens.py::TestMarkTokens::test_fstrings - AssertionEr...
FAILED tests/test_mark_tokens.py::TestMarkTokens::test_sys_modules - Assertio...
FAILED tests/test_tokenless.py::TestTokenless::test_get_text_tokenless - Asse...
FAILED tests/test_util.py::test_combine_tokens - AssertionError: assert [Toke...
============ 9 failed, 106 passed, 1 skipped, 25 warnings in 4.10s =============

[alexmojaki]

Most of the test failures aren't astroid failures, so this is probably just about Python 3.12.

It looks like this is caused by f-strings having actual tokens now, e.g. this code:

foo = f'a {b} c {d} e'

is tokenized as:

0,0-0,0:            ENCODING       'utf-8'        
1,0-1,3:            NAME           'foo'          
1,4-1,5:            OP             '='            
1,6-1,8:            FSTRING_START  "f'"           
1,8-1,10:           FSTRING_MIDDLE 'a '           
1,10-1,11:          OP             '{'            
1,11-1,12:          NAME           'b'            
1,12-1,13:          OP             '}'            
1,13-1,16:          FSTRING_MIDDLE ' c '          
1,16-1,17:          OP             '{'            
1,17-1,18:          NAME           'd'            
1,18-1,19:          OP             '}'            
1,19-1,21:          FSTRING_MIDDLE ' e'           
1,21-1,22:          FSTRING_END    "'"            
1,22-1,23:          NEWLINE        '\n'           
2,0-2,0:            ENDMARKER      ''