Test failure with Python 3.11

[fabaff]

The tests are passing with Python 3.10 but not with Python 3.11 for me.

============================= test session starts ==============================
platform linux -- Python 3.11.3, pytest-7.2.1, pluggy-1.0.0
rootdir: /build/source, configfile: pyproject.toml
collected 371 items                                                            

stories/story1/test_parser.py ..                                         [  0%]
[...]
........................................................................ [ 86%]
.....................FF..........                                        [ 95%]
tests/python_parser/test_unsupported_syntax.py .................         [100%]

=================================== FAILURES ===================================
______ test_invalid_def_stmt[def f:-SyntaxError-expected '('-start4-end4] ______

python_parse_file = <function parse_file at 0x7ffff62b7380>
python_parse_str = <function parse_string at 0x7ffff5078e00>
tmp_path = PosixPath('/build/pytest-of-nixbld/pytest-0/test_invalid_def_stmt_def_f__S0')
source = 'def f:', exception = <class 'SyntaxError'>, message = "expected '('"
start = (1, 6), end = (1, 6)

    @pytest.mark.parametrize(
        "source, exception, message, start, end",
        [
            (
                "def f():\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            (
                "async def f():\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            (
                "def f(a,):\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            (
                "def f() -> None:\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            ("def f:", SyntaxError, "expected '('", (1, 6), (1, 6)),
            ("async def f:", SyntaxError, "expected '('", (1, 12), (1, 12)),
            # (
            #     "def f():\n# type: () -> int\n# type: () -> str\n\tpass",
            #     SyntaxError,
            #     "expected an indented block after function definition on line 1",
            # ),
        ],
    )
    def test_invalid_def_stmt(
        python_parse_file, python_parse_str, tmp_path, source, exception, message, start, end
    ):
>       parse_invalid_syntax(
            python_parse_file,
            python_parse_str,
            tmp_path,
            source,
            exception,
            message,
            start,
            end,
            (3, 11) if exception is SyntaxError else (3, 10),
        )

tests/python_parser/test_syntax_error_handling.py:1228: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

python_parse_file = <function parse_file at 0x7ffff62b7380>
python_parse_str = <function parse_string at 0x7ffff5078e00>
tmp_path = PosixPath('/build/pytest-of-nixbld/pytest-0/test_invalid_def_stmt_def_f__S0')
source = 'def f:', exc_cls = <class 'SyntaxError'>, message = "expected '('"
start = (1, 6), end = (1, 6), min_python_version = (3, 11)

    def parse_invalid_syntax(
        python_parse_file,
        python_parse_str,
        tmp_path,
        source,
        exc_cls,
        message,
        start,
        end,
        min_python_version=(3, 10),
    ):

        # Check we obtain the expected error from Python
        try:
            exec(source, {}, {})
        except exc_cls as py_e:
            py_exc = py_e
        except Exception as py_e:
            assert (
                False
            ), f"Python produced {py_e.__class__.__name__} instead of {exc_cls.__name__}: {py_e}"
        else:
            assert False, f"Python did not throw any exception, expected {exc_cls}"

        # Check our parser raises both from str and file mode.
        with pytest.raises(exc_cls) as e:
            python_parse_str(source, "exec")

        print(str(e.exconly()))
        assert message in str(e.exconly())

        test_file = tmp_path / "test.py"
        with open(test_file, "w") as f:
            f.write(source)

        with pytest.raises(exc_cls) as e:
            python_parse_file(str(test_file))

        # Check Python message but do not expect message to match for earlier Python versions
        if sys.version_info >= min_python_version:
            # This fails for Python < 3.10.5 but keeping the fix for a patch version is not
            # worth it
            assert message in py_exc.args[0]

        print(str(e.exconly()))
        assert message in str(e.exconly())

        # Check start/end line/column on Python 3.10
        for parser, exc in ([("Python", py_exc)] if sys.version_info >= min_python_version else []) + [
            ("pegen", e.value)
        ]:
            if (
                exc.lineno != start[0]
                or exc.offset != start[1]
                # Do not check end for indentation errors
                or (
                    sys.version_info >= (3, 10)
                    and not isinstance(e, IndentationError)
                    and exc.end_lineno != end[0]
                )
                or (
                    sys.version_info >= (3, 10)
                    and not isinstance(e, IndentationError)
                    and (end[1] is not None and exc.end_offset != end[1])
                )
            ):
                if sys.version_info >= (3, 10):
>                   raise ValueError(
                        f"Expected locations of {start} and {end}, but got "
                        f"{(exc.lineno, exc.offset)} and {(exc.end_lineno, exc.end_offset)} "
                        f"from {parser}"
                    )
E                   ValueError: Expected locations of (1, 6) and (1, 6), but got (1, 6) and (1, 7) from Python

tests/python_parser/test_syntax_error_handling.py:74: ValueError
----------------------------- Captured stdout call -----------------------------
  File "<unknown>", line 1
    def f:
         ^
SyntaxError: expected '('
  File "test.py", line 1
    def f:
         ^
SyntaxError: expected '('
___ test_invalid_def_stmt[async def f:-SyntaxError-expected '('-start5-end5] ___

python_parse_file = <function parse_file at 0x7ffff62b7380>
python_parse_str = <function parse_string at 0x7ffff5078e00>
tmp_path = PosixPath('/build/pytest-of-nixbld/pytest-0/test_invalid_def_stmt_async_de1')
source = 'async def f:', exception = <class 'SyntaxError'>
message = "expected '('", start = (1, 12), end = (1, 12)

    @pytest.mark.parametrize(
        "source, exception, message, start, end",
        [
            (
                "def f():\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            (
                "async def f():\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            (
                "def f(a,):\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            (
                "def f() -> None:\npass",
                IndentationError,
                "expected an indented block after function definition on line 1",
                (2, 1),
                (2, 5),
            ),
            ("def f:", SyntaxError, "expected '('", (1, 6), (1, 6)),
            ("async def f:", SyntaxError, "expected '('", (1, 12), (1, 12)),
            # (
            #     "def f():\n# type: () -> int\n# type: () -> str\n\tpass",
            #     SyntaxError,
            #     "expected an indented block after function definition on line 1",
            # ),
        ],
    )
    def test_invalid_def_stmt(
        python_parse_file, python_parse_str, tmp_path, source, exception, message, start, end
    ):
>       parse_invalid_syntax(
            python_parse_file,
            python_parse_str,
            tmp_path,
            source,
            exception,
            message,
            start,
            end,
            (3, 11) if exception is SyntaxError else (3, 10),
        )

tests/python_parser/test_syntax_error_handling.py:1228: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

python_parse_file = <function parse_file at 0x7ffff62b7380>
python_parse_str = <function parse_string at 0x7ffff5078e00>
tmp_path = PosixPath('/build/pytest-of-nixbld/pytest-0/test_invalid_def_stmt_async_de1')
source = 'async def f:', exc_cls = <class 'SyntaxError'>
message = "expected '('", start = (1, 12), end = (1, 12)
min_python_version = (3, 11)

    def parse_invalid_syntax(
        python_parse_file,
        python_parse_str,
        tmp_path,
        source,
        exc_cls,
        message,
        start,
        end,
        min_python_version=(3, 10),
    ):

        # Check we obtain the expected error from Python
        try:
            exec(source, {}, {})
        except exc_cls as py_e:
            py_exc = py_e
        except Exception as py_e:
            assert (
                False
            ), f"Python produced {py_e.__class__.__name__} instead of {exc_cls.__name__}: {py_e}"
        else:
            assert False, f"Python did not throw any exception, expected {exc_cls}"

        # Check our parser raises both from str and file mode.
        with pytest.raises(exc_cls) as e:
            python_parse_str(source, "exec")

        print(str(e.exconly()))
        assert message in str(e.exconly())

        test_file = tmp_path / "test.py"
        with open(test_file, "w") as f:
            f.write(source)

        with pytest.raises(exc_cls) as e:
            python_parse_file(str(test_file))

        # Check Python message but do not expect message to match for earlier Python versions
        if sys.version_info >= min_python_version:
            # This fails for Python < 3.10.5 but keeping the fix for a patch version is not
            # worth it
            assert message in py_exc.args[0]

        print(str(e.exconly()))
        assert message in str(e.exconly())

        # Check start/end line/column on Python 3.10
        for parser, exc in ([("Python", py_exc)] if sys.version_info >= min_python_version else []) + [
            ("pegen", e.value)
        ]:
            if (
                exc.lineno != start[0]
                or exc.offset != start[1]
                # Do not check end for indentation errors
                or (
                    sys.version_info >= (3, 10)
                    and not isinstance(e, IndentationError)
                    and exc.end_lineno != end[0]
                )
                or (
                    sys.version_info >= (3, 10)
                    and not isinstance(e, IndentationError)
                    and (end[1] is not None and exc.end_offset != end[1])
                )
            ):
                if sys.version_info >= (3, 10):
>                   raise ValueError(
                        f"Expected locations of {start} and {end}, but got "
                        f"{(exc.lineno, exc.offset)} and {(exc.end_lineno, exc.end_offset)} "
                        f"from {parser}"
                    )
E                   ValueError: Expected locations of (1, 12) and (1, 12), but got (1, 12) and (1, 13) from Python

tests/python_parser/test_syntax_error_handling.py:74: ValueError
----------------------------- Captured stdout call -----------------------------
  File "<unknown>", line 1
    async def f:
               ^
SyntaxError: expected '('
  File "test.py", line 1
    async def f:
               ^
SyntaxError: expected '('
=========================== short test summary info ============================
FAILED tests/python_parser/test_syntax_error_handling.py::test_invalid_def_stmt[def f:-SyntaxError-expected '('-start4-end4] - ValueError: Expected locations of (1, 6) and (1, 6), but got (1, 6) and (1,...
FAILED tests/python_parser/test_syntax_error_handling.py::test_invalid_def_stmt[async def f:-SyntaxError-expected '('-start5-end5] - ValueError: Expected locations of (1, 12) and (1, 12), but got (1, 12) and ...
======================== 2 failed, 369 passed in 5.74s =========================
/nix/store/37p8gq9zijbw6pj3lpi1ckqiv18j2g62-stdenv-linux/setup: line 1594: pop_var_context: head of shell_variables not a function context
error: builder for '/nix/store/px2vanhxlchdwlan0n6xjn0xvh0d715n-python3.11-pegen-0.2.0.drv' failed with exit code 1;
       last 10 log lines:
       > SyntaxError: expected '('
       >   File "test.py", line 1
       >     async def f:
       >                ^
       > SyntaxError: expected '('
       > =========================== short test summary info ============================
       > FAILED tests/python_parser/test_syntax_error_handling.py::test_invalid_def_stmt[def f:-SyntaxError-expected '('-start4-end4] - ValueError: Expected locations of (1, 6) and (1, 6), but got (1, 6) and (1,...
       > FAILED tests/python_parser/test_syntax_error_handling.py::test_invalid_def_stmt[async def f:-SyntaxError-expected '('-start5-end5] - ValueError: Expected locations of (1, 12) and (1, 12), but got (1, 12) and ...
       > ======================== 2 failed, 369 passed in 5.74s =========================

[phorward]

Hello @fabaff,

I've updated the test suite to only run on Python 3.10 and 3.11 in my fork anypeg. Feel free to check it out.

Generally, regarding pegen, I would like to propose to only support Python 3.10+ and throw away support for 3.8 and 3.9. What do the maintainers think about this change?