IndexError: string index out of range

autopep8 -i model.py Traceback (most recent call last): File "/.venv/bin/autopep8", line 8, in sys.exit(main()) ^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 4590, in main results = fix_multiple_files(args.files, args, sys.stdout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 4485, in fix_multiple_files ret = _fix_file((name, options, output)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 4455, in _fix_file return fix_file(*parameters) ^^^^^^^^^^^^^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 3629, in fix_file fixed_source = fix_lines(fixed_source, options, filename=filename) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 3609, in fix_lines fixed_source = fix.fix() ^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 623, in fix results = _execute_pep8(pep8_options, self.source) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/autopep8.py", line 3021, in _execute_pep8 checker.check_all() File "/.venv/lib/python3.12/site-packages/pycodestyle.py", line 2115, in check_all self.check_logical() File "/.venv/lib/python3.12/site-packages/pycodestyle.py", line 1973, in check_logical mapping = self.build_tokens_line() ^^^^^^^^^^^^^^^^^^^^^^^^ File "/.venv/lib/python3.12/site-packages/pycodestyle.py", line 1956, in build_tokens_line prev_text = self.lines[prev_row - 1][prev_col - 1]


IndexError: string index out of range

## Python Code
```python

private
```

## Command Line and Configuration
`.pep8`, `setup.cfg`, ...

```ini
[pep8]

```

Command Line
```console
$ autopep8 
```

## Your Environment
* Python version: Python 3.12.4
* autopep8 version: autopep8 2.3.1 (pycodestyle: 2.12.0)
* Platform:  linux

📝

Example code to be reproduced

# target.py
doc, doc2 = ("""
World:
1 + "programming"
+ " pythonあ""",
"""abc"""
)

with Python3.12 (3.12.4)

$ python --version
Python 3.12.4
$ pycodestyle --version
2.12.0
$ pycodestyle target.py
Traceback (most recent call last):
  File "/Users/hideohattori/.pyenv/versions/common3124/bin/pycodestyle", line 8, in <module>
    sys.exit(_main())
             ^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2671, in _main
    report = style_guide.check_files()
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2371, in check_files
    runner(path)
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2383, in input_file
    return fchecker.check_all(expected=expected, line_offset=line_offset)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2137, in check_all
    self.check_logical()
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 1993, in check_logical
    mapping = self.build_tokens_line()
              ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 1973, in build_tokens_line
    prev_text = self.lines[prev_row - 1][prev_col - 1]
                ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
IndexError: string index out of range

with Python3.11 (3.11.7)

$ python --version
Python 3.11.7
$ pycodestyle --version
2.12.0
$ pycodestyle target.py
t3_2.py:6:1: E128 continuation line under-indented for visual indent
t3_2.py:7:1: E124 closing bracket does not match visual indentation

Why?

The behaviour of tokenize.generate_tokens in Python 3.12 may have changed.

# sample1.py
import tokenize
from io import StringIO

code = """doc, doc2 = (\"\"\"
World:
1 + "programming"
+ " pythonあ\"\"\",
'''abc'''
)
"""

def tokenize_code(code):
    tokens = []
    for tok in tokenize.generate_tokens(StringIO(code).readline):
        tokens.append((tok.type, tok.string, tok.start, tok.end, tok.line))
    return tokens

tokens_311 = tokenize_code(code)
for token in tokens_311:
    print(token)

print(tokenize.untokenize(tokens_311))

with Python3.11.7

~/.pyenv/versions/3.11.7/bin/python sample1.py
(1, 'doc', (1, 0), (1, 3), 'doc, doc2 = ("""\n')
(54, ',', (1, 3), (1, 4), 'doc, doc2 = ("""\n')
(1, 'doc2', (1, 5), (1, 9), 'doc, doc2 = ("""\n')
(54, '=', (1, 10), (1, 11), 'doc, doc2 = ("""\n')
(54, '(', (1, 12), (1, 13), 'doc, doc2 = ("""\n')
(3, '"""\nWorld:\n1 + "programming"\n+ " pythonあ"""', (1, 13), (4, 14), 'doc, doc2 = ("""\nWorld:\n1 + "programming"\n+ " pythonあ""",\n')
(54, ',', (4, 14), (4, 15), '+ " pythonあ""",\n')
(62, '\n', (4, 15), (4, 16), '+ " pythonあ""",\n')
(3, "'''abc'''", (5, 0), (5, 9), "'''abc'''\n")
(62, '\n', (5, 9), (5, 10), "'''abc'''\n")
(54, ')', (6, 0), (6, 1), ')\n')
(4, '\n', (6, 1), (6, 2), ')\n')
(0, '', (7, 0), (7, 0), '')
doc, doc2 = ("""
World:
1 + "programming"
+ " pythonあ""",
'''abc'''
)

with Python3.12.4

~/.pyenv/versions/3.12.4/bin/python sample1.py
(1, 'doc', (1, 0), (1, 3), 'doc, doc2 = ("""\n')
(55, ',', (1, 3), (1, 4), 'doc, doc2 = ("""\n')
(1, 'doc2', (1, 5), (1, 9), 'doc, doc2 = ("""\n')
(55, '=', (1, 10), (1, 11), 'doc, doc2 = ("""\n')
(55, '(', (1, 12), (1, 13), 'doc, doc2 = ("""\n')
(3, '"""\nWorld:\n1 + "programming"\n+ " pythonあ"""', (1, 13), (4, 14), 'doc, doc2 = ("""\nWorld:\n1 + "programming"\n+ " pythonあ""",\n')
(55, ',', (4, 16), (4, 17), '+ " pythonあ""",\n')
(65, '\n', (4, 17), (4, 18), '+ " pythonあ""",\n')
(3, "'''abc'''", (5, 0), (5, 9), "'''abc'''\n")
(65, '\n', (5, 9), (5, 10), "'''abc'''\n")
(55, ')', (6, 0), (6, 1), ')\n')
(4, '\n', (6, 1), (6, 2), ')\n')
(0, '', (7, 0), (7, 0), '')
doc, doc2 = ("""
World:
1 + "programming"
+ " pythonあ"""  ,
'''abc'''
)

hhatto / autopep8