Internal error parsing corrupt expression

maleadt commented 10 months ago

Attached input causes JuliaSyntax to crash with an internal error: bug.jl.zip

❯ ./julia /tmp/bug.jl
┌ Error: JuliaSyntax parser failed — falling back to flisp!
│   exception =
│    Internal error: k == K"string"
│    Stacktrace:
│      [1] error(::String, ::String)
│        @ Base ./error.jl:44
│      [2] internal_error(strs::String)
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/utils.jl:23
│      [3] _string_to_Expr(k::Base.JuliaSyntax.Kind, args::Vector{Any})
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/expr.jl:144
│      [4] _internal_node_to_Expr(source::Base.JuliaSyntax.SourceFile, srcrange::UnitRange{Int64}, head::Base.JuliaSyntax.SyntaxHead, childranges::Vector{UnitRange{Int64}}, childheads::Vector{Base.JuliaSyntax.SyntaxHead}, args::Vector{Any})
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/expr.jl:194
│      [5] _to_expr(node::Base.JuliaSyntax.SyntaxNode)
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/expr.jl:499
│      [6] _to_expr(node::Base.JuliaSyntax.SyntaxNode) (repeats 3 times)
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/expr.jl:498
│      [7] Expr(node::Base.JuliaSyntax.SyntaxNode)
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/expr.jl:503
│      [8] core_parser_hook(code::String, filename::String, lineno::Int64, offset::Int64, options::Symbol)
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/hooks.jl:209
│      [9] invoke_in_world(::UInt64, ::Any, ::Any, ::Vararg{Any}; kwargs::@Kwargs{})
│        @ Base ./essentials.jl:929
│     [10] invoke_in_world(::UInt64, ::Any, ::Any, ::Vararg{Any})
│        @ Base ./essentials.jl:926
│     [11] (::Base.JuliaSyntax.var"#invoke_fixedworld#122"{Base.JuliaSyntax.var"#invoke_fixedworld#119#123"{typeof(Base.JuliaSyntax.core_parser_hook), UInt64}})(::String, ::Vararg{Any}; kws::@Kwargs{})
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/hooks.jl:118
│     [12] (::Base.JuliaSyntax.var"#invoke_fixedworld#122"{Base.JuliaSyntax.var"#invoke_fixedworld#119#123"{typeof(Base.JuliaSyntax.core_parser_hook), UInt64}})(::String, ::Vararg{Any})
│        @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/hooks.jl:117
│     [13] _parse_string(text::String, filename::String, lineno::Int64, index::Int64, options::Symbol)
│        @ Base.Meta ./meta.jl:200
│     [14] #parseall#6
│        @ Base.Meta ./meta.jl:294 [inlined]
│     [15] parseall
│        @ Base.Meta ./meta.jl:293 [inlined]
│     [16] include_string(mapexpr::typeof(identity), mod::Module, code::String, filename::String)
│        @ Base ./loading.jl:2056
│     [17] _include(mapexpr::Function, mod::Module, _path::String)
│        @ Base ./loading.jl:2130
│     [18] include(mod::Module, _path::String)
│        @ Base ./Base.jl:493
│     [19] exec_options(opts::Base.JLOptions)
│        @ Base ./client.jl:317
│     [20] _start()
│        @ Base ./client.jl:552
│   offset = 0
│   code = "a = 10\x11J\nb = a\x80\0=19\x11J\nb = a\x80\0=\0\0\n`\x0e=* \xa0 a \0\0\n` =* b a \npr"
└ @ Base.JuliaSyntax ~/Julia/src/julia-afl/base/JuliaSyntax/src/hooks.jl:258

c42f commented 8 months ago

This is a bug in the hook (and associated SyntaxNode -> Expr conversion, which only happens on the error code path)

Somewhat more targeted reproduction:

julia> text = "a = 10\x11J\nb = a\x80\0=19\x11J\nb = a\x80\0=\0\0\n`\x0e=* \xa0 a \0\0\n` =* b a \npr\n"
"a = 10\x11J\nb = a\x80\0=19\x11J\nb = a\x80\0=\0\0\n`\x0e=* \xa0 a \0\0\n` =* b a \npr\n"

julia> ex = parseall(SyntaxNode, text, ignore_errors=true)
line:col│ tree                                   │ file_name
   1:1  │[toplevel]
   1:1  │  [=]
   1:1  │    a
   1:5  │    [juxtapose]
   1:5  │      10
   1:7  │      ✘
   1:8  │      J
   2:1  │  [=]
   2:1  │    b
   2:5  │    [=]
   2:5  │      [juxtapose]
   2:5  │        a
   2:6  │        ✘
   2:7  │        ✘
   2:9  │      [juxtapose]
   2:9  │        19
   2:11 │        ✘
   2:12 │        J
   3:1  │  [=]
   3:1  │    b
   3:5  │    [=]
   3:5  │      [juxtapose]
   3:5  │        a
   3:6  │        ✘
   3:7  │        ✘
   3:9  │      [juxtapose]
   3:9  │        ✘
   3:10 │        ✘
   4:1  │  [=]
   4:1  │    [macrocall]
   4:1  │      core_@cmd
   4:1  │      [cmdstring-r]
   4:2  │        ✘
   5:4  │    [call-pre]
   5:4  │      [error]
   5:4  │        *
   5:6  │      b
   5:7  │  [error-t]
   5:8  │    a
   6:1  │  pr

julia> Expr(ex)
ERROR: Internal error: k == K"string"
Stacktrace:
 [1] error(::String, ::String)
   @ Base ./error.jl:44
 [2] internal_error(strs::String)
   @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/utils.jl:23
 [3] _string_to_Expr(k::JuliaSyntax.Kind, args::Vector{Any})
   @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:144
 [4] _internal_node_to_Expr(source::SourceFile, srcrange::UnitRange{Int64}, head::SyntaxHead, childranges::Vector{UnitRange{Int64}}, childheads::Vector{SyntaxHead}, args::Vector{Any})
   @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:204
 [5] _to_expr(node::SyntaxNode)
   @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:525
 [6] _to_expr(node::SyntaxNode) (repeats 3 times)
   @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:524
 [7] Expr(node::SyntaxNode)
   @ JuliaSyntax ~/.julia/dev/JuliaSyntax/src/expr.jl:529
 [8] top-level scope
   @ REPL[17]:1

c42f commented 8 months ago

Minimal reproduction:

julia> Expr(parseall(SyntaxNode, "`", ignore_errors=true))
ERROR: Internal error: k == K"string"
Stacktrace:
 [1] error(::String, ::String)
   @ Base ./error.jl:44
[...]

JuliaLang / JuliaSyntax.jl

Internal error parsing corrupt expression #353