search

JuliaLang / Pkg.jl

Pkg - Package manager for the Julia programming language
https://pkgdocs.julialang.org
Other
621 stars 268 forks source link

1.7.0-rc1: clean up bad registry tarball on EOF exception #2743

Open johnnychen94 opened 3 years ago

johnnychen94 commented 3 years ago

I'm not sure how to reproduce it, but with some bad network conditions, I get a bad General.tar.gz file in ~/.julia/registries and then hit the following error in Julia 1.7.0-rc1:

julia> versioninfo()
Julia Version 1.7.0-rc1
Commit 9eade6195e (2021-09-12 06:45 UTC)
Platform Info:
  OS: macOS (x86_64-apple-darwin19.6.0)
  CPU: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-12.0.1 (ORCJIT, skylake)
Environment:
  JULIA_NUM_THREADS = 8

(@v1.7) pkg> up
ERROR: Unexpected end of data : General.tar
ERROR: EOFError: read end of file
Stacktrace:
  [1] read_data(tar::Base.Process, file::IOBuffer; size::Int64, buf::Vector{UInt8}, tee::Base.DevNull)
    @ Tar /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Tar/src/extract.jl:706
  [2] #12
    @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:201 [inlined]
  [3] read_tarball(callback::Pkg.Registry.var"#12#15"{Base.Process, IOBuffer, Vector{UInt8}, Dict{String, String}}, predicate::Pkg.Registry.var"#13#16", tar::Base.Process; buf::Vector{UInt8}, skeleton::Base.DevNull)
    @ Tar /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Tar/src/extract.jl:399
  [4] #11
    @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:199 [inlined]
  [5] open(::Pkg.Registry.var"#11#14"{IOBuffer, Vector{UInt8}, Dict{String, String}}, ::Cmd; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./process.jl:395
  [6] open
    @ ./process.jl:393 [inlined]
  [7] uncompress_registry(tar_gz::String)
    @ Pkg.Registry /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:198
  [8] Pkg.Registry.RegistryInstance(path::String)
    @ Pkg.Registry /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:266
  [9] reachable_registries(; depots::Vector{String})
    @ Pkg.Registry /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:373
 [10] reachable_registries
    @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:345 [inlined]
 [11] download_default_registries(io::Base.TTY; only_if_empty::Bool)
    @ Pkg.Registry /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/Registry.jl:101
 [12] download_default_registries
    @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/Registry/Registry.jl:101 [inlined]
 [13] up(pkgs::Vector{Pkg.Types.PackageSpec}; io::Base.TTY, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Pkg.API /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/API.jl:144
 [14] up(pkgs::Vector{Pkg.Types.PackageSpec})
    @ Pkg.API /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/API.jl:144
 [15] do_cmd!(command::Pkg.REPLMode.Command, repl::REPL.LineEditREPL)
    @ Pkg.REPLMode /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:407
 [16] do_cmd(repl::REPL.LineEditREPL, input::String; do_rethrow::Bool)
    @ Pkg.REPLMode /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:385
 [17] do_cmd
    @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:376 [inlined]
 [18] (::Pkg.REPLMode.var"#24#27"{REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::IOBuffer, ok::Bool)
    @ Pkg.REPLMode /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:549
 [19] #invokelatest#2
    @ ./essentials.jl:716 [inlined]
 [20] invokelatest
    @ ./essentials.jl:714 [inlined]
 [21] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
    @ REPL.LineEdit /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/REPL/src/LineEdit.jl:2493
 [22] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
    @ REPL /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/REPL/src/REPL.jl:1230
 [23] (::REPL.var"#49#54"{REPL.LineEditREPL, REPL.REPLBackendRef})()
    @ REPL ./task.jl:411

My understanding is that we need some exception handling in https://github.com/JuliaLang/Pkg.jl/blob/13b7861518dcfceebfc01566c329a2b2faa62623/src/Registry/registry_instance.jl#L368-L373 to clean up the bad registry tarball.

Vaibhavdixit02 commented 2 years ago

I hit this as well

pkg> registry status
ERROR: Unexpected end of data : General.tar
ERROR: EOFError: read end of file
Stacktrace:
  [1] read_data(tar::Base.Process, file::IOBuffer; size::Int64, buf::Vector{UInt8}, tee::Base.DevNull)
    @ Tar /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Tar/src/extract.jl:706
  [2] #12
    @ /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:201 [inlined]
  [3] read_tarball(callback::Pkg.Registry.var"#12#15"{Base.Process, IOBuffer, Vector{UInt8}, Dict{String, String}}, predicate::Pkg.Registry.var"#13#16", tar::Base.Process; buf::Vector{UInt8}, skeleton::Base.DevNull)
    @ Tar /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Tar/src/extract.jl:399
  [4] #11
    @ /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:199 [inlined]
  [5] open(::Pkg.Registry.var"#11#14"{IOBuffer, Vector{UInt8}, Dict{String, String}}, ::Cmd; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./process.jl:395
  [6] open
    @ ./process.jl:393 [inlined]
  [7] uncompress_registry(tar_gz::String)
    @ Pkg.Registry /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:198
  [8] Pkg.Registry.RegistryInstance(path::String)
    @ Pkg.Registry /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:266
  [9] reachable_registries(; depots::Vector{String})
    @ Pkg.Registry /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:373
 [10] reachable_registries
    @ /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/registry_instance.jl:345 [inlined]
 [11] status(io::Base.TTY) (repeats 2 times)
    @ Pkg.Registry /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/Registry/Registry.jl:470
 [12] do_cmd!(command::Pkg.REPLMode.Command, repl::REPL.LineEditREPL)
    @ Pkg.REPLMode /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:404
 [13] do_cmd(repl::REPL.LineEditREPL, input::String; do_rethrow::Bool)
    @ Pkg.REPLMode /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:385
 [14] do_cmd
    @ /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:376 [inlined]
 [15] (::Pkg.REPLMode.var"#24#27"{REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::IOBuffer, ok::Bool)
    @ Pkg.REPLMode /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/Pkg/src/REPLMode/REPLMode.jl:549
 [16] #invokelatest#2
    @ ./essentials.jl:716 [inlined]
 [17] invokelatest
    @ ./essentials.jl:714 [inlined]
 [18] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
    @ REPL.LineEdit /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/REPL/src/LineEdit.jl:2493
 [19] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
    @ REPL /Applications/Julia-1.7.app/Contents/Resources/julia/share/julia/stdlib/v1.7/REPL/src/REPL.jl:1230
 [20] (::REPL.var"#49#54"{REPL.LineEditREPL, REPL.REPLBackendRef})()
    @ REPL ./task.jl:423
julia> versioninfo()
Julia Version 1.7.0
Commit 3bf9d17731 (2021-11-30 12:12 UTC)
Platform Info:
  OS: macOS (x86_64-apple-darwin19.5.0)
  CPU: Intel(R) Core(TM) i5-1038NG7 CPU @ 2.00GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-12.0.1 (ORCJIT, icelake-client)
Environment:
  JULIA_EDITOR = code
  JULIA_NUM_THREADS = 8