Skip to content

Commit

Permalink
Excise REPL and its dependencies from sysimg (#51399)
Browse files Browse the repository at this point in the history
  • Loading branch information
vchuravy authored Sep 23, 2023
1 parent a127ab7 commit 2defa57
Show file tree
Hide file tree
Showing 18 changed files with 349 additions and 226 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-%

debug release : % : julia-% stdlibs-cache-%

docs: julia-sysimg-$(JULIA_BUILD_MODE)
docs: julia-sysimg-$(JULIA_BUILD_MODE) stdlibs-cache-$(JULIA_BUILD_MODE)
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no'

docs-revise:
Expand Down Expand Up @@ -184,7 +184,7 @@ $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(buil
@$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c)

julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \
@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) JULIA_FALLBACK_REPL=1 WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \
$(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \
$(call cygpath_w,$(build_datarootdir)/julia/base.cache))

Expand Down
25 changes: 20 additions & 5 deletions base/client.jl
Original file line number Diff line number Diff line change
Expand Up @@ -405,13 +405,28 @@ function load_InteractiveUtils(mod::Module=Main)
return getfield(mod, :InteractiveUtils)
end

function load_REPL()
# load interactive-only libraries
try
return Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL"))
catch ex
@warn "Failed to import REPL" exception=(ex, catch_backtrace())
end
return nothing
end

global active_repl

# run the requested sort of evaluation loop on stdio
function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool)
load_InteractiveUtils()

fallback_repl = get_bool_env("JULIA_FALLBACK_REPL", false)
fallback_repl = parse(Bool, get(ENV, "JULIA_FALLBACK_REPL", "false"))
if !fallback_repl && interactive
load_InteractiveUtils()
if !isassigned(REPL_MODULE_REF)
load_REPL()
end
end
# TODO cleanup REPL_MODULE_REF

if !fallback_repl && interactive && isassigned(REPL_MODULE_REF)
invokelatest(REPL_MODULE_REF[]) do REPL
Expand All @@ -435,8 +450,8 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_f
end
else
# otherwise provide a simple fallback
if interactive && !quiet
@warn "REPL provider not available: using basic fallback"
if !fallback_repl && interactive && !quiet
@warn "REPL provider not available: using basic fallback" LOAD_PATH=join(Base.LOAD_PATH, Sys.iswindows() ? ';' : ':')
end
banner == :no || Base.banner(short=banner==:short)
let input = stdin
Expand Down
2 changes: 1 addition & 1 deletion base/docs/Docs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,7 @@ function docm(source::LineNumberNode, mod::Module, ex)
elseif isassigned(Base.REPL_MODULE_REF)
# TODO: this is a shim to continue to allow `@doc` for looking up docstrings
REPL = Base.REPL_MODULE_REF[]
return REPL.lookup_doc(ex)
return invokelatest(REPL.lookup_doc, ex)
end
return nothing
end
Expand Down
3 changes: 0 additions & 3 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1818,9 +1818,6 @@ function __require_prelocked(uuidkey::PkgId, env=nothing)
insert_extension_triggers(uuidkey)
# After successfully loading, notify downstream consumers
run_package_callbacks(uuidkey)
if uuidkey == REPL_PKGID
REPL_MODULE_REF[] = newm
end
else
newm = root_module(uuidkey)
end
Expand Down
46 changes: 11 additions & 35 deletions base/sysimg.jl
Original file line number Diff line number Diff line change
Expand Up @@ -31,43 +31,19 @@ let
# Run with the `--exclude-jlls` option to filter out all JLL packages
stdlibs = [
# No dependencies
:ArgTools,
:Artifacts,
:Base64,
:CRC32c,
:FileWatching,
:Libdl,
:Logging,
:Mmap,
:NetworkOptions,
:SHA,
:Serialization,
:Sockets,
:Unicode,
:FileWatching, # used by loading.jl -- implicit assumption that init runs
:Libdl, # Transitive through LinAlg
:Artifacts, # Transitive through LinAlg
:SHA, # transitive through Random
:Sockets, # used by stream.jl

# Transitive through LingAlg
# OpenBLAS_jll
# libblastrampoline_jll

# 1-depth packages
:LinearAlgebra,
:Markdown,
:Printf,
:Random,
:Tar,

# 2-depth packages
:Dates,
:Future,
:InteractiveUtils,
:LibGit2,
:UUIDs,

# 3-depth packages
:REPL,
:TOML,

# 4-depth packages
:LibCURL,

# 5-depth packages
:Downloads,
:LinearAlgebra, # Commits type-piracy and GEMM
:Random, # Can't be removed due to rand being exported by Base
]
# PackageCompiler can filter out stdlibs so it can be empty
maxlen = maximum(textwidth.(string.(stdlibs)); init=0)
Expand Down
21 changes: 12 additions & 9 deletions base/terminfo.jl
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ function read(data::IO, ::Type{TermInfoRaw})
throw(ArgumentError("Terminfo did not contain a null byte after the flag section, expected to position the start of the numbers section on an even byte"))
end
# Numbers, Strings, Table
numbers = reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))) .|> ltoh
string_indices = reinterpret(UInt16, read(data, string_count * sizeof(UInt16))) .|> ltoh
numbers = map(ltoh, reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))))
string_indices = map(ltoh, reinterpret(UInt16, read(data, string_count * sizeof(UInt16))))
strings_table = read(data, table_bytes)
strings = map(string_indices) do idx
if idx (0xffff, 0xfffe)
Expand All @@ -107,7 +107,7 @@ function read(data::IO, ::Type{TermInfoRaw})
end
end
TermInfoRaw(term_names, flags, numbers, strings,
if !eof(data) extendedterminfo(data; NumInt) end)
if !eof(data) extendedterminfo(data, NumInt) end)
end

"""
Expand All @@ -119,7 +119,7 @@ This will accept any terminfo content that conforms with `term(5)`.
See also: `read(::IO, ::Type{TermInfoRaw})`
"""
function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}})
function extendedterminfo(data::IO, NumInt::Union{Type{UInt16}, Type{UInt32}})
# Extended info
if position(data) % 2 != 0
0x00 == read(data, UInt8) ||
Expand All @@ -138,12 +138,15 @@ function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}})
throw(ArgumentError("Terminfo did not contain a null byte after the extended flag section, expected to position the start of the numbers section on an even byte"))
end
numbers = map(n -> Int(ltoh(n)), reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))))
table_indices = reinterpret(UInt16, read(data, table_count * sizeof(UInt16))) .|> ltoh
table_indices = map(ltoh, reinterpret(UInt16, read(data, table_count * sizeof(UInt16))))
table_strings = [String(readuntil(data, 0x00)) for _ in 1:length(table_indices)]
info = Dict{Symbol, Union{Bool, Int, String}}()
strings = table_strings[1:string_count]
labels = Symbol.(table_strings[string_count+1:end])
Dict{Symbol, Union{Bool, Int, String}}(
labels .=> vcat(flags, numbers, strings))
labels = table_strings[string_count+1:end]
for (label, val) in zip(labels, vcat(flags, numbers, strings))
info[Symbol(label)] = val
end
return info
end

"""
Expand Down Expand Up @@ -178,7 +181,7 @@ function TermInfo(raw::TermInfoRaw)
Symbol[]
end
TermInfo(raw.names, length(raw.flags),
raw.numbers .!= typemax(eltype(raw.numbers)),
map(n-> n != typemax(typeof(n)), raw.numbers),
map(!isnothing, raw.strings),
extensions, capabilities)
end
Expand Down
4 changes: 3 additions & 1 deletion base/util.jl
Original file line number Diff line number Diff line change
Expand Up @@ -700,7 +700,9 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2),
catch
buf = PipeBuffer()
original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib")
Base.require(Base, :InteractiveUtils).versioninfo(buf)
let InteractiveUtils = Base.require(Base, :InteractiveUtils)
@invokelatest InteractiveUtils.versioninfo(buf)
end
empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path)
error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" *
"including error messages above and the output of versioninfo():\n$(read(buf, String))")
Expand Down
Loading

3 comments on commit 2defa57

@vtjnash
Copy link
Member

@vtjnash vtjnash commented on 2defa57 Oct 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nanosoldier runbenchmarks("inference", vs="@a127ab7ceeb543483aa8cb86e985c32556c60248")

@vtjnash
Copy link
Member

@vtjnash vtjnash commented on 2defa57 Oct 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nanosoldier runbenchmarks("inference", vs="@a127ab7ceeb543483aa8cb86e985c32556c60248")

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your benchmark job has completed - possible performance regressions were detected. A full report can be found here.

Please sign in to comment.