From c671969c54e2efe0ac3975b040a4a8eb4f8d04b8 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Fri, 29 Sep 2023 12:22:52 +0800 Subject: [PATCH] Revert "Excise REPL and its dependencies from sysimg (#51399)" This reverts commit 2defa573a0e3d0d206afef53782177143fb1763e. --- Makefile | 4 +- base/client.jl | 25 +--- base/docs/Docs.jl | 2 +- base/loading.jl | 3 + base/sysimg.jl | 46 ++++++-- base/terminfo.jl | 21 ++-- base/util.jl | 4 +- contrib/generate_precompile.jl | 169 ++++++++++++++++++++++++--- doc/Manifest.toml | 27 +---- pkgimage.mk | 9 +- stdlib/REPL/Project.toml | 2 +- stdlib/REPL/src/REPL.jl | 13 --- stdlib/REPL/src/precompile.jl | 207 --------------------------------- stdlib/REPL/test/repl.jl | 2 - stdlib/stdlib.mk | 18 +-- test/cmdlineargs.jl | 9 +- test/loading.jl | 2 +- test/precompile.jl | 12 +- 18 files changed, 226 insertions(+), 349 deletions(-) delete mode 100644 stdlib/REPL/src/precompile.jl diff --git a/Makefile b/Makefile index d205a4cf8deb8c..5e9ea8a44c66ff 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% debug release : % : julia-% stdlibs-cache-% -docs: julia-sysimg-$(JULIA_BUILD_MODE) stdlibs-cache-$(JULIA_BUILD_MODE) +docs: julia-sysimg-$(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no' docs-revise: @@ -184,7 +184,7 @@ $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(buil @$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c) julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia - @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) JULIA_FALLBACK_REPL=1 WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ + @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ $(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \ $(call cygpath_w,$(build_datarootdir)/julia/base.cache)) diff --git a/base/client.jl b/base/client.jl index 35abb26c7ff43d..88d8321d092781 100644 --- a/base/client.jl +++ b/base/client.jl @@ -405,28 +405,13 @@ function load_InteractiveUtils(mod::Module=Main) return getfield(mod, :InteractiveUtils) end -function load_REPL() - # load interactive-only libraries - try - return Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) - catch ex - @warn "Failed to import REPL" exception=(ex, catch_backtrace()) - end - return nothing -end - global active_repl # run the requested sort of evaluation loop on stdio function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool) - fallback_repl = parse(Bool, get(ENV, "JULIA_FALLBACK_REPL", "false")) - if !fallback_repl && interactive - load_InteractiveUtils() - if !isassigned(REPL_MODULE_REF) - load_REPL() - end - end - # TODO cleanup REPL_MODULE_REF + load_InteractiveUtils() + + fallback_repl = get_bool_env("JULIA_FALLBACK_REPL", false) if !fallback_repl && interactive && isassigned(REPL_MODULE_REF) invokelatest(REPL_MODULE_REF[]) do REPL @@ -450,8 +435,8 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_f end else # otherwise provide a simple fallback - if !fallback_repl && interactive && !quiet - @warn "REPL provider not available: using basic fallback" LOAD_PATH=join(Base.LOAD_PATH, Sys.iswindows() ? ';' : ':') + if interactive && !quiet + @warn "REPL provider not available: using basic fallback" end banner == :no || Base.banner(short=banner==:short) let input = stdin diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 4ca384361c4e34..28ef5df6d619c3 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -535,7 +535,7 @@ function docm(source::LineNumberNode, mod::Module, ex) elseif isassigned(Base.REPL_MODULE_REF) # TODO: this is a shim to continue to allow `@doc` for looking up docstrings REPL = Base.REPL_MODULE_REF[] - return invokelatest(REPL.lookup_doc, ex) + return REPL.lookup_doc(ex) end return nothing end diff --git a/base/loading.jl b/base/loading.jl index f1780e089a0773..ed88b515937b4e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1818,6 +1818,9 @@ function __require_prelocked(uuidkey::PkgId, env=nothing) insert_extension_triggers(uuidkey) # After successfully loading, notify downstream consumers run_package_callbacks(uuidkey) + if uuidkey == REPL_PKGID + REPL_MODULE_REF[] = newm + end else newm = root_module(uuidkey) end diff --git a/base/sysimg.jl b/base/sysimg.jl index 1bdbe60479e914..8c289bf501618e 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -31,19 +31,43 @@ let # Run with the `--exclude-jlls` option to filter out all JLL packages stdlibs = [ # No dependencies - :FileWatching, # used by loading.jl -- implicit assumption that init runs - :Libdl, # Transitive through LinAlg - :Artifacts, # Transitive through LinAlg - :SHA, # transitive through Random - :Sockets, # used by stream.jl - - # Transitive through LingAlg - # OpenBLAS_jll - # libblastrampoline_jll + :ArgTools, + :Artifacts, + :Base64, + :CRC32c, + :FileWatching, + :Libdl, + :Logging, + :Mmap, + :NetworkOptions, + :SHA, + :Serialization, + :Sockets, + :Unicode, # 1-depth packages - :LinearAlgebra, # Commits type-piracy and GEMM - :Random, # Can't be removed due to rand being exported by Base + :LinearAlgebra, + :Markdown, + :Printf, + :Random, + :Tar, + + # 2-depth packages + :Dates, + :Future, + :InteractiveUtils, + :LibGit2, + :UUIDs, + + # 3-depth packages + :REPL, + :TOML, + + # 4-depth packages + :LibCURL, + + # 5-depth packages + :Downloads, ] # PackageCompiler can filter out stdlibs so it can be empty maxlen = maximum(textwidth.(string.(stdlibs)); init=0) diff --git a/base/terminfo.jl b/base/terminfo.jl index ff7e6fab7f1f78..7bd3151cdb42f1 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -95,8 +95,8 @@ function read(data::IO, ::Type{TermInfoRaw}) throw(ArgumentError("Terminfo did not contain a null byte after the flag section, expected to position the start of the numbers section on an even byte")) end # Numbers, Strings, Table - numbers = map(ltoh, reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt)))) - string_indices = map(ltoh, reinterpret(UInt16, read(data, string_count * sizeof(UInt16)))) + numbers = reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))) .|> ltoh + string_indices = reinterpret(UInt16, read(data, string_count * sizeof(UInt16))) .|> ltoh strings_table = read(data, table_bytes) strings = map(string_indices) do idx if idx ∉ (0xffff, 0xfffe) @@ -107,7 +107,7 @@ function read(data::IO, ::Type{TermInfoRaw}) end end TermInfoRaw(term_names, flags, numbers, strings, - if !eof(data) extendedterminfo(data, NumInt) end) + if !eof(data) extendedterminfo(data; NumInt) end) end """ @@ -119,7 +119,7 @@ This will accept any terminfo content that conforms with `term(5)`. See also: `read(::IO, ::Type{TermInfoRaw})` """ -function extendedterminfo(data::IO, NumInt::Union{Type{UInt16}, Type{UInt32}}) +function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}}) # Extended info if position(data) % 2 != 0 0x00 == read(data, UInt8) || @@ -138,15 +138,12 @@ function extendedterminfo(data::IO, NumInt::Union{Type{UInt16}, Type{UInt32}}) throw(ArgumentError("Terminfo did not contain a null byte after the extended flag section, expected to position the start of the numbers section on an even byte")) end numbers = map(n -> Int(ltoh(n)), reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt)))) - table_indices = map(ltoh, reinterpret(UInt16, read(data, table_count * sizeof(UInt16)))) + table_indices = reinterpret(UInt16, read(data, table_count * sizeof(UInt16))) .|> ltoh table_strings = [String(readuntil(data, 0x00)) for _ in 1:length(table_indices)] - info = Dict{Symbol, Union{Bool, Int, String}}() strings = table_strings[1:string_count] - labels = table_strings[string_count+1:end] - for (label, val) in zip(labels, vcat(flags, numbers, strings)) - info[Symbol(label)] = val - end - return info + labels = Symbol.(table_strings[string_count+1:end]) + Dict{Symbol, Union{Bool, Int, String}}( + labels .=> vcat(flags, numbers, strings)) end """ @@ -181,7 +178,7 @@ function TermInfo(raw::TermInfoRaw) Symbol[] end TermInfo(raw.names, length(raw.flags), - map(n-> n != typemax(typeof(n)), raw.numbers), + raw.numbers .!= typemax(eltype(raw.numbers)), map(!isnothing, raw.strings), extensions, capabilities) end diff --git a/base/util.jl b/base/util.jl index fe04eaf04bd47f..0a983d454b7953 100644 --- a/base/util.jl +++ b/base/util.jl @@ -700,9 +700,7 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), catch buf = PipeBuffer() original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib") - let InteractiveUtils = Base.require(Base, :InteractiveUtils) - @invokelatest InteractiveUtils.versioninfo(buf) - end + Base.require(Base, :InteractiveUtils).versioninfo(buf) empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path) error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" * "including error messages above and the output of versioninfo():\n$(read(buf, String))") diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 23320df96e7302..7df9992c41f169 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -33,19 +33,6 @@ UP_ARROW = "\e[A" DOWN_ARROW = "\e[B" hardcoded_precompile_statements = """ -precompile(Base.unsafe_string, (Ptr{UInt8},)) -precompile(Base.unsafe_string, (Ptr{Int8},)) - -# loading.jl -precompile(Base.__require_prelocked, (Base.PkgId, Nothing)) -precompile(Base._require, (Base.PkgId, Nothing)) - -# REPL -precompile(isequal, (String, String)) -precompile(Base.check_open, (Base.TTY,)) -precompile(Base.getproperty, (Base.TTY, Symbol)) -precompile(write, (Base.TTY, String)) - # used by Revise.jl precompile(Tuple{typeof(Base.parse_cache_header), String}) precompile(Base.read_dependency_src, (String, String)) @@ -79,6 +66,30 @@ for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base hardcoded_precompile_statements *= "precompile(Tuple{typeof(show), $IO, $T})\n" end +repl_script = """ +2+2 +print("") +printstyled("a", "b") +display([1]) +display([1 2; 3 4]) +foo(x) = 1 +@time @eval foo(1) +; pwd +$CTRL_C +$CTRL_R$CTRL_C +? reinterpret +using Ra\t$CTRL_C +\\alpha\t$CTRL_C +\e[200~paste here ;)\e[201~"$CTRL_C +$UP_ARROW$DOWN_ARROW$CTRL_C +123\b\b\b$CTRL_C +\b\b$CTRL_C +f(x) = x03 +f(1,2) +[][1] +cd("complet_path\t\t$CTRL_C +""" + precompile_script = """ # NOTE: these were moved to the end of Base.jl. TODO: move back here. # # Used by Revise & its dependencies @@ -116,6 +127,14 @@ precompile_script = """ julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) +have_repl = haskey(Base.loaded_modules, + Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")) +if have_repl + hardcoded_precompile_statements *= """ + precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol}) + """ +end + Artifacts = get(Base.loaded_modules, Base.PkgId(Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts"), nothing) @@ -154,12 +173,27 @@ if Libdl !== nothing """ end +InteractiveUtils = get(Base.loaded_modules, + Base.PkgId(Base.UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240"), "InteractiveUtils"), + nothing) +if InteractiveUtils !== nothing + repl_script *= """ + @time_imports using Random + """ +end + +const JULIA_PROMPT = "julia> " +const SHELL_PROMPT = "shell> " +const HELP_PROMPT = "help?> " + # Printing the current state let global print_state print_lk = ReentrantLock() status = Dict{String, String}( "step1" => "W", + "step2" => "W", + "repl" => "0/0", "step3" => "W", "clock" => "◐", ) @@ -180,6 +214,8 @@ let isempty(args) || push!(status, args...) print("\r└ Collect (Basic: ") print_status("step1") + print(", REPL ", status["repl"], ": ") + print_status("step2") print(") => Execute ") print_status("step3") end @@ -194,8 +230,7 @@ procenv = Dict{String,Any}( "JULIA_PROJECT" => nothing, # remove from environment "JULIA_LOAD_PATH" => "@stdlib", "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", - "TERM" => "", - "JULIA_FALLBACK_REPL" => "true") + "TERM" => "") generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed start_time = time_ns() @@ -203,6 +238,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Extract the precompile statements from the precompile file statements_step1 = Channel{String}(Inf) + statements_step2 = Channel{String}(Inf) # From hardcoded statements for statement in split(hardcoded_precompile_statements::String, '\n') @@ -217,7 +253,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe anim_chars = ["◐","◓","◑","◒"] current = 1 if fancyprint - while isopen(statements_step1) || !isempty(statements_step1) + while isopen(statements_step2) || !isempty(statements_step2) print_state("clock" => anim_chars[current]) wait(t) current = current == 4 ? 1 : current + 1 @@ -261,9 +297,105 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step1" => "F$n_step1") return :ok end - Base.errormonitor(step1) !PARALLEL_PRECOMPILATION && wait(step1) + step2 = @async mktemp() do precompile_file, precompile_file_h + print_state("step2" => "R") + # Collect statements from running a REPL process and replaying our REPL script + touch(precompile_file) + pts, ptm = open_fake_pty() + if have_repl + cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'` + else + cmdargs = `-e nothing` + end + p = run(addenv(addenv(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg + --cpu-target=native --startup-file=no --color=yes -i $cmdargs```, procenv), + "JULIA_PKG_PRECOMPILE_AUTO" => "0"), + pts, pts, pts; wait=false) + Base.close_stdio(pts) + # Prepare a background process to copy output from process until `pts` is closed + output_copy = Base.BufferStream() + tee = @async try + while !eof(ptm) + l = readavailable(ptm) + write(debug_output, l) + Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue? + write(output_copy, l) + end + catch ex + if !(ex isa Base.IOError && ex.code == Base.UV_EIO) + rethrow() # ignore EIO on ptm after pts dies + end + finally + close(output_copy) + close(ptm) + end + repl_inputter = @async begin + # wait for the definitive prompt before start writing to the TTY + readuntil(output_copy, JULIA_PROMPT) + sleep(0.1) + readavailable(output_copy) + # Input our script + if have_repl + precompile_lines = split(repl_script::String, '\n'; keepempty=false) + curr = 0 + for l in precompile_lines + sleep(0.1) + curr += 1 + print_state("repl" => "$curr/$(length(precompile_lines))") + # consume any other output + bytesavailable(output_copy) > 0 && readavailable(output_copy) + # push our input + write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") + write(ptm, l, "\n") + readuntil(output_copy, "\n") + # wait for the next prompt-like to appear + readuntil(output_copy, "\n") + strbuf = "" + while !eof(output_copy) + strbuf *= String(readavailable(output_copy)) + occursin(JULIA_PROMPT, strbuf) && break + occursin(SHELL_PROMPT, strbuf) && break + occursin(HELP_PROMPT, strbuf) && break + sleep(0.1) + end + end + end + write(ptm, "exit()\n") + wait(tee) + success(p) || Base.pipeline_error(p) + close(ptm) + write(debug_output, "\n#### FINISHED ####\n") + end + + n_step2 = 0 + precompile_copy = Base.BufferStream() + buffer_reader = @async for statement in eachline(precompile_copy) + print_state("step2" => "R$n_step2") + push!(statements_step2, statement) + n_step2 += 1 + end + + open(precompile_file, "r") do io + while true + # We need to allways call eof(io) for bytesavailable(io) to work + eof(io) && istaskdone(repl_inputter) && eof(io) && break + if bytesavailable(io) == 0 + sleep(0.1) + continue + end + write(precompile_copy, readavailable(io)) + end + end + close(precompile_copy) + wait(buffer_reader) + close(statements_step2) + print_state("step2" => "F$n_step2") + return :ok + end + !PARALLEL_PRECOMPILATION && wait(step2) + # Create a staging area where all the loaded packages are available PrecompileStagingArea = Module() for (_pkgid, _mod) in Base.loaded_modules @@ -276,7 +408,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Make statements unique statements = Set{String}() # Execute the precompile statements - for sts in [statements_step1,], statement in sts + for sts in [statements_step1, statements_step2], statement in sts # Main should be completely clean occursin("Main.", statement) && continue Base.in!(statement, statements) && continue @@ -315,6 +447,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe n_succeeded > (have_repl ? 650 : 90) || @warn "Only $n_succeeded precompile statements" fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.") + fetch(step2) == :ok || throw("Step 2 of collecting precompiles failed.") tot_time = time_ns() - start_time println("Precompilation complete. Summary:") diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 31eb3634fa7093..cf50a1d41ddbd2 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.0-DEV" +julia_version = "1.9.0-DEV" manifest_format = "2.0" project_hash = "e0c77beb18dc1f6cce661ebd60658c0c1a77390f" @@ -9,9 +9,6 @@ git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" version = "0.0.1" -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -48,22 +45,9 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.21.3" [[deps.LibGit2]] -deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.1+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.0+1" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -71,11 +55,6 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -[[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+1" - [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" @@ -98,7 +77,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] -deps = ["SHA"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.SHA]] diff --git a/pkgimage.mk b/pkgimage.mk index 9a91488955420a..03330e1ea9cc17 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -11,8 +11,6 @@ export JULIA_LOAD_PATH := @stdlib unexport JULIA_PROJECT := unexport JULIA_BINDIR := -export JULIA_FALLBACK_REPL := true - default: release release: all-release debug: all-debug @@ -84,7 +82,6 @@ $(eval $(call stdlib_builder,Zlib_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,dSFMT_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,libLLVM_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,libblastrampoline_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,p7zip_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,OpenBLAS_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,Markdown,Base64)) $(eval $(call stdlib_builder,Printf,Unicode)) @@ -100,8 +97,8 @@ $(eval $(call stdlib_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_ $(eval $(call stdlib_builder,Dates,Printf)) $(eval $(call stdlib_builder,Distributed,Random Serialization Sockets)) $(eval $(call stdlib_builder,Future,Random)) -$(eval $(call stdlib_builder,UUIDs,Random SHA)) $(eval $(call stdlib_builder,InteractiveUtils,Markdown)) +$(eval $(call stdlib_builder,UUIDs,Random SHA)) # 3-depth packages $(eval $(call stdlib_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl)) @@ -119,9 +116,7 @@ $(eval $(call stdlib_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) $(eval $(call stdlib_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) # 6-depth packages -$(eval $(call stdlib_builder,Pkg, Artifacts Dates Downloads FileWatching LibGit2 Libdl\ - Logging Markdown Printf REPL Random SHA Serialization\ - TOML Tar UUIDs p7zip_jll)) +$(eval $(call stdlib_builder,Pkg,Dates LibGit2 Libdl Logging Printf Random SHA UUIDs)) # Markdown REPL # 7-depth packages $(eval $(call stdlib_builder,LazyArtifacts,Artifacts Pkg)) diff --git a/stdlib/REPL/Project.toml b/stdlib/REPL/Project.toml index 2c3ab32fdc3272..4f77157da01461 100644 --- a/stdlib/REPL/Project.toml +++ b/stdlib/REPL/Project.toml @@ -8,8 +8,8 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [extras] -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [targets] test = ["Test", "Random"] diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 81aa56631eaaf6..29f4a9780ff992 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -14,10 +14,6 @@ REPL.run_repl(repl) """ module REPL -function __init__() - Base.REPL_MODULE_REF[] = REPL -end - Base.Experimental.@optlevel 1 Base.Experimental.@max_methods 1 @@ -1535,13 +1531,4 @@ end import .Numbered.numbered_prompt! -# this assignment won't survive precompilation, -# but will stick if REPL is baked into a sysimg. -# Needs to occur after this module is finished. -Base.REPL_MODULE_REF[] = REPL - -if Base.generating_output() - include("precompile.jl") -end - end # module diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl deleted file mode 100644 index 7985cf426a250f..00000000000000 --- a/stdlib/REPL/src/precompile.jl +++ /dev/null @@ -1,207 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -module Precompile -# Can't use this during incremental: `@eval Module() begin`` - -import ..REPL - -# Ugly hack for our cache file to not have a dependency edge on FakePTYs. -Base._track_dependencies[] = false -try - Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl")) - import .FakePTYs: open_fake_pty -finally - Base._track_dependencies[] = true -end -using Base.Meta - -import Markdown - -## Debugging options -# Disable parallel precompiles generation by setting `false` -const PARALLEL_PRECOMPILATION = true - -# View the code sent to the repl by setting this to `stdout` -const debug_output = devnull # or stdout - -CTRL_C = '\x03' -CTRL_R = '\x12' -UP_ARROW = "\e[A" -DOWN_ARROW = "\e[B" - -repl_script = """ -2+2 -print("") -printstyled("a", "b") -display([1]) -display([1 2; 3 4]) -foo(x) = 1 -@time @eval foo(1) -; pwd -$CTRL_C -$CTRL_R$CTRL_C -? reinterpret -using Ra\t$CTRL_C -\\alpha\t$CTRL_C -\e[200~paste here ;)\e[201~"$CTRL_C -$UP_ARROW$DOWN_ARROW$CTRL_C -123\b\b\b$CTRL_C -\b\b$CTRL_C -f(x) = x03 -f(1,2) -[][1] -cd("complet_path\t\t$CTRL_C -""" - -julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) - -const JULIA_PROMPT = "julia> " -const PKG_PROMPT = "pkg> " -const SHELL_PROMPT = "shell> " -const HELP_PROMPT = "help?> " - -blackhole = Sys.isunix() ? "/dev/null" : "nul" -procenv = Dict{String,Any}( - "JULIA_HISTORY" => blackhole, - "JULIA_PROJECT" => nothing, # remove from environment - "JULIA_LOAD_PATH" => "@stdlib", - "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", - "TERM" => "", - "JULIA_FALLBACK_REPL" => "0") # Turn REPL.jl on in subprocess - -generate_precompile_statements() = try - # Extract the precompile statements from the precompile file - statements_step = Channel{String}(Inf) - - step = @async mktemp() do precompile_file, precompile_file_h - # Collect statements from running a REPL process and replaying our REPL script - touch(precompile_file) - pts, ptm = open_fake_pty() - cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'` - p = run(addenv(addenv(```$(julia_exepath()) -O0 --trace-compile=$precompile_file - --cpu-target=native --startup-file=no --compiled-modules=existing --color=yes -i $cmdargs```, procenv), - "JULIA_PKG_PRECOMPILE_AUTO" => "0"), - pts, pts, pts; wait=false) - Base.close_stdio(pts) - # Prepare a background process to copy output from process until `pts` is closed - output_copy = Base.BufferStream() - tee = @async try - while !eof(ptm) - l = readavailable(ptm) - write(debug_output, l) - Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue? - write(output_copy, l) - end - catch ex - if !(ex isa Base.IOError && ex.code == Base.UV_EIO) - rethrow() # ignore EIO on ptm after pts dies - end - finally - close(output_copy) - close(ptm) - end - Base.errormonitor(tee) - repl_inputter = @async begin - # wait for the definitive prompt before start writing to the TTY - readuntil(output_copy, JULIA_PROMPT) - sleep(0.1) - readavailable(output_copy) - # Input our script - precompile_lines = split(repl_script::String, '\n'; keepempty=false) - curr = 0 - for l in precompile_lines - sleep(0.1) - curr += 1 - # consume any other output - bytesavailable(output_copy) > 0 && readavailable(output_copy) - # push our input - write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") - write(ptm, l, "\n") - readuntil(output_copy, "\n") - # wait for the next prompt-like to appear - readuntil(output_copy, "\n") - strbuf = "" - while !eof(output_copy) - strbuf *= String(readavailable(output_copy)) - occursin(JULIA_PROMPT, strbuf) && break - occursin(PKG_PROMPT, strbuf) && break - occursin(SHELL_PROMPT, strbuf) && break - occursin(HELP_PROMPT, strbuf) && break - sleep(0.1) - end - end - write(ptm, "exit()\n") - wait(tee) - success(p) || Base.pipeline_error(p) - close(ptm) - write(debug_output, "\n#### FINISHED ####\n") - end - Base.errormonitor(repl_inputter) - - n_step = 0 - precompile_copy = Base.BufferStream() - buffer_reader = @async for statement in eachline(precompile_copy) - push!(statements_step, statement) - n_step += 1 - end - - open(precompile_file, "r") do io - while true - # We need to allways call eof(io) for bytesavailable(io) to work - eof(io) && istaskdone(repl_inputter) && eof(io) && break - if bytesavailable(io) == 0 - sleep(0.1) - continue - end - write(precompile_copy, readavailable(io)) - end - end - close(precompile_copy) - wait(buffer_reader) - close(statements_step) - return :ok - end - !PARALLEL_PRECOMPILATION && wait(step) - - # Make statements unique - statements = Set{String}() - # Execute the precompile statements - for statement in statements_step - # Main should be completely clean - occursin("Main.", statement) && continue - Base.in!(statement, statements) && continue - try - ps = Meta.parse(statement) - if !isexpr(ps, :call) - # these are typically comments - @debug "skipping statement because it does not parse as an expression" statement - delete!(statements, statement) - continue - end - popfirst!(ps.args) # precompile(...) - ps.head = :tuple - # println(ps) - ps = eval(ps) - if !precompile(ps...) - @warn "Failed to precompile expression" form=statement _module=nothing _file=nothing _line=0 - end - catch ex - # See #28808 - @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0 - end - end - - fetch(step) == :ok || throw("Collecting precompiles failed.") - return nothing -finally - GC.gc(true); GC.gc(false); # reduce memory footprint -end - -generate_precompile_statements() - -# As a last step in system image generation, -# remove some references to build time environment for a more reproducible build. -Base.Filesystem.temp_cleanup_purge(force=true) - -precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol}) -end # Precompile diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 0fb41ddacefc7f..b5365ca4fb98c1 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -6,8 +6,6 @@ using Random import REPL.LineEdit using Markdown -@test isassigned(Base.REPL_MODULE_REF) - const BASE_TEST_PATH = joinpath(Sys.BINDIR, "..", "share", "julia", "test") isdefined(Main, :FakePTYs) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "FakePTYs.jl")) import .Main.FakePTYs: with_fake_pty diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index 99bdefc66fa90e..f2717a07679e25 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -1,15 +1,15 @@ STDLIBS_WITHIN_SYSIMG := \ - Artifacts FileWatching Libdl SHA libblastrampoline_jll OpenBLAS_jll Random \ - LinearAlgebra Sockets + ArgTools Artifacts Base64 CRC32c FileWatching Libdl NetworkOptions SHA Serialization \ + MbedTLS_jll libblastrampoline_jll OpenBLAS_jll Printf Random Tar LibSSH2_jll LibGit2_jll \ + LinearAlgebra Dates Future LibGit2 UUIDs TOML LibCURL Downloads Dates Logging \ + Sockets Unicode Markdown InteractiveUtils REPL nghttp2_jll LibCURL_jll MozillaCACerts_jll \ + Mmap INDEPENDENT_STDLIBS := \ - ArgTools Base64 CRC32c Dates DelimitedFiles Distributed Downloads Future \ - InteractiveUtils LazyArtifacts LibGit2 LibCURL Logging Markdown Mmap \ - NetworkOptions Profile Printf Pkg REPL Serialization SharedArrays SparseArrays \ - Statistics Tar Test TOML Unicode UUIDs \ - dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ - LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ - MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll + GMP_jll LLVMLibUnwind_jll LibUV_jll LibUnwind_jll OpenLibm_jll PCRE2_jll \ + Zlib_jll dSFMT_jll libLLVM_jll LLD_jll MPFR_jll \ + DelimitedFiles Distributed SharedArrays SparseArrays Statistics Test LazyArtifacts \ + Profile Pkg STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index b51d95669975c1..b29904fb5eb6ca 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -140,8 +140,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` "JULIA_LOAD_PATH" => "", "JULIA_DEPOT_PATH" => ";:", "HOME" => homedir())) - # @which is undefined - @test_broken v == ("false\nREPL: InteractiveUtilstrue\n", true) + @test v == ("false\nREPL: InteractiveUtilstrue\n", true) end let v = writereadpipeline("println(\"REPL: \", InteractiveUtils)", setenv(`$exename -i -e 'const InteractiveUtils = 3'`, @@ -160,11 +159,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` # make sure this is a non-fatal error and the REPL still loads @test v[1] @test isempty(v[2]) - # Can't load REPL if it's outside the sysimg if we break the load path. - # Need to rewrite this test nicer - # ┌ Warning: REPL provider not available: using basic fallback - # └ @ Base client.jl:459 - @test_broken startswith(v[3], "┌ Warning: Failed to import InteractiveUtils into module Main\n") + @test startswith(v[3], "┌ Warning: Failed to import InteractiveUtils into module Main\n") end real_threads = string(ccall(:jl_cpu_threads, Int32, ())) for nc in ("0", "-2", "x", "2x", " ", "") diff --git a/test/loading.jl b/test/loading.jl index 72a07835dd3390..d002d10d0dab36 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -60,7 +60,7 @@ let exename = `$(Base.julia_cmd()) --compiled-modules=yes --startup-file=no --co @test !endswith(s_dir, Base.Filesystem.path_separator) end -@test Base.in_sysimage(Base.PkgId(Base.UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb"), "Libdl")) +@test Base.in_sysimage(Base.PkgId(Base.UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4"), "UUIDs")) @test Base.in_sysimage(Base.PkgId(Base.UUID("3a7fdc7e-7467-41b4-9f64-ea033d046d5b"), "NotAPackage")) == false ## Unit tests for safe file operations ## diff --git a/test/precompile.jl b/test/precompile.jl index ccc37a32e41c58..4841dfb07ddcdd 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -407,17 +407,7 @@ precompile_test_harness(false) do dir Base.PkgId(m) => Base.module_build_id(m) end for s in [Symbol(x.name) for x in Base._sysimage_modules if !(x.name in ["Base", "Core", "Main"])]), # plus test module, - Dict(Base.PkgId(Base.root_module(Base, :Test)) => Base.module_build_id(Base.root_module(Base, :Test))), - # plus dependencies of test module - Dict(Base.PkgId(Base.root_module(Base, :InteractiveUtils)) => Base.module_build_id(Base.root_module(Base, :InteractiveUtils))), - Dict(Base.PkgId(Base.root_module(Base, :Logging)) => Base.module_build_id(Base.root_module(Base, :Logging))), - Dict(Base.PkgId(Base.root_module(Base, :Random)) => Base.module_build_id(Base.root_module(Base, :Random))), - Dict(Base.PkgId(Base.root_module(Base, :Serialization)) => Base.module_build_id(Base.root_module(Base, :Serialization))), - # and their dependencies - Dict(Base.PkgId(Base.root_module(Base, :SHA)) => Base.module_build_id(Base.root_module(Base, :SHA))), - Dict(Base.PkgId(Base.root_module(Base, :Markdown)) => Base.module_build_id(Base.root_module(Base, :Markdown))), - # and their dependencies - Dict(Base.PkgId(Base.root_module(Base, :Base64)) => Base.module_build_id(Base.root_module(Base, :Base64))), + Dict(Base.PkgId(Base.root_module(Base, :Test)) => Base.module_build_id(Base.root_module(Base, :Test))) ) @test Dict(modules) == modules_ok