From 5b0defc0960d7bda07755911258fbe894b0dd4d3 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 16 Sep 2023 12:09:18 -0400 Subject: [PATCH 1/4] Excise REPL from sysimg. --- Makefile | 4 +- base/client.jl | 29 ++++- base/docs/Docs.jl | 2 +- base/sysimg.jl | 46 ++------ base/util.jl | 4 +- contrib/generate_precompile.jl | 169 +++------------------------ doc/Manifest.toml | 27 ++++- pkgimage.mk | 9 +- stdlib/REPL/Project.toml | 2 +- stdlib/REPL/src/REPL.jl | 4 + stdlib/REPL/src/precompile.jl | 207 +++++++++++++++++++++++++++++++++ stdlib/REPL/test/repl.jl | 2 + stdlib/stdlib.mk | 18 +-- test/cmdlineargs.jl | 9 +- test/loading.jl | 2 +- test/precompile.jl | 12 +- 16 files changed, 332 insertions(+), 214 deletions(-) create mode 100644 stdlib/REPL/src/precompile.jl diff --git a/Makefile b/Makefile index 5e9ea8a44c66f..d205a4cf8deb8 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% debug release : % : julia-% stdlibs-cache-% -docs: julia-sysimg-$(JULIA_BUILD_MODE) +docs: julia-sysimg-$(JULIA_BUILD_MODE) stdlibs-cache-$(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no' docs-revise: @@ -184,7 +184,7 @@ $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(buil @$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c) julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia - @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ + @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) JULIA_FALLBACK_REPL=1 WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ $(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \ $(call cygpath_w,$(build_datarootdir)/julia/base.cache)) diff --git a/base/client.jl b/base/client.jl index 88d8321d09278..d6386f624e722 100644 --- a/base/client.jl +++ b/base/client.jl @@ -405,13 +405,32 @@ function load_InteractiveUtils(mod::Module=Main) return getfield(mod, :InteractiveUtils) end +function load_REPL() + # load interactive-only libraries + try + return Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) + catch ex + @warn "Failed to import REPL" exception=(ex, catch_backtrace()) + end + return nothing +end + global active_repl # run the requested sort of evaluation loop on stdio function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool) - load_InteractiveUtils() - - fallback_repl = get_bool_env("JULIA_FALLBACK_REPL", false) + fallback_repl = parse(Bool, get(ENV, "JULIA_FALLBACK_REPL", "false")) + if !fallback_repl && interactive + load_InteractiveUtils() + if !isassigned(REPL_MODULE_REF) + let REPL = load_REPL() + # If someone imported Pkg (which has a dependency on REPL) + # we will side-step the special hook in Base. + REPL_MODULE_REF[] = REPL + end + end + end + # TODO cleanup REPL_MODULE_REF if !fallback_repl && interactive && isassigned(REPL_MODULE_REF) invokelatest(REPL_MODULE_REF[]) do REPL @@ -435,8 +454,8 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_f end else # otherwise provide a simple fallback - if interactive && !quiet - @warn "REPL provider not available: using basic fallback" + if !fallback_repl && interactive && !quiet + @warn "REPL provider not available: using basic fallback" LOAD_PATH=join(Base.LOAD_PATH, Sys.iswindows() ? ';' : ':') end banner == :no || Base.banner(short=banner==:short) let input = stdin diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 28ef5df6d619c..4ca384361c4e3 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -535,7 +535,7 @@ function docm(source::LineNumberNode, mod::Module, ex) elseif isassigned(Base.REPL_MODULE_REF) # TODO: this is a shim to continue to allow `@doc` for looking up docstrings REPL = Base.REPL_MODULE_REF[] - return REPL.lookup_doc(ex) + return invokelatest(REPL.lookup_doc, ex) end return nothing end diff --git a/base/sysimg.jl b/base/sysimg.jl index 8c289bf501618..1bdbe60479e91 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -31,43 +31,19 @@ let # Run with the `--exclude-jlls` option to filter out all JLL packages stdlibs = [ # No dependencies - :ArgTools, - :Artifacts, - :Base64, - :CRC32c, - :FileWatching, - :Libdl, - :Logging, - :Mmap, - :NetworkOptions, - :SHA, - :Serialization, - :Sockets, - :Unicode, + :FileWatching, # used by loading.jl -- implicit assumption that init runs + :Libdl, # Transitive through LinAlg + :Artifacts, # Transitive through LinAlg + :SHA, # transitive through Random + :Sockets, # used by stream.jl + + # Transitive through LingAlg + # OpenBLAS_jll + # libblastrampoline_jll # 1-depth packages - :LinearAlgebra, - :Markdown, - :Printf, - :Random, - :Tar, - - # 2-depth packages - :Dates, - :Future, - :InteractiveUtils, - :LibGit2, - :UUIDs, - - # 3-depth packages - :REPL, - :TOML, - - # 4-depth packages - :LibCURL, - - # 5-depth packages - :Downloads, + :LinearAlgebra, # Commits type-piracy and GEMM + :Random, # Can't be removed due to rand being exported by Base ] # PackageCompiler can filter out stdlibs so it can be empty maxlen = maximum(textwidth.(string.(stdlibs)); init=0) diff --git a/base/util.jl b/base/util.jl index 0a983d454b795..fe04eaf04bd47 100644 --- a/base/util.jl +++ b/base/util.jl @@ -700,7 +700,9 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), catch buf = PipeBuffer() original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib") - Base.require(Base, :InteractiveUtils).versioninfo(buf) + let InteractiveUtils = Base.require(Base, :InteractiveUtils) + @invokelatest InteractiveUtils.versioninfo(buf) + end empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path) error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" * "including error messages above and the output of versioninfo():\n$(read(buf, String))") diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 7df9992c41f16..23320df96e730 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -33,6 +33,19 @@ UP_ARROW = "\e[A" DOWN_ARROW = "\e[B" hardcoded_precompile_statements = """ +precompile(Base.unsafe_string, (Ptr{UInt8},)) +precompile(Base.unsafe_string, (Ptr{Int8},)) + +# loading.jl +precompile(Base.__require_prelocked, (Base.PkgId, Nothing)) +precompile(Base._require, (Base.PkgId, Nothing)) + +# REPL +precompile(isequal, (String, String)) +precompile(Base.check_open, (Base.TTY,)) +precompile(Base.getproperty, (Base.TTY, Symbol)) +precompile(write, (Base.TTY, String)) + # used by Revise.jl precompile(Tuple{typeof(Base.parse_cache_header), String}) precompile(Base.read_dependency_src, (String, String)) @@ -66,30 +79,6 @@ for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base hardcoded_precompile_statements *= "precompile(Tuple{typeof(show), $IO, $T})\n" end -repl_script = """ -2+2 -print("") -printstyled("a", "b") -display([1]) -display([1 2; 3 4]) -foo(x) = 1 -@time @eval foo(1) -; pwd -$CTRL_C -$CTRL_R$CTRL_C -? reinterpret -using Ra\t$CTRL_C -\\alpha\t$CTRL_C -\e[200~paste here ;)\e[201~"$CTRL_C -$UP_ARROW$DOWN_ARROW$CTRL_C -123\b\b\b$CTRL_C -\b\b$CTRL_C -f(x) = x03 -f(1,2) -[][1] -cd("complet_path\t\t$CTRL_C -""" - precompile_script = """ # NOTE: these were moved to the end of Base.jl. TODO: move back here. # # Used by Revise & its dependencies @@ -127,14 +116,6 @@ precompile_script = """ julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) -have_repl = haskey(Base.loaded_modules, - Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")) -if have_repl - hardcoded_precompile_statements *= """ - precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol}) - """ -end - Artifacts = get(Base.loaded_modules, Base.PkgId(Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts"), nothing) @@ -173,27 +154,12 @@ if Libdl !== nothing """ end -InteractiveUtils = get(Base.loaded_modules, - Base.PkgId(Base.UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240"), "InteractiveUtils"), - nothing) -if InteractiveUtils !== nothing - repl_script *= """ - @time_imports using Random - """ -end - -const JULIA_PROMPT = "julia> " -const SHELL_PROMPT = "shell> " -const HELP_PROMPT = "help?> " - # Printing the current state let global print_state print_lk = ReentrantLock() status = Dict{String, String}( "step1" => "W", - "step2" => "W", - "repl" => "0/0", "step3" => "W", "clock" => "◐", ) @@ -214,8 +180,6 @@ let isempty(args) || push!(status, args...) print("\r└ Collect (Basic: ") print_status("step1") - print(", REPL ", status["repl"], ": ") - print_status("step2") print(") => Execute ") print_status("step3") end @@ -230,7 +194,8 @@ procenv = Dict{String,Any}( "JULIA_PROJECT" => nothing, # remove from environment "JULIA_LOAD_PATH" => "@stdlib", "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", - "TERM" => "") + "TERM" => "", + "JULIA_FALLBACK_REPL" => "true") generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed start_time = time_ns() @@ -238,7 +203,6 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Extract the precompile statements from the precompile file statements_step1 = Channel{String}(Inf) - statements_step2 = Channel{String}(Inf) # From hardcoded statements for statement in split(hardcoded_precompile_statements::String, '\n') @@ -253,7 +217,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe anim_chars = ["◐","◓","◑","◒"] current = 1 if fancyprint - while isopen(statements_step2) || !isempty(statements_step2) + while isopen(statements_step1) || !isempty(statements_step1) print_state("clock" => anim_chars[current]) wait(t) current = current == 4 ? 1 : current + 1 @@ -297,105 +261,9 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step1" => "F$n_step1") return :ok end + Base.errormonitor(step1) !PARALLEL_PRECOMPILATION && wait(step1) - step2 = @async mktemp() do precompile_file, precompile_file_h - print_state("step2" => "R") - # Collect statements from running a REPL process and replaying our REPL script - touch(precompile_file) - pts, ptm = open_fake_pty() - if have_repl - cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'` - else - cmdargs = `-e nothing` - end - p = run(addenv(addenv(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg - --cpu-target=native --startup-file=no --color=yes -i $cmdargs```, procenv), - "JULIA_PKG_PRECOMPILE_AUTO" => "0"), - pts, pts, pts; wait=false) - Base.close_stdio(pts) - # Prepare a background process to copy output from process until `pts` is closed - output_copy = Base.BufferStream() - tee = @async try - while !eof(ptm) - l = readavailable(ptm) - write(debug_output, l) - Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue? - write(output_copy, l) - end - catch ex - if !(ex isa Base.IOError && ex.code == Base.UV_EIO) - rethrow() # ignore EIO on ptm after pts dies - end - finally - close(output_copy) - close(ptm) - end - repl_inputter = @async begin - # wait for the definitive prompt before start writing to the TTY - readuntil(output_copy, JULIA_PROMPT) - sleep(0.1) - readavailable(output_copy) - # Input our script - if have_repl - precompile_lines = split(repl_script::String, '\n'; keepempty=false) - curr = 0 - for l in precompile_lines - sleep(0.1) - curr += 1 - print_state("repl" => "$curr/$(length(precompile_lines))") - # consume any other output - bytesavailable(output_copy) > 0 && readavailable(output_copy) - # push our input - write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") - write(ptm, l, "\n") - readuntil(output_copy, "\n") - # wait for the next prompt-like to appear - readuntil(output_copy, "\n") - strbuf = "" - while !eof(output_copy) - strbuf *= String(readavailable(output_copy)) - occursin(JULIA_PROMPT, strbuf) && break - occursin(SHELL_PROMPT, strbuf) && break - occursin(HELP_PROMPT, strbuf) && break - sleep(0.1) - end - end - end - write(ptm, "exit()\n") - wait(tee) - success(p) || Base.pipeline_error(p) - close(ptm) - write(debug_output, "\n#### FINISHED ####\n") - end - - n_step2 = 0 - precompile_copy = Base.BufferStream() - buffer_reader = @async for statement in eachline(precompile_copy) - print_state("step2" => "R$n_step2") - push!(statements_step2, statement) - n_step2 += 1 - end - - open(precompile_file, "r") do io - while true - # We need to allways call eof(io) for bytesavailable(io) to work - eof(io) && istaskdone(repl_inputter) && eof(io) && break - if bytesavailable(io) == 0 - sleep(0.1) - continue - end - write(precompile_copy, readavailable(io)) - end - end - close(precompile_copy) - wait(buffer_reader) - close(statements_step2) - print_state("step2" => "F$n_step2") - return :ok - end - !PARALLEL_PRECOMPILATION && wait(step2) - # Create a staging area where all the loaded packages are available PrecompileStagingArea = Module() for (_pkgid, _mod) in Base.loaded_modules @@ -408,7 +276,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Make statements unique statements = Set{String}() # Execute the precompile statements - for sts in [statements_step1, statements_step2], statement in sts + for sts in [statements_step1,], statement in sts # Main should be completely clean occursin("Main.", statement) && continue Base.in!(statement, statements) && continue @@ -447,7 +315,6 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe n_succeeded > (have_repl ? 650 : 90) || @warn "Only $n_succeeded precompile statements" fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.") - fetch(step2) == :ok || throw("Step 2 of collecting precompiles failed.") tot_time = time_ns() - start_time println("Precompilation complete. Summary:") diff --git a/doc/Manifest.toml b/doc/Manifest.toml index cf50a1d41ddbd..31eb3634fa709 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.0-DEV" +julia_version = "1.11.0-DEV" manifest_format = "2.0" project_hash = "e0c77beb18dc1f6cce661ebd60658c0c1a77390f" @@ -9,6 +9,9 @@ git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" version = "0.0.1" +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -45,9 +48,22 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.21.3" [[deps.LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.1+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -55,6 +71,11 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+1" + [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" @@ -77,7 +98,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] -deps = ["SHA", "Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.SHA]] diff --git a/pkgimage.mk b/pkgimage.mk index 03330e1ea9cc1..9a91488955420 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -11,6 +11,8 @@ export JULIA_LOAD_PATH := @stdlib unexport JULIA_PROJECT := unexport JULIA_BINDIR := +export JULIA_FALLBACK_REPL := true + default: release release: all-release debug: all-debug @@ -82,6 +84,7 @@ $(eval $(call stdlib_builder,Zlib_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,dSFMT_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,libLLVM_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,libblastrampoline_jll,Artifacts Libdl)) +$(eval $(call stdlib_builder,p7zip_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,OpenBLAS_jll,Artifacts Libdl)) $(eval $(call stdlib_builder,Markdown,Base64)) $(eval $(call stdlib_builder,Printf,Unicode)) @@ -97,8 +100,8 @@ $(eval $(call stdlib_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_ $(eval $(call stdlib_builder,Dates,Printf)) $(eval $(call stdlib_builder,Distributed,Random Serialization Sockets)) $(eval $(call stdlib_builder,Future,Random)) -$(eval $(call stdlib_builder,InteractiveUtils,Markdown)) $(eval $(call stdlib_builder,UUIDs,Random SHA)) +$(eval $(call stdlib_builder,InteractiveUtils,Markdown)) # 3-depth packages $(eval $(call stdlib_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl)) @@ -116,7 +119,9 @@ $(eval $(call stdlib_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) $(eval $(call stdlib_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) # 6-depth packages -$(eval $(call stdlib_builder,Pkg,Dates LibGit2 Libdl Logging Printf Random SHA UUIDs)) # Markdown REPL +$(eval $(call stdlib_builder,Pkg, Artifacts Dates Downloads FileWatching LibGit2 Libdl\ + Logging Markdown Printf REPL Random SHA Serialization\ + TOML Tar UUIDs p7zip_jll)) # 7-depth packages $(eval $(call stdlib_builder,LazyArtifacts,Artifacts Pkg)) diff --git a/stdlib/REPL/Project.toml b/stdlib/REPL/Project.toml index 4f77157da0146..2c3ab32fdc327 100644 --- a/stdlib/REPL/Project.toml +++ b/stdlib/REPL/Project.toml @@ -8,8 +8,8 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "Random"] diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 29f4a9780ff99..9120e3451238a 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1531,4 +1531,8 @@ end import .Numbered.numbered_prompt! +if Base.generating_output() + include("precompile.jl") +end + end # module diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl new file mode 100644 index 0000000000000..7985cf426a250 --- /dev/null +++ b/stdlib/REPL/src/precompile.jl @@ -0,0 +1,207 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Precompile +# Can't use this during incremental: `@eval Module() begin`` + +import ..REPL + +# Ugly hack for our cache file to not have a dependency edge on FakePTYs. +Base._track_dependencies[] = false +try + Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl")) + import .FakePTYs: open_fake_pty +finally + Base._track_dependencies[] = true +end +using Base.Meta + +import Markdown + +## Debugging options +# Disable parallel precompiles generation by setting `false` +const PARALLEL_PRECOMPILATION = true + +# View the code sent to the repl by setting this to `stdout` +const debug_output = devnull # or stdout + +CTRL_C = '\x03' +CTRL_R = '\x12' +UP_ARROW = "\e[A" +DOWN_ARROW = "\e[B" + +repl_script = """ +2+2 +print("") +printstyled("a", "b") +display([1]) +display([1 2; 3 4]) +foo(x) = 1 +@time @eval foo(1) +; pwd +$CTRL_C +$CTRL_R$CTRL_C +? reinterpret +using Ra\t$CTRL_C +\\alpha\t$CTRL_C +\e[200~paste here ;)\e[201~"$CTRL_C +$UP_ARROW$DOWN_ARROW$CTRL_C +123\b\b\b$CTRL_C +\b\b$CTRL_C +f(x) = x03 +f(1,2) +[][1] +cd("complet_path\t\t$CTRL_C +""" + +julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) + +const JULIA_PROMPT = "julia> " +const PKG_PROMPT = "pkg> " +const SHELL_PROMPT = "shell> " +const HELP_PROMPT = "help?> " + +blackhole = Sys.isunix() ? "/dev/null" : "nul" +procenv = Dict{String,Any}( + "JULIA_HISTORY" => blackhole, + "JULIA_PROJECT" => nothing, # remove from environment + "JULIA_LOAD_PATH" => "@stdlib", + "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", + "TERM" => "", + "JULIA_FALLBACK_REPL" => "0") # Turn REPL.jl on in subprocess + +generate_precompile_statements() = try + # Extract the precompile statements from the precompile file + statements_step = Channel{String}(Inf) + + step = @async mktemp() do precompile_file, precompile_file_h + # Collect statements from running a REPL process and replaying our REPL script + touch(precompile_file) + pts, ptm = open_fake_pty() + cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'` + p = run(addenv(addenv(```$(julia_exepath()) -O0 --trace-compile=$precompile_file + --cpu-target=native --startup-file=no --compiled-modules=existing --color=yes -i $cmdargs```, procenv), + "JULIA_PKG_PRECOMPILE_AUTO" => "0"), + pts, pts, pts; wait=false) + Base.close_stdio(pts) + # Prepare a background process to copy output from process until `pts` is closed + output_copy = Base.BufferStream() + tee = @async try + while !eof(ptm) + l = readavailable(ptm) + write(debug_output, l) + Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue? + write(output_copy, l) + end + catch ex + if !(ex isa Base.IOError && ex.code == Base.UV_EIO) + rethrow() # ignore EIO on ptm after pts dies + end + finally + close(output_copy) + close(ptm) + end + Base.errormonitor(tee) + repl_inputter = @async begin + # wait for the definitive prompt before start writing to the TTY + readuntil(output_copy, JULIA_PROMPT) + sleep(0.1) + readavailable(output_copy) + # Input our script + precompile_lines = split(repl_script::String, '\n'; keepempty=false) + curr = 0 + for l in precompile_lines + sleep(0.1) + curr += 1 + # consume any other output + bytesavailable(output_copy) > 0 && readavailable(output_copy) + # push our input + write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") + write(ptm, l, "\n") + readuntil(output_copy, "\n") + # wait for the next prompt-like to appear + readuntil(output_copy, "\n") + strbuf = "" + while !eof(output_copy) + strbuf *= String(readavailable(output_copy)) + occursin(JULIA_PROMPT, strbuf) && break + occursin(PKG_PROMPT, strbuf) && break + occursin(SHELL_PROMPT, strbuf) && break + occursin(HELP_PROMPT, strbuf) && break + sleep(0.1) + end + end + write(ptm, "exit()\n") + wait(tee) + success(p) || Base.pipeline_error(p) + close(ptm) + write(debug_output, "\n#### FINISHED ####\n") + end + Base.errormonitor(repl_inputter) + + n_step = 0 + precompile_copy = Base.BufferStream() + buffer_reader = @async for statement in eachline(precompile_copy) + push!(statements_step, statement) + n_step += 1 + end + + open(precompile_file, "r") do io + while true + # We need to allways call eof(io) for bytesavailable(io) to work + eof(io) && istaskdone(repl_inputter) && eof(io) && break + if bytesavailable(io) == 0 + sleep(0.1) + continue + end + write(precompile_copy, readavailable(io)) + end + end + close(precompile_copy) + wait(buffer_reader) + close(statements_step) + return :ok + end + !PARALLEL_PRECOMPILATION && wait(step) + + # Make statements unique + statements = Set{String}() + # Execute the precompile statements + for statement in statements_step + # Main should be completely clean + occursin("Main.", statement) && continue + Base.in!(statement, statements) && continue + try + ps = Meta.parse(statement) + if !isexpr(ps, :call) + # these are typically comments + @debug "skipping statement because it does not parse as an expression" statement + delete!(statements, statement) + continue + end + popfirst!(ps.args) # precompile(...) + ps.head = :tuple + # println(ps) + ps = eval(ps) + if !precompile(ps...) + @warn "Failed to precompile expression" form=statement _module=nothing _file=nothing _line=0 + end + catch ex + # See #28808 + @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0 + end + end + + fetch(step) == :ok || throw("Collecting precompiles failed.") + return nothing +finally + GC.gc(true); GC.gc(false); # reduce memory footprint +end + +generate_precompile_statements() + +# As a last step in system image generation, +# remove some references to build time environment for a more reproducible build. +Base.Filesystem.temp_cleanup_purge(force=true) + +precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol}) +end # Precompile diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index b5365ca4fb98c..0fb41ddacefc7 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -6,6 +6,8 @@ using Random import REPL.LineEdit using Markdown +@test isassigned(Base.REPL_MODULE_REF) + const BASE_TEST_PATH = joinpath(Sys.BINDIR, "..", "share", "julia", "test") isdefined(Main, :FakePTYs) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "FakePTYs.jl")) import .Main.FakePTYs: with_fake_pty diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index f2717a07679e2..99bdefc66fa90 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -1,15 +1,15 @@ STDLIBS_WITHIN_SYSIMG := \ - ArgTools Artifacts Base64 CRC32c FileWatching Libdl NetworkOptions SHA Serialization \ - MbedTLS_jll libblastrampoline_jll OpenBLAS_jll Printf Random Tar LibSSH2_jll LibGit2_jll \ - LinearAlgebra Dates Future LibGit2 UUIDs TOML LibCURL Downloads Dates Logging \ - Sockets Unicode Markdown InteractiveUtils REPL nghttp2_jll LibCURL_jll MozillaCACerts_jll \ - Mmap + Artifacts FileWatching Libdl SHA libblastrampoline_jll OpenBLAS_jll Random \ + LinearAlgebra Sockets INDEPENDENT_STDLIBS := \ - GMP_jll LLVMLibUnwind_jll LibUV_jll LibUnwind_jll OpenLibm_jll PCRE2_jll \ - Zlib_jll dSFMT_jll libLLVM_jll LLD_jll MPFR_jll \ - DelimitedFiles Distributed SharedArrays SparseArrays Statistics Test LazyArtifacts \ - Profile Pkg + ArgTools Base64 CRC32c Dates DelimitedFiles Distributed Downloads Future \ + InteractiveUtils LazyArtifacts LibGit2 LibCURL Logging Markdown Mmap \ + NetworkOptions Profile Printf Pkg REPL Serialization SharedArrays SparseArrays \ + Statistics Tar Test TOML Unicode UUIDs \ + dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ + LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ + MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index b29904fb5eb6c..b51d95669975c 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -140,7 +140,8 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` "JULIA_LOAD_PATH" => "", "JULIA_DEPOT_PATH" => ";:", "HOME" => homedir())) - @test v == ("false\nREPL: InteractiveUtilstrue\n", true) + # @which is undefined + @test_broken v == ("false\nREPL: InteractiveUtilstrue\n", true) end let v = writereadpipeline("println(\"REPL: \", InteractiveUtils)", setenv(`$exename -i -e 'const InteractiveUtils = 3'`, @@ -159,7 +160,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` # make sure this is a non-fatal error and the REPL still loads @test v[1] @test isempty(v[2]) - @test startswith(v[3], "┌ Warning: Failed to import InteractiveUtils into module Main\n") + # Can't load REPL if it's outside the sysimg if we break the load path. + # Need to rewrite this test nicer + # ┌ Warning: REPL provider not available: using basic fallback + # └ @ Base client.jl:459 + @test_broken startswith(v[3], "┌ Warning: Failed to import InteractiveUtils into module Main\n") end real_threads = string(ccall(:jl_cpu_threads, Int32, ())) for nc in ("0", "-2", "x", "2x", " ", "") diff --git a/test/loading.jl b/test/loading.jl index d002d10d0dab3..72a07835dd339 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -60,7 +60,7 @@ let exename = `$(Base.julia_cmd()) --compiled-modules=yes --startup-file=no --co @test !endswith(s_dir, Base.Filesystem.path_separator) end -@test Base.in_sysimage(Base.PkgId(Base.UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4"), "UUIDs")) +@test Base.in_sysimage(Base.PkgId(Base.UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb"), "Libdl")) @test Base.in_sysimage(Base.PkgId(Base.UUID("3a7fdc7e-7467-41b4-9f64-ea033d046d5b"), "NotAPackage")) == false ## Unit tests for safe file operations ## diff --git a/test/precompile.jl b/test/precompile.jl index 4841dfb07ddcd..ccc37a32e41c5 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -407,7 +407,17 @@ precompile_test_harness(false) do dir Base.PkgId(m) => Base.module_build_id(m) end for s in [Symbol(x.name) for x in Base._sysimage_modules if !(x.name in ["Base", "Core", "Main"])]), # plus test module, - Dict(Base.PkgId(Base.root_module(Base, :Test)) => Base.module_build_id(Base.root_module(Base, :Test))) + Dict(Base.PkgId(Base.root_module(Base, :Test)) => Base.module_build_id(Base.root_module(Base, :Test))), + # plus dependencies of test module + Dict(Base.PkgId(Base.root_module(Base, :InteractiveUtils)) => Base.module_build_id(Base.root_module(Base, :InteractiveUtils))), + Dict(Base.PkgId(Base.root_module(Base, :Logging)) => Base.module_build_id(Base.root_module(Base, :Logging))), + Dict(Base.PkgId(Base.root_module(Base, :Random)) => Base.module_build_id(Base.root_module(Base, :Random))), + Dict(Base.PkgId(Base.root_module(Base, :Serialization)) => Base.module_build_id(Base.root_module(Base, :Serialization))), + # and their dependencies + Dict(Base.PkgId(Base.root_module(Base, :SHA)) => Base.module_build_id(Base.root_module(Base, :SHA))), + Dict(Base.PkgId(Base.root_module(Base, :Markdown)) => Base.module_build_id(Base.root_module(Base, :Markdown))), + # and their dependencies + Dict(Base.PkgId(Base.root_module(Base, :Base64)) => Base.module_build_id(Base.root_module(Base, :Base64))), ) @test Dict(modules) == modules_ok From e8fc31768bc287146e5cc504e0e1b777f0be0568 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 20 Sep 2023 18:43:35 -0400 Subject: [PATCH 2/4] Make terminfo easier to compile --- base/terminfo.jl | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/base/terminfo.jl b/base/terminfo.jl index 7bd3151cdb42f..ff7e6fab7f1f7 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -95,8 +95,8 @@ function read(data::IO, ::Type{TermInfoRaw}) throw(ArgumentError("Terminfo did not contain a null byte after the flag section, expected to position the start of the numbers section on an even byte")) end # Numbers, Strings, Table - numbers = reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt))) .|> ltoh - string_indices = reinterpret(UInt16, read(data, string_count * sizeof(UInt16))) .|> ltoh + numbers = map(ltoh, reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt)))) + string_indices = map(ltoh, reinterpret(UInt16, read(data, string_count * sizeof(UInt16)))) strings_table = read(data, table_bytes) strings = map(string_indices) do idx if idx ∉ (0xffff, 0xfffe) @@ -107,7 +107,7 @@ function read(data::IO, ::Type{TermInfoRaw}) end end TermInfoRaw(term_names, flags, numbers, strings, - if !eof(data) extendedterminfo(data; NumInt) end) + if !eof(data) extendedterminfo(data, NumInt) end) end """ @@ -119,7 +119,7 @@ This will accept any terminfo content that conforms with `term(5)`. See also: `read(::IO, ::Type{TermInfoRaw})` """ -function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}}) +function extendedterminfo(data::IO, NumInt::Union{Type{UInt16}, Type{UInt32}}) # Extended info if position(data) % 2 != 0 0x00 == read(data, UInt8) || @@ -138,12 +138,15 @@ function extendedterminfo(data::IO; NumInt::Union{Type{UInt16}, Type{UInt32}}) throw(ArgumentError("Terminfo did not contain a null byte after the extended flag section, expected to position the start of the numbers section on an even byte")) end numbers = map(n -> Int(ltoh(n)), reinterpret(NumInt, read(data, numbers_count * sizeof(NumInt)))) - table_indices = reinterpret(UInt16, read(data, table_count * sizeof(UInt16))) .|> ltoh + table_indices = map(ltoh, reinterpret(UInt16, read(data, table_count * sizeof(UInt16)))) table_strings = [String(readuntil(data, 0x00)) for _ in 1:length(table_indices)] + info = Dict{Symbol, Union{Bool, Int, String}}() strings = table_strings[1:string_count] - labels = Symbol.(table_strings[string_count+1:end]) - Dict{Symbol, Union{Bool, Int, String}}( - labels .=> vcat(flags, numbers, strings)) + labels = table_strings[string_count+1:end] + for (label, val) in zip(labels, vcat(flags, numbers, strings)) + info[Symbol(label)] = val + end + return info end """ @@ -178,7 +181,7 @@ function TermInfo(raw::TermInfoRaw) Symbol[] end TermInfo(raw.names, length(raw.flags), - raw.numbers .!= typemax(eltype(raw.numbers)), + map(n-> n != typemax(typeof(n)), raw.numbers), map(!isnothing, raw.strings), extensions, capabilities) end From e60fb97ac2fd7ddb84ec4479483675ab02b8c627 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 22 Sep 2023 11:45:21 -0400 Subject: [PATCH 3/4] move REPL_MODULE_REF back to REPL.jl --- base/client.jl | 6 +----- base/loading.jl | 3 --- stdlib/REPL/src/REPL.jl | 7 +++++++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/client.jl b/base/client.jl index d6386f624e722..35abb26c7ff43 100644 --- a/base/client.jl +++ b/base/client.jl @@ -423,11 +423,7 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_f if !fallback_repl && interactive load_InteractiveUtils() if !isassigned(REPL_MODULE_REF) - let REPL = load_REPL() - # If someone imported Pkg (which has a dependency on REPL) - # we will side-step the special hook in Base. - REPL_MODULE_REF[] = REPL - end + load_REPL() end end # TODO cleanup REPL_MODULE_REF diff --git a/base/loading.jl b/base/loading.jl index 5ebb5dc120282..eb7503a5e2e85 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1818,9 +1818,6 @@ function __require_prelocked(uuidkey::PkgId, env=nothing) insert_extension_triggers(uuidkey) # After successfully loading, notify downstream consumers run_package_callbacks(uuidkey) - if uuidkey == REPL_PKGID - REPL_MODULE_REF[] = newm - end else newm = root_module(uuidkey) end diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 9120e3451238a..3d9ca0d8375ca 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -14,6 +14,13 @@ REPL.run_repl(repl) """ module REPL +# this assignment won't survive precompilation, +# but will stick if REPL is baked into a sysimg. +Base.REPL_MODULE_REF[] = REPL +function __init__() + Base.REPL_MODULE_REF[] = REPL +end + Base.Experimental.@optlevel 1 Base.Experimental.@max_methods 1 From d25c84e8a907940b3dd46547b66706ffdd9daff9 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 22 Sep 2023 18:57:57 -0400 Subject: [PATCH 4/4] fixup! move REPL_MODULE_REF back to REPL.jl --- stdlib/REPL/src/REPL.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 3d9ca0d8375ca..81aa56631eaaf 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -14,9 +14,6 @@ REPL.run_repl(repl) """ module REPL -# this assignment won't survive precompilation, -# but will stick if REPL is baked into a sysimg. -Base.REPL_MODULE_REF[] = REPL function __init__() Base.REPL_MODULE_REF[] = REPL end @@ -1538,6 +1535,11 @@ end import .Numbered.numbered_prompt! +# this assignment won't survive precompilation, +# but will stick if REPL is baked into a sysimg. +# Needs to occur after this module is finished. +Base.REPL_MODULE_REF[] = REPL + if Base.generating_output() include("precompile.jl") end