From 05b57dec148027065a810817653686943b93b9cf Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Thu, 8 Jul 2021 04:51:03 +0200 Subject: [PATCH] Support prerendering of code blocks. Passing `prerender=true` to `Documenter.HTML` now renders syntax highlighting of code blocks when building the page rather than in the browser when the page is loaded. This requires a `node` (NodeJS) executable in PATH or that a path to an executable is passed as the `node` keyword argument. The `highlightjs` keyword argument can be used to give a (local) file path to a custrom hightlight.js library. Example configuration: using Documenter, NodeJS_16_jll makedocs(; format = Documenter.HTML( prerender = true, # enable prerendering node = NodeJS_16_jll.node(), # specify node executable (required if not available in PATH) # ... ) # ... ) This feature is marked as experimental for now. --- .github/workflows/CI.yml | 31 +++++++++- .gitignore | 1 + CHANGELOG.md | 16 ++++++ src/Writers/HTMLWriter.jl | 110 ++++++++++++++++++++++++++++++++---- test/prerender/Project.toml | 3 + test/prerender/prerender.jl | 99 ++++++++++++++++++++++++++++++++ test/prerender/src/index.md | 22 ++++++++ 7 files changed, 271 insertions(+), 11 deletions(-) create mode 100644 test/prerender/Project.toml create mode 100644 test/prerender/prerender.jl create mode 100644 test/prerender/src/index.md diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7c21f4590f..2cf1a1e555 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -74,7 +74,7 @@ jobs: Pkg.instantiate() Pkg.develop(PackageSpec(path=pwd())) Pkg.add(["IOCapture", "DocumenterMarkdown"])' - - run: julia --project=test/examples test/examples/tests_latex.jl + - run: julia --project=test/examples --code-coverage test/examples/tests_latex.jl - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: @@ -105,6 +105,35 @@ jobs: Pkg.develop(PackageSpec(path=pwd()))' - run: julia --project=test/themes test/themes/themes.jl + prerender: + name: "NodeJS for prerender" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - run: | + julia --project=test/prerender/ -e ' + using Pkg + Pkg.instantiate() + Pkg.develop(PackageSpec(path=pwd()))' + - run: julia --project=test/prerender --code-coverage test/prerender/prerender.jl + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info + docs: name: 'Documentation: ${{ matrix.makejl }}' runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 1b98cd498b..48273d5142 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ test/formats/builds/ test/missingdocs/build/ test/nongit/build/ test/errors/build/ +test/prerender/build/ test/workdir/builds/ docs/build/ docs/pdf/build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c1e1c729..faf42be4fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ * ![Feature][badge-feature] `@example`- and `@repl`-blocks now support colored output by mapping ANSI escape sequences to HTML. This requires Julia >= 1.6 and passing `ansicolor=true` to `Documenter.HTML` (e.g. `makedocs(format=Documenter.HTML(ansicolor=true, ...), ...)`). In Documenter 0.28.0 this will be the default so to (preemptively) opt-out pass `ansicolor=false`. ([#1441][github-1441], [#1628][github-1628]) +* ![Experimental][badge-experimental] ![Feature][badge-feature] Documenter's HTML output can now prerender syntax highlighting of code blocks, i.e. syntax highlighting is applied when generating the HTML page rather than on the fly in the browser after the page is loaded. This requires (i) passing `prerender=true` to `Documenter.HTML` and (ii) a `node` (NodeJS) executable available in `PATH`. A path to a `node` executable can be specified by passing the `node` keyword argument to `Documenter.HTML` (for example from the `NodeJS_16_jll` Julia package). In addition, the `highlightjs` keyword argument can be used to specify a file path to a highlight.js library (if this is not given the release used by Documenter will be used). Example configuration: + ```julia + using Documenter, NodeJS_16_jll + + makedocs(; + format = Documenter.HTML( + prerender = true, # enable prerendering + node = NodeJS_16_jll.node(), # specify node executable (required if not available in PATH) + # ... + ) + # ... + ) + ``` + _This feature is experimental and subject to change in future releases._ ([#1627][github-1627]) + * ![Enhancement][badge-enhancement] The sandbox module used for evaluating `@repl` and `@example` blocks is now removed (replaced with `Main`) in text output. ([#1633][github-1633]) * ![Enhancement][badge-enhancement] `@repl`, `@example`, and `@eval` blocks now have `LineNumberNodes` inserted such that e.g. `@__FILE__` and `@__LINE__` give better output and not just `"none"` for the file and `1` for the line. This requires Julia 1.6 or higher (no change on earlier Julia versions). ([#1634][github-1634]) @@ -856,6 +871,7 @@ [github-1616]: https://github.com/JuliaDocs/Documenter.jl/pull/1616 [github-1617]: https://github.com/JuliaDocs/Documenter.jl/pull/1617 [github-1625]: https://github.com/JuliaDocs/Documenter.jl/pull/1625 +[github-1627]: https://github.com/JuliaDocs/Documenter.jl/pull/1627 [github-1628]: https://github.com/JuliaDocs/Documenter.jl/pull/1628 [github-1633]: https://github.com/JuliaDocs/Documenter.jl/pull/1633 [github-1634]: https://github.com/JuliaDocs/Documenter.jl/pull/1634 diff --git a/src/Writers/HTMLWriter.jl b/src/Writers/HTMLWriter.jl index ad0477adce..7aa1d5c50a 100644 --- a/src/Writers/HTMLWriter.jl +++ b/src/Writers/HTMLWriter.jl @@ -325,7 +325,7 @@ Defaults to `true`. **`highlights`** can be used to add highlighting for additional languages. By default, Documenter already highlights all the ["Common" highlight.js](https://highlightjs.org/download/) -languages and Julia (`julia`, `julia-repl`). Additional languages must be specified by" +languages and Julia (`julia`, `julia-repl`). Additional languages must be specified by their filenames as they appear on [CDNJS](https://cdnjs.com/libraries/highlight.js) for the highlight.js version Documenter is using. E.g. to include highlighting for YAML and LLVM IR, you would set `highlights = ["llvm", "yaml"]`. Note that no verification is done whether the @@ -349,6 +349,16 @@ and the [Julia Programming Language](https://julialang.org/)."`. **`warn_outdated`** inserts a warning if the current page is not the newest version of the documentation. +## Experimental options + +**`prerender`** a boolean (`true` or `false` (default)) for enabling prerendering/build +time application of syntax highlighting of code blocks. Requires a `node` (NodeJS) +executable to be available in `PATH` or to be passed as the `node` keyword. + +**`node`** path to a `node` (NodeJS) executable used for prerendering. + +**`highlightjs`** file path to custom highglight.js library to be used with prerendering. + # Default and custom assets Documenter copies all files under the source directory (e.g. `/docs/src/`) over @@ -395,6 +405,9 @@ struct HTML <: Documenter.Writer ansicolor :: Bool lang :: String warn_outdated :: Bool + prerender :: Bool + node :: Union{Cmd,String,Nothing} + highlightjs :: Union{String,Nothing} function HTML(; prettyurls :: Bool = true, @@ -409,12 +422,21 @@ struct HTML <: Documenter.Writer mathengine :: Union{MathEngine,Nothing} = KaTeX(), footer :: Union{String, Nothing} = "Powered by [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl) and the [Julia Programming Language](https://julialang.org/).", ansicolor :: Bool = false, # true in 0.28 + lang :: String = "en", + warn_outdated :: Bool = true, + + # experimental keywords + prerender :: Bool = false, + node :: Union{Cmd,String,Nothing} = nothing, + highlightjs :: Union{String,Nothing} = nothing, + # deprecated keywords edit_branch :: Union{String, Nothing, Default} = Default(nothing), - lang :: String = "en", - warn_outdated :: Bool = true ) collapselevel >= 1 || throw(ArgumentError("collapselevel must be >= 1")) + if prerender + prerender, node, highlightjs = prepare_prerendering(prerender, node, highlightjs, highlights) + end assets = map(assets) do asset isa(asset, HTMLAsset) && return asset isa(asset, AbstractString) && return HTMLAsset(assetclass(asset), asset, true) @@ -440,8 +462,49 @@ struct HTML <: Documenter.Writer end isa(edit_link, Default) && (edit_link = edit_link[]) new(prettyurls, disable_git, edit_link, canonical, assets, analytics, - collapselevel, sidebar_sitename, highlights, mathengine, footer, ansicolor, lang, warn_outdated) + collapselevel, sidebar_sitename, highlights, mathengine, footer, + ansicolor, lang, warn_outdated, prerender, node, highlightjs) + end +end + +# Cache of downloaded highlight.js bundles +const HLJSFILES = Dict{String,String}() +# Look for node and highlight.js +function prepare_prerendering(prerender, node, highlightjs, highlights) + node = node === nothing ? Sys.which("node") : node + if node === nothing + @error "HTMLWriter: no node executable given or found on the system. Setting `prerender=false`." + return false, node, highlightjs + end + if !success(`$node --version`) + @error "HTMLWriter: bad node executable at $node. Setting `prerender=false`." + return false, node, highlightjs + end + if highlightjs === nothing + # Try to download + curl = Sys.which("curl") + if curl === nothing + @error "HTMLWriter: no highlight.js file given and no curl executable found " * + "on the system. Setting `prerender=false`." + return false, node, highlightjs + end + @debug "HTMLWriter: downloading highlightjs" + r = Utilities.JSDependencies.RequireJS([]) + RD.highlightjs!(r, highlights) + libs = sort!(collect(r.libraries); by = first) # puts highlight first + key = join((x.first for x in libs), ',') + highlightjs = get!(HLJSFILES, key) do + path, io = mktemp() + for lib in libs + println(io, "// $(lib.first)") + run(pipeline(`$(curl) -fsSL $(lib.second.url)`; stdout=io)) + println(io) + end + close(io) + return path + end end + return prerender, node, highlightjs end "Provides a namespace for remote dependencies." @@ -485,7 +548,7 @@ module RD "highlight", "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/$(hljs_version)/highlight.min.js" )) - prepend!(languages, ["julia", "julia-repl"]) + languages = ["julia", "julia-repl", languages...] for language in languages language = jsescape(language) push!(r, RemoteLibrary( @@ -663,7 +726,9 @@ function render(doc::Documents.Document, settings::HTML=HTML()) RD.jquery, RD.jqueryui, RD.headroom, RD.headroom_jquery, ]) RD.mathengine!(r, settings.mathengine) - RD.highlightjs!(r, settings.highlights) + if !settings.prerender + RD.highlightjs!(r, settings.highlights) + end for filename in readdir(joinpath(ASSETS, "js")) path = joinpath(ASSETS, "js", filename) endswith(filename, ".js") && isfile(path) || continue @@ -1364,7 +1429,7 @@ end function domify(ctx, navnode, node) fixlinks!(ctx, navnode, node) - mdconvert(node, Markdown.MD(); footnotes=ctx.footnotes) + mdconvert(node, Markdown.MD(); footnotes=ctx.footnotes, settings=ctx.settings) end function domify(ctx, navnode, anchor::Anchors.Anchor) @@ -1694,15 +1759,18 @@ mdconvert(b::Markdown.BlockQuote, parent; kwargs...) = Tag(:blockquote)(mdconver mdconvert(b::Markdown.Bold, parent; kwargs...) = Tag(:strong)(mdconvert(b.text, parent; kwargs...)) -function mdconvert(c::Markdown.Code, parent::MDBlockContext; kwargs...) +function mdconvert(c::Markdown.Code, parent::MDBlockContext; settings::Union{HTML,Nothing}=nothing, kwargs...) @tags pre code language = Utilities.codelang(c.language) class = isempty(language) ? "nohighlight" : "language-$(language)" if language == "documenter-ansi" # From @repl blocks (through MultiCodeBlock) return pre(domify_ansicoloredtext(c.code, "nohighlight")) - else - return pre(code[".$class"](c.code)) + elseif settings !== nothing && settings.prerender && + !(isempty(language) || language == "nohighlight") + r = hljs_prerender(c, settings) + r !== nothing && return r end + return pre(code[".$class"](c.code)) end function mdconvert(mcb::Documents.MultiCodeBlock, parent::MDBlockContext; kwargs...) @tags pre br @@ -1723,6 +1791,28 @@ function mdconvert(mcb::Documents.MultiCodeBlock, parent::MDBlockContext; kwargs end mdconvert(c::Markdown.Code, parent; kwargs...) = Tag(:code)(c.code) +function hljs_prerender(c::Markdown.Code, settings::HTML) + @assert settings.prerender "unreachable" + @tags pre code + lang = Utilities.codelang(c.language) + hljs = settings.highlightjs + js = """ + const hljs = require('$(hljs)'); + console.log(hljs.highlight($(repr(c.code)), {'language': "$(lang)"}).value); + """ + out, err = IOBuffer(), IOBuffer() + try + run(pipeline(`$(settings.node) -e "$(js)"`; stdout=out, stderr=err)) + str = String(take!(out)) + # prepend nohighlight to stop runtime highlighting + # return pre(code[".nohighlight $(lang) .hljs"](Tag(Symbol("#RAW#"))(str))) + return pre(code[".language-$(lang) .hljs"](Tag(Symbol("#RAW#"))(str))) + catch e + @error "HTMLWriter: prerendering failed" exception=e stderr=String(take!(err)) + end + return nothing +end + mdconvert(h::Markdown.Header{N}, parent; kwargs...) where {N} = DOM.Tag(Symbol("h$N"))(mdconvert(h.text, h; kwargs...)) mdconvert(::Markdown.HorizontalRule, parent; kwargs...) = Tag(:hr)() diff --git a/test/prerender/Project.toml b/test/prerender/Project.toml new file mode 100644 index 0000000000..fbe35508fb --- /dev/null +++ b/test/prerender/Project.toml @@ -0,0 +1,3 @@ +[deps] +NodeJS_16_jll = "a4b94fbf-73f5-58ff-888d-48a8396e17f6" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/prerender/prerender.jl b/test/prerender/prerender.jl new file mode 100644 index 0000000000..5b8a38bd06 --- /dev/null +++ b/test/prerender/prerender.jl @@ -0,0 +1,99 @@ +using Documenter, Test +import NodeJS_16_jll + +function read_assets() + path = joinpath(@__DIR__, "build", "assets", "documenter.js") + return read(path, String) +end +function read_index() + path = joinpath(@__DIR__, "build", "index.html") + return read(path, String) +end + +@testset "prerender with NodeJS" begin + +# Regular makedocs +makedocs(; + sitename = "Prerendering code blocks", + format = Documenter.HTML( + highlights = ["llvm"], + ), +) +assets = read_assets() +@test occursin("'highlight-julia'", assets) +@test occursin("/languages/julia.min", assets) +@test occursin("'highlight-julia-repl'", assets) +@test occursin("/languages/julia-repl.min", assets) +@test occursin("'highlight-llvm'", assets) +@test occursin("/languages/llvm.min", assets) +index = read_index() +@test occursin("function f()", index) +@test occursin("julia> function f()", index) +@test occursin("; @ int.jl:87 within", index) + +# With prerender +HLJSFILES = Documenter.Writers.HTMLWriter.HLJSFILES +for _ in 1:2 # test with and without highlightjs file given + makedocs(; + sitename = "Prerendering code blocks", + format = Documenter.HTML( + highlights = ["llvm"], + prerender = true, + node = NodeJS_16_jll.node(), + highlightjs = length(HLJSFILES) == 0 ? nothing : last(first(HLJSFILES)), + ), + ) + local assets = read_assets() + @test !occursin("'highlight-julia'", assets) + @test !occursin("/languages/julia.min", assets) + @test !occursin("'highlight-julia-repl'", assets) + @test !occursin("/languages/julia-repl.min", assets) + @test !occursin("'highlight-llvm'", assets) + @test !occursin("/languages/llvm.min", assets) + local index = read_index() + @test !occursin("function f()", index) + @test !occursin("julia> function f()", index) + @test !occursin("; @ int.jl:87 within", index) + @test occursin("function f()", index) + @test occursin("julia>", index) + @test occursin("; @ int.jl:87", index) + @test length(HLJSFILES) == 1 +end + +## missing language (llvm) +@test_logs (:error, "HTMLWriter: prerendering failed") match_mode=:any begin + makedocs(; + sitename = "Prerendering code blocks", + format = Documenter.HTML( + prerender = true, + node = NodeJS_16_jll.node(), + ), + ) +end +assets = read_assets() +@test !occursin("'highlight-julia'", assets) +@test !occursin("/languages/julia.min", assets) +@test !occursin("'highlight-julia-repl'", assets) +@test !occursin("/languages/julia-repl.min", assets) +@test !occursin("'highlight-llvm'", assets) +@test !occursin("/languages/llvm.min", assets) +index = read_index() +@test !occursin("function f()", index) +@test !occursin("julia> function f()", index) +@test occursin("; @ int.jl:87 within", index) +@test occursin("function f()", index) +@test occursin("julia>", index) +@test !occursin("; @ int.jl:87", index) + +@test length(HLJSFILES) == 2 + +# Some failure modes +@test_throws Base.IOError makedocs(; + sitename = "Prerendering code blocks", + format = Documenter.HTML( + prerender = true, + node = "nope", + ), +) + +end # testset diff --git a/test/prerender/src/index.md b/test/prerender/src/index.md new file mode 100644 index 0000000000..a7c2c448f3 --- /dev/null +++ b/test/prerender/src/index.md @@ -0,0 +1,22 @@ +# Code block prerendering using NodeJS + +```julia +function f() + print("hello, world") +end +``` + +```julia-repl +julia> function f() + print("hello, world") + end +``` + +```llvm +; @ int.jl:87 within `+' +define i64 @"julia_+_212"(i64 signext %0, i64 signext %1) { +top: + %2 = add i64 %1, %0 + ret i64 %2 +} +```