Support prerendering of code blocks.

.github/workflows/CI.yml

@@ -74,7 +74,7 @@ jobs:
             Pkg.instantiate()
             Pkg.develop(PackageSpec(path=pwd()))
             Pkg.add(["IOCapture", "DocumenterMarkdown"])'
-      - run: julia --project=test/examples test/examples/tests_latex.jl
+      - run: julia --project=test/examples --code-coverage test/examples/tests_latex.jl
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:
@@ -105,6 +105,35 @@ jobs:
             Pkg.develop(PackageSpec(path=pwd()))'
       - run: julia --project=test/themes test/themes/themes.jl
 
+  prerender:
+    name: "NodeJS for prerender"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1'
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - run: |
+          julia --project=test/prerender/ -e '
+            using Pkg
+            Pkg.instantiate()
+            Pkg.develop(PackageSpec(path=pwd()))'
+      - run: julia --project=test/prerender --code-coverage test/prerender/prerender.jl
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v1
+        with:
+          file: lcov.info
+
   docs:
     name: 'Documentation: ${{ matrix.makejl }}'
     runs-on: ubuntu-latest

.gitignore

@@ -9,6 +9,7 @@ test/formats/builds/
 test/missingdocs/build/
 test/nongit/build/
 test/errors/build/
+test/prerender/build/
 test/workdir/builds/
 docs/build/
 docs/pdf/build/

CHANGELOG.md

@@ -4,6 +4,21 @@
 
 * ![Feature][badge-feature] `@example`- and `@repl`-blocks now support colored output by mapping ANSI escape sequences to HTML. This requires Julia >= 1.6 and passing `ansicolor=true` to `Documenter.HTML` (e.g. `makedocs(format=Documenter.HTML(ansicolor=true, ...), ...)`). In Documenter 0.28.0 this will be the default so to (preemptively) opt-out pass `ansicolor=false`. ([#1441][github-1441], [#1628][github-1628])
 
+* ![Experimental][badge-experimental] ![Feature][badge-feature] Documenter's HTML output can now prerender syntax highlighting of code blocks, i.e. syntax highlighting is applied when generating the HTML page rather than on the fly in the browser after the page is loaded. This requires (i) passing `prerender=true` to `Documenter.HTML` and (ii) a `node` (NodeJS) executable available in `PATH`. A path to a `node` executable can be specified by passing the `node` keyword argument to `Documenter.HTML` (for example from the `NodeJS_16_jll` Julia package). In addition, the `highlightjs` keyword argument can be used to specify a file path to a highlight.js library (if this is not given the release used by Documenter will be used). Example configuration:
+  ```julia
+  using Documenter, NodeJS_16_jll
+
+  makedocs(;
+      format = Documenter.HTML(
+          prerender = true,            # enable prerendering
+          node = NodeJS_16_jll.node(), # specify node executable (required if not available in PATH)
+          # ...
+      )
+      # ...
+  )
+  ```
+  _This feature is experimental and subject to change in future releases._ ([#1627][github-1627])
+
 * ![Enhancement][badge-enhancement] The sandbox module used for evaluating `@repl` and `@example` blocks is now removed (replaced with `Main`) in text output. ([#1633][github-1633])
 
 * ![Enhancement][badge-enhancement] `@repl`, `@example`, and `@eval` blocks now have `LineNumberNodes` inserted such that e.g. `@__FILE__` and `@__LINE__` give better output and not just `"none"` for the file and `1` for the line. This requires Julia 1.6 or higher (no change on earlier Julia versions). ([#1634][github-1634])
@@ -856,6 +871,7 @@
 [github-1616]: https://github.com/JuliaDocs/Documenter.jl/pull/1616
 [github-1617]: https://github.com/JuliaDocs/Documenter.jl/pull/1617
 [github-1625]: https://github.com/JuliaDocs/Documenter.jl/pull/1625
+[github-1627]: https://github.com/JuliaDocs/Documenter.jl/pull/1627
 [github-1628]: https://github.com/JuliaDocs/Documenter.jl/pull/1628
 [github-1633]: https://github.com/JuliaDocs/Documenter.jl/pull/1633
 [github-1634]: https://github.com/JuliaDocs/Documenter.jl/pull/1634

src/Writers/HTMLWriter.jl

@@ -325,7 +325,7 @@ Defaults to `true`.
 
 **`highlights`** can be used to add highlighting for additional languages. By default,
 Documenter already highlights all the ["Common" highlight.js](https://highlightjs.org/download/)
-languages and Julia (`julia`, `julia-repl`). Additional languages must be specified by"
+languages and Julia (`julia`, `julia-repl`). Additional languages must be specified by
 their filenames as they appear on [CDNJS](https://cdnjs.com/libraries/highlight.js) for the
 highlight.js version Documenter is using. E.g. to include highlighting for YAML and LLVM IR,
 you would set `highlights = ["llvm", "yaml"]`. Note that no verification is done whether the
@@ -349,6 +349,16 @@ and the [Julia Programming Language](https://julialang.org/)."`.
 **`warn_outdated`** inserts a warning if the current page is not the newest version of the
 documentation.
 
+## Experimental options
+
+**`prerender`** a boolean (`true` or `false` (default)) for enabling prerendering/build
+time application of syntax highlighting of code blocks. Requires a `node` (NodeJS)
+executable to be available in `PATH` or to be passed as the `node` keyword.
+
+**`node`** path to a `node` (NodeJS) executable used for prerendering.
+
+**`highlightjs`** file path to custom highglight.js library to be used with prerendering.
+
 # Default and custom assets
 
 Documenter copies all files under the source directory (e.g. `/docs/src/`) over
@@ -395,6 +405,9 @@ struct HTML <: Documenter.Writer
     ansicolor     :: Bool
     lang          :: String
     warn_outdated :: Bool
+    prerender     :: Bool
+    node          :: Union{Cmd,String,Nothing}
+    highlightjs   :: Union{String,Nothing}
 
     function HTML(;
             prettyurls    :: Bool = true,
@@ -409,12 +422,21 @@ struct HTML <: Documenter.Writer
             mathengine    :: Union{MathEngine,Nothing} = KaTeX(),
             footer        :: Union{String, Nothing} = "Powered by [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl) and the [Julia Programming Language](https://julialang.org/).",
             ansicolor     :: Bool = false, # true in 0.28
+            lang          :: String = "en",
+            warn_outdated :: Bool = true,
+
+            # experimental keywords
+            prerender     :: Bool = false,
+            node          :: Union{Cmd,String,Nothing} = nothing,
+            highlightjs   :: Union{String,Nothing} = nothing,
+
             # deprecated keywords
             edit_branch   :: Union{String, Nothing, Default} = Default(nothing),
-            lang          :: String = "en",
-            warn_outdated :: Bool = true
         )
         collapselevel >= 1 || throw(ArgumentError("collapselevel must be >= 1"))
+        if prerender
+            prerender, node, highlightjs = prepare_prerendering(prerender, node, highlightjs, highlights)
+        end
         assets = map(assets) do asset
             isa(asset, HTMLAsset) && return asset
             isa(asset, AbstractString) && return HTMLAsset(assetclass(asset), asset, true)
@@ -440,8 +462,49 @@ struct HTML <: Documenter.Writer
         end
         isa(edit_link, Default) && (edit_link = edit_link[])
         new(prettyurls, disable_git, edit_link, canonical, assets, analytics,
-            collapselevel, sidebar_sitename, highlights, mathengine, footer, ansicolor, lang, warn_outdated)
+            collapselevel, sidebar_sitename, highlights, mathengine, footer,
+            ansicolor, lang, warn_outdated, prerender, node, highlightjs)
+    end
+end
+
+# Cache of downloaded highlight.js bundles
+const HLJSFILES = Dict{String,String}()
+# Look for node and highlight.js
+function prepare_prerendering(prerender, node, highlightjs, highlights)
+    node = node === nothing ? Sys.which("node") : node
+    if node === nothing
+        @error "HTMLWriter: no node executable given or found on the system. Setting `prerender=false`."
+        return false, node, highlightjs
+    end
+    if !success(`$node --version`)
+        @error "HTMLWriter: bad node executable at $node. Setting `prerender=false`."
+        return false, node, highlightjs
+    end
+    if highlightjs === nothing
+        # Try to download
+        curl = Sys.which("curl")
+        if curl === nothing
+            @error "HTMLWriter: no highlight.js file given and no curl executable found " *
+                   "on the system. Setting `prerender=false`."
+            return false, node, highlightjs
+        end
+        @debug "HTMLWriter: downloading highlightjs"
+        r = Utilities.JSDependencies.RequireJS([])
+        RD.highlightjs!(r, highlights)
+        libs = sort!(collect(r.libraries); by = first) # puts highlight first
+        key = join((x.first for x in libs), ',')
+        highlightjs = get!(HLJSFILES, key) do
+            path, io = mktemp()
+            for lib in libs
+                println(io, "// $(lib.first)")
+                run(pipeline(`$(curl) -fsSL $(lib.second.url)`; stdout=io))
+                println(io)
+            end
+            close(io)
+            return path
+        end
     end
+    return prerender, node, highlightjs
 end
 
 "Provides a namespace for remote dependencies."
@@ -485,7 +548,7 @@ module RD
             "highlight",
             "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/$(hljs_version)/highlight.min.js"
         ))
-        prepend!(languages, ["julia", "julia-repl"])
+        languages = ["julia", "julia-repl", languages...]
         for language in languages
             language = jsescape(language)
             push!(r, RemoteLibrary(
@@ -663,7 +726,9 @@ function render(doc::Documents.Document, settings::HTML=HTML())
             RD.jquery, RD.jqueryui, RD.headroom, RD.headroom_jquery,
         ])
         RD.mathengine!(r, settings.mathengine)
-        RD.highlightjs!(r, settings.highlights)
+        if !settings.prerender
+            RD.highlightjs!(r, settings.highlights)
+        end
         for filename in readdir(joinpath(ASSETS, "js"))
             path = joinpath(ASSETS, "js", filename)
             endswith(filename, ".js") && isfile(path) || continue
@@ -1364,7 +1429,7 @@ end
 
 function domify(ctx, navnode, node)
     fixlinks!(ctx, navnode, node)
-    mdconvert(node, Markdown.MD(); footnotes=ctx.footnotes)
+    mdconvert(node, Markdown.MD(); footnotes=ctx.footnotes, settings=ctx.settings)
 end
 
 function domify(ctx, navnode, anchor::Anchors.Anchor)
@@ -1694,15 +1759,18 @@ mdconvert(b::Markdown.BlockQuote, parent; kwargs...) = Tag(:blockquote)(mdconver
 
 mdconvert(b::Markdown.Bold, parent; kwargs...) = Tag(:strong)(mdconvert(b.text, parent; kwargs...))
 
-function mdconvert(c::Markdown.Code, parent::MDBlockContext; kwargs...)
+function mdconvert(c::Markdown.Code, parent::MDBlockContext; settings::Union{HTML,Nothing}=nothing, kwargs...)
     @tags pre code
     language = Utilities.codelang(c.language)
     class = isempty(language) ? "nohighlight" : "language-$(language)"
     if language == "documenter-ansi" # From @repl blocks (through MultiCodeBlock)
         return pre(domify_ansicoloredtext(c.code, "nohighlight"))
-    else
-        return pre(code[".$class"](c.code))
+    elseif settings !== nothing && settings.prerender &&
+           !(isempty(language) || language == "nohighlight")
+        r = hljs_prerender(c, settings)
+        r !== nothing && return r
     end
+    return pre(code[".$class"](c.code))
 end
 function mdconvert(mcb::Documents.MultiCodeBlock, parent::MDBlockContext; kwargs...)
     @tags pre br
@@ -1723,6 +1791,28 @@ function mdconvert(mcb::Documents.MultiCodeBlock, parent::MDBlockContext; kwargs
 end
 mdconvert(c::Markdown.Code, parent; kwargs...) = Tag(:code)(c.code)
 
+function hljs_prerender(c::Markdown.Code, settings::HTML)
+    @assert settings.prerender "unreachable"
+    @tags pre code
+    lang = Utilities.codelang(c.language)
+    hljs = settings.highlightjs
+    js = """
+    const hljs = require('$(hljs)');
+    console.log(hljs.highlight($(repr(c.code)), {'language': "$(lang)"}).value);
+    """
+    out, err = IOBuffer(), IOBuffer()
+    try
+        run(pipeline(`$(settings.node) -e "$(js)"`; stdout=out, stderr=err))
+        str = String(take!(out))
+        # prepend nohighlight to stop runtime highlighting
+        # return pre(code[".nohighlight $(lang) .hljs"](Tag(Symbol("#RAW#"))(str)))
+        return pre(code[".language-$(lang) .hljs"](Tag(Symbol("#RAW#"))(str)))
+    catch e
+        @error "HTMLWriter: prerendering failed" exception=e stderr=String(take!(err))
+    end
+    return nothing
+end
+
 mdconvert(h::Markdown.Header{N}, parent; kwargs...) where {N} = DOM.Tag(Symbol("h$N"))(mdconvert(h.text, h; kwargs...))
 
 mdconvert(::Markdown.HorizontalRule, parent; kwargs...) = Tag(:hr)()

test/prerender/Project.toml

@@ -0,0 +1,3 @@
+[deps]
+NodeJS_16_jll = "a4b94fbf-73f5-58ff-888d-48a8396e17f6"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/prerender/prerender.jl

@@ -0,0 +1,99 @@
+using Documenter, Test
+import NodeJS_16_jll
+
+function read_assets()
+    path = joinpath(@__DIR__, "build", "assets", "documenter.js")
+    return read(path, String)
+end
+function read_index()
+    path = joinpath(@__DIR__, "build", "index.html")
+    return read(path, String)
+end
+
+@testset "prerender with NodeJS" begin
+
+# Regular makedocs
+makedocs(;
+    sitename = "Prerendering code blocks",
+    format = Documenter.HTML(
+        highlights = ["llvm"],
+    ),
+)
+assets = read_assets()
+@test occursin("'highlight-julia'", assets)
+@test occursin("/languages/julia.min", assets)
+@test occursin("'highlight-julia-repl'", assets)
+@test occursin("/languages/julia-repl.min", assets)
+@test occursin("'highlight-llvm'", assets)
+@test occursin("/languages/llvm.min", assets)
+index = read_index()
+@test occursin("<code class=\"language-julia\">function f()", index)
+@test occursin("<code class=\"language-julia-repl\">julia&gt; function f()", index)
+@test occursin("<code class=\"language-llvm\">;  @ int.jl:87 within", index)
+
+# With prerender
+HLJSFILES = Documenter.Writers.HTMLWriter.HLJSFILES
+for _ in 1:2 # test with and without highlightjs file given
+    makedocs(;
+        sitename = "Prerendering code blocks",
+        format = Documenter.HTML(
+            highlights = ["llvm"],
+            prerender = true,
+            node = NodeJS_16_jll.node(),
+            highlightjs = length(HLJSFILES) == 0 ? nothing : last(first(HLJSFILES)),
+        ),
+    )
+    local assets = read_assets()
+    @test !occursin("'highlight-julia'", assets)
+    @test !occursin("/languages/julia.min", assets)
+    @test !occursin("'highlight-julia-repl'", assets)
+    @test !occursin("/languages/julia-repl.min", assets)
+    @test !occursin("'highlight-llvm'", assets)
+    @test !occursin("/languages/llvm.min", assets)
+    local index = read_index()
+    @test !occursin("<code class=\"language-julia\">function f()", index)
+    @test !occursin("<code class=\"language-julia-repl\">julia&gt; function f()", index)
+    @test !occursin("<code class=\"language-llvm\">;  @ int.jl:87 within", index)
+    @test occursin("<code class=\"language-julia hljs\"><span class=\"hljs-keyword\">function</span> f()", index)
+    @test occursin("<code class=\"language-julia-repl hljs\"><span class=\"hljs-meta\">julia&gt;</span>", index)
+    @test occursin("<code class=\"language-llvm hljs\"><span class=\"hljs-comment\">;  @ int.jl:87", index)
+    @test length(HLJSFILES) == 1
+end
+
+## missing language (llvm)
+@test_logs (:error, "HTMLWriter: prerendering failed") match_mode=:any begin
+    makedocs(;
+        sitename = "Prerendering code blocks",
+        format = Documenter.HTML(
+            prerender = true,
+            node = NodeJS_16_jll.node(),
+        ),
+    )
+end
+assets = read_assets()
+@test !occursin("'highlight-julia'", assets)
+@test !occursin("/languages/julia.min", assets)
+@test !occursin("'highlight-julia-repl'", assets)
+@test !occursin("/languages/julia-repl.min", assets)
+@test !occursin("'highlight-llvm'", assets)
+@test !occursin("/languages/llvm.min", assets)
+index = read_index()
+@test !occursin("<code class=\"language-julia\">function f()", index)
+@test !occursin("<code class=\"language-julia-repl\">julia&gt; function f()", index)
+@test occursin("<code class=\"language-llvm\">;  @ int.jl:87 within", index)
+@test occursin("<code class=\"language-julia hljs\"><span class=\"hljs-keyword\">function</span> f()", index)
+@test occursin("<code class=\"language-julia-repl hljs\"><span class=\"hljs-meta\">julia&gt;</span>", index)
+@test !occursin("<code class=\"language-llvm hljs\"><span class=\"hljs-comment\">;  @ int.jl:87", index)
+
+@test length(HLJSFILES) == 2
+
+# Some failure modes
+@test_throws Base.IOError makedocs(;
+    sitename = "Prerendering code blocks",
+    format = Documenter.HTML(
+        prerender = true,
+        node = "nope",
+    ),
+)
+
+end # testset