From 58d06fd621e38302be636795ba51949eea768d2f Mon Sep 17 00:00:00 2001 From: Carlos Scheidegger Date: Fri, 14 Jul 2023 11:53:29 -0300 Subject: [PATCH 1/4] add text-to-AST quarto API entries --- src/resources/filters/common/pandoc.lua | 33 +++++++++++++++++-- .../filters/normalize/extractquartodom.lua | 16 +++------ src/resources/pandoc/datadir/init.lua | 4 ++- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/resources/filters/common/pandoc.lua b/src/resources/filters/common/pandoc.lua index d74cffcd47..507d2bf06d 100644 --- a/src/resources/filters/common/pandoc.lua +++ b/src/resources/filters/common/pandoc.lua @@ -10,7 +10,7 @@ end -- read attribute w/ default function attribute(el, name, default) -- FIXME: Doesn't attributes respond to __index? - for k,v in pairs(el.attr.attributes) do + for k,v in pairs(el.attributes) do if k == name then return v end @@ -71,15 +71,20 @@ function combineFilters(filters) end function inlinesToString(inlines) + local pt = pandoc.utils.type(inlines) + if pt ~= "Inlines" then + fail("inlinesToString: expected Inlines, got " .. pt) + return "" + end return pandoc.utils.stringify(pandoc.Span(inlines)) end -- lua string to pandoc inlines function stringToInlines(str) if str then - return pandoc.List({pandoc.Str(str)}) + return pandoc.Inlines({pandoc.Str(str)}) else - return pandoc.List({}) + return pandoc.Inlines({}) end end @@ -170,3 +175,25 @@ function compileTemplate(template, meta) end end +local md_shortcode = require("lpegshortcode") + +-- FIXME pick a better name for this. +function string_to_quarto_ast_blocks(text) + local after_shortcodes = md_shortcode.md_shortcode:match(text) or "" + local after_reading = pandoc.read(after_shortcodes, "markdown") + + -- FIXME we should run the whole normalization pipeline here + local after_parsing = after_reading:walk(parse_extended_nodes()):walk(compute_flags()) + return after_parsing.blocks +end + +function string_to_quarto_ast_inlines(text) + local blocks = string_to_quarto_ast_blocks(text) + if #blocks ~= 1 then + fail("Expected a single block, got " .. #blocks) + end + if blocks[1].t ~= "Para" and blocks[1].t ~= "Plain" then + fail("Expected a Para or Plain block, got " .. blocks[1].t) + end + return blocks[1].content +end \ No newline at end of file diff --git a/src/resources/filters/normalize/extractquartodom.lua b/src/resources/filters/normalize/extractquartodom.lua index 0bb193b250..9647210440 100644 --- a/src/resources/filters/normalize/extractquartodom.lua +++ b/src/resources/filters/normalize/extractquartodom.lua @@ -1,32 +1,26 @@ -local md_shortcode = require("lpegshortcode") - local function process_quarto_markdown_input_element(el) if el.attributes.qmd == nil and el.attributes["qmd-base64"] == nil then error("process_quarto_markdown_input_element called with element that does not have qmd or qmd-base64 attribute") return el end local text = el.attributes.qmd or quarto.base64.decode(el.attributes["qmd-base64"]) - local after_shortcodes = md_shortcode.md_shortcode:match(text) or "" - local after_reading = pandoc.read(after_shortcodes, "markdown") - local after_parsing = after_reading:walk(parse_extended_nodes()):walk(compute_flags()) - return after_parsing + return string_to_quarto_ast_blocks(text) end function parse_md_in_html_rawblocks() return { Div = function(div) if div.attributes.qmd ~= nil or div.attributes["qmd-base64"] ~= nil then - local doc = process_quarto_markdown_input_element(div) - return doc.blocks + return process_quarto_markdown_input_element(div) end end, Span = function(span) if span.attributes.qmd ~= nil or span.attributes["qmd-base64"] ~= nil then - local doc = process_quarto_markdown_input_element(span) - if #doc.blocks < 1 then + local blocks = process_quarto_markdown_input_element(span) + if #blocks < 1 then return pandoc.Span({}) end - return doc.blocks[1].content + return blocks[1].content end end } diff --git a/src/resources/pandoc/datadir/init.lua b/src/resources/pandoc/datadir/init.lua index 6cc981498b..e2b3ac1e46 100644 --- a/src/resources/pandoc/datadir/init.lua +++ b/src/resources/pandoc/datadir/init.lua @@ -2058,7 +2058,9 @@ quarto = { resolve_path = resolvePathExt, resolve_path_relative_to_document = resolvePath, as_inlines = utils.as_inlines, - as_blocks = utils.as_blocks + as_blocks = utils.as_blocks, + string_to_blocks = string_to_quarto_ast_blocks, + string_to_inlines = string_to_quarto_ast_inlines, }, json = json, base64 = base64, From 25b868db752af715ea8eed8ac797228550e93677 Mon Sep 17 00:00:00 2001 From: Carlos Scheidegger Date: Fri, 14 Jul 2023 13:45:04 -0300 Subject: [PATCH 2/4] revert internal inlinesToString and stringToInlines --- src/resources/filters/common/pandoc.lua | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/resources/filters/common/pandoc.lua b/src/resources/filters/common/pandoc.lua index 507d2bf06d..85af08c09d 100644 --- a/src/resources/filters/common/pandoc.lua +++ b/src/resources/filters/common/pandoc.lua @@ -71,20 +71,15 @@ function combineFilters(filters) end function inlinesToString(inlines) - local pt = pandoc.utils.type(inlines) - if pt ~= "Inlines" then - fail("inlinesToString: expected Inlines, got " .. pt) - return "" - end return pandoc.utils.stringify(pandoc.Span(inlines)) end -- lua string to pandoc inlines function stringToInlines(str) if str then - return pandoc.Inlines({pandoc.Str(str)}) + return pandoc.List({pandoc.Str(str)}) else - return pandoc.Inlines({}) + return pandoc.List({}) end end From 64709c4537ba5418a9ade9839985482d8b5a7b9f Mon Sep 17 00:00:00 2001 From: Carlos Scheidegger Date: Fri, 14 Jul 2023 16:18:07 -0300 Subject: [PATCH 3/4] add smoke test and correct import of lua functions --- news/changelog-1.4.md | 1 + src/resources/pandoc/datadir/init.lua | 19 +++++++++++++++---- tests/docs/smoke-all/lua/6215/test.lua | 5 +++++ tests/docs/smoke-all/lua/6215/test.qmd | 15 +++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 tests/docs/smoke-all/lua/6215/test.lua create mode 100644 tests/docs/smoke-all/lua/6215/test.qmd diff --git a/news/changelog-1.4.md b/news/changelog-1.4.md index dee4628c33..cf168b7a7b 100644 --- a/news/changelog-1.4.md +++ b/news/changelog-1.4.md @@ -115,6 +115,7 @@ - Add support for relative paths in `require()` calls. - ([#5242](https://github.com/quarto-dev/quarto-cli/issues/5242)): Add line numbers to error messages. - Add support `quarto.doc.add_resource` and `quarto.doc.add_supporting`. `add_resource` will add a resource file to the current render, copying that file to the same relative location in the output directory. `add_supporting` will add a supporting file to the current render, moving that file file to the same relative location in the output directory. +- ([#6215](https://github.com/quarto-dev/quarto-cli/issues/6215)): Add `quarto.utils.string_to_inlines` and `quarto.utils.string_to_blocks` to Lua API to convert a string to a list of inlines or blocks taking into account quarto's AST structure. ## Books diff --git a/src/resources/pandoc/datadir/init.lua b/src/resources/pandoc/datadir/init.lua index e2b3ac1e46..4845d42eb1 100644 --- a/src/resources/pandoc/datadir/init.lua +++ b/src/resources/pandoc/datadir/init.lua @@ -1293,7 +1293,6 @@ local json = require '_json' local utils = require '_utils' local logging = require 'logging' - -- determines whether a path is a relative path local function isRelativeRef(ref) return ref:find("^/") == nil and @@ -1897,7 +1896,19 @@ _quarto = { exists = file_exists, remove = remove_file } - } +} + +-- this injection here is ugly but gets around +-- a hairy order-of-import issue that would otherwise happen +-- because string_to_inlines requires some filter code that is only +-- later imported + +_quarto.utils.string_to_inlines = function(s) + return string_to_quarto_ast_inlines(s) +end +_quarto.utils.string_to_blocks = function(s) + return string_to_quarto_ast_blocks(s) +end -- The main exports of the quarto module quarto = { @@ -2059,8 +2070,8 @@ quarto = { resolve_path_relative_to_document = resolvePath, as_inlines = utils.as_inlines, as_blocks = utils.as_blocks, - string_to_blocks = string_to_quarto_ast_blocks, - string_to_inlines = string_to_quarto_ast_inlines, + string_to_blocks = utils.string_to_blocks, + string_to_inlines = utils.string_to_inlines, }, json = json, base64 = base64, diff --git a/tests/docs/smoke-all/lua/6215/test.lua b/tests/docs/smoke-all/lua/6215/test.lua new file mode 100644 index 0000000000..ae8e051cf2 --- /dev/null +++ b/tests/docs/smoke-all/lua/6215/test.lua @@ -0,0 +1,5 @@ +function Pandoc(doc) + quarto.utils.dump(quarto.utils) + doc.blocks:extend(quarto.utils.string_to_blocks("Some markdown.\n\n[{{< meta test_key >}}]{#test_span}")) + return doc +end \ No newline at end of file diff --git a/tests/docs/smoke-all/lua/6215/test.qmd b/tests/docs/smoke-all/lua/6215/test.qmd new file mode 100644 index 0000000000..7b7d767017 --- /dev/null +++ b/tests/docs/smoke-all/lua/6215/test.qmd @@ -0,0 +1,15 @@ +--- +filters: + - test.lua +test_key: _test_value_ +_quarto: + _tests: + html: + ensureHtmlElements: + - ["span#test_span em"] # checks for emph to ensure shortcode was executed. + - [] +--- + +## A header + +Some content. \ No newline at end of file From b99e0c3555755ae224d7e16423dd4810d9e2c70e Mon Sep 17 00:00:00 2001 From: Carlos Scheidegger Date: Fri, 14 Jul 2023 17:10:05 -0300 Subject: [PATCH 4/4] add typings --- src/resources/lua-types/quarto/utils.lua | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/resources/lua-types/quarto/utils.lua b/src/resources/lua-types/quarto/utils.lua index e9f121e4a3..66907f7693 100644 --- a/src/resources/lua-types/quarto/utils.lua +++ b/src/resources/lua-types/quarto/utils.lua @@ -21,3 +21,18 @@ not be visible to the user. ---@return string function quarto.utils.resolve_path(path) end +--[[ +Converts a string to a list of Pandoc Inlines, processing any Quarto custom +syntax in the string. +]] +---@param path string String to be converted +---@return pandoc.Inlines +function quarto.utils.string_to_inlines(path) end + +--[[ +Converts a string to a list of Pandoc Blocks, processing any Quarto custom +syntax in the string. +]] +---@param path string String to be converted +---@return pandoc.Blocks +function quarto.utils.string_to_blocks(path) end