refactor: Introduce ParserSetting type

Right now it includes only the extensions. Soon it will be extended with title/header config settings.

stack-info: PR: #367, branch: artempyanykh/stack/11

Benchmarks/Program.fs

@@ -6,6 +6,7 @@ open BenchmarkDotNet.Running
 open Ionide.LanguageServerProtocol.Types
 
 open Marksman.Misc
+open Marksman.Config
 open Marksman.Names
 open Marksman.Paths
 open Marksman.Cst
@@ -32,7 +33,7 @@ type ReferenceResolution() =
                 let contentLines = $"# Doc {i}" :: links
                 let content = String.concat "\n" contentLines
                 let text = Text.mkText content
-                Doc.mk Config.defaultMarkdownExtensions docId None text)
+                Doc.mk ParserSettings.Default docId None text)
 
         Folder.multiFile "docs" folderId docs None
 

Marksman/Ast.fs

@@ -109,7 +109,7 @@ module Element =
         | _ -> None
 
     // TODO: instead of checking for 'all whitespace' symbols all the time, create smart constructors
-    let toSym (exts: seq<string>) (el: Element) : option<Sym> =
+    let toSym (parserSettings: Config.ParserSettings) (el: Element) : option<Sym> =
         match el with
         | Element.H { level = level; id = id } ->
             if Slug.isEmpty id then
@@ -144,7 +144,9 @@ module Element =
         // Markdown links mapping
         | Element.ML { url = url; anchor = anchor } ->
             let urlIsRef =
-                url |> Option.map (fun url -> url, isPotentiallyInternalRef exts url)
+                url
+                |> Option.map (fun url ->
+                    url, isPotentiallyInternalRef parserSettings.mdFileExt url)
 
             match urlIsRef, anchor with
             | None, None -> None

Marksman/Config.fs

@@ -331,3 +331,10 @@ module Config =
 
 let defaultMarkdownExtensions =
     Config.Default.CoreMarkdownFileExtensions() |> Seq.ofArray
+
+type ParserSettings = {
+    mdFileExt: string[]
+} with
+
+    static member OfConfig(config: Config) = { mdFileExt = config.CoreMarkdownFileExtensions() }
+    static member Default = ParserSettings.OfConfig(Config.Default)

Marksman/Doc.fs

@@ -62,8 +62,8 @@ module Doc =
 
     let logger = LogProvider.getLoggerByName "Doc"
 
-    let mk exts id version text =
-        let structure = Parser.parse exts text
+    let mk parserSettings id version text =
+        let structure = Parser.parse parserSettings text
         let index = Index.ofCst (Structure.concreteElements structure)
 
         {
@@ -77,8 +77,8 @@ module Doc =
     let id { id = id } = id
     let text doc = doc.text
 
-    let withText exts newText doc =
-        let newStructure = Parser.parse exts newText
+    let withText config newText doc =
+        let newStructure = Parser.parse config newText
         let newIndex = Index.ofCst (Structure.concreteElements newStructure)
 
         {
@@ -89,7 +89,7 @@ module Doc =
         }
 
 
-    let applyLspChange exts (change: DidChangeTextDocumentParams) (doc: Doc) : Doc =
+    let applyLspChange parserSettings (change: DidChangeTextDocumentParams) (doc: Doc) : Doc =
         let newVersion = change.TextDocument.Version
 
         logger.trace (
@@ -101,16 +101,16 @@ module Doc =
 
         let newText = applyTextChange change.ContentChanges doc.text
 
-        { withText exts newText doc with version = Some newVersion }
+        { withText parserSettings newText doc with version = Some newVersion }
 
-    let fromLsp exts (folderId: FolderId) (item: TextDocumentItem) : Doc =
+    let fromLsp parserSettings (folderId: FolderId) (item: TextDocumentItem) : Doc =
         let path = LocalPath.ofUri item.Uri
         let id = DocId(UriWith.mkRooted folderId path)
         let text = mkText item.Text
 
-        mk exts id (Some item.Version) text
+        mk parserSettings id (Some item.Version) text
 
-    let tryLoad exts (folderId: FolderId) (path: LocalPath) : option<Doc> =
+    let tryLoad parserSettings (folderId: FolderId) (path: LocalPath) : option<Doc> =
         try
             let content =
                 using (new StreamReader(LocalPath.toSystem path)) (fun f -> f.ReadToEnd())
@@ -119,7 +119,7 @@ module Doc =
 
             let id = DocId(UriWith.mkRooted folderId path)
 
-            Some(mk exts id None text)
+            Some(mk parserSettings id None text)
         with :? FileNotFoundException ->
             None
 

Marksman/Doc.fsi

@@ -3,6 +3,7 @@ module Marksman.Doc
 open Ionide.LanguageServerProtocol.Types
 
 open Marksman.Misc
+open Marksman.Config
 open Marksman.Names
 open Marksman.Structure
 open Marksman.Paths
@@ -41,7 +42,7 @@ module Doc =
     val slug: Doc -> Slug
     val index: Doc -> Index
 
-    val tryLoad: exts: seq<string> -> FolderId -> path: LocalPath -> option<Doc>
-    val mk: exts: seq<string> -> DocId -> version: option<int> -> Text -> Doc
-    val fromLsp: exts: seq<string> -> FolderId -> TextDocumentItem -> Doc
-    val applyLspChange: exts: seq<string> -> DidChangeTextDocumentParams -> Doc -> Doc
+    val tryLoad: ParserSettings -> FolderId -> path: LocalPath -> option<Doc>
+    val mk: ParserSettings -> DocId -> version: option<int> -> Text -> Doc
+    val fromLsp: ParserSettings -> FolderId -> TextDocumentItem -> Doc
+    val applyLspChange: ParserSettings -> DidChangeTextDocumentParams -> Doc -> Doc

Marksman/Folder.fs

@@ -301,7 +301,7 @@ module Folder =
 
         lines.ToArray()
 
-    let private loadDocs (configuredExts: array<string>) (folderId: FolderId) : seq<Doc> =
+    let private loadDocs (parserSettings: ParserSettings) (folderId: FolderId) : seq<Doc> =
         let rec collect (cur: LocalPath) (ignoreMatchers: list<GlobMatcher>) =
             let ignoreMatchers =
                 match readIgnoreFiles cur with
@@ -317,12 +317,12 @@ module Folder =
                 seq {
                     for file in files do
                         if
-                            (isMarkdownFile configuredExts file.FullName)
+                            (isMarkdownFile parserSettings.mdFileExt file.FullName)
                             && not (GlobMatcher.ignoresAny ignoreMatchers file.FullName)
                         then
                             let pathUri = LocalPath.ofSystem file.FullName
 
-                            let document = Doc.tryLoad configuredExts folderId pathUri
+                            let document = Doc.tryLoad parserSettings folderId pathUri
 
                             match document with
                             | Some document -> yield document
@@ -505,8 +505,9 @@ module Folder =
                 (Option.defaultValue Config.Default folderConfig)
                     .CoreMarkdownFileExtensions()
 
-            let documents = loadDocs configuredExts folderId
+            let parserSettings = { mdFileExt = configuredExts }
 
+            let documents = loadDocs parserSettings folderId
 
             multiFile name folderId documents folderConfig |> Some
         else
@@ -612,12 +613,14 @@ module Folder =
             else
                 None
 
+    let parserSettings folder = ParserSettings.OfConfig(configOrDefault folder)
+
     let closeDoc (docId: DocId) (folder: Folder) : option<Folder> =
-        let exts = (configOrDefault folder).CoreMarkdownFileExtensions()
+        let parserSettings = parserSettings folder
 
         match folder.data with
         | MultiFile { root = root } ->
-            match Doc.tryLoad exts root (Abs <| RootedRelPath.toAbs docId.Path) with
+            match Doc.tryLoad parserSettings root (Abs <| RootedRelPath.toAbs docId.Path) with
             | Some doc -> withDoc doc folder |> Some
             | _ -> withoutDoc docId folder
         | SingleFile { doc = doc } ->

Marksman/Folder.fsi

@@ -19,6 +19,7 @@ module Folder =
     val withConfig: option<Config> -> Folder -> Folder
 
     val configuredMarkdownExts: Folder -> seq<string>
+    val parserSettings: Folder -> ParserSettings
 
     val docs: Folder -> seq<Doc>
     val docCount: Folder -> int

Marksman/Parser.fs

@@ -476,11 +476,11 @@ module Markdown =
 
         { elements = elements; childMap = childMap }
 
-let parse (exts: seq<string>) (text: Text) : Structure =
+let parse (parserSettings: Config.ParserSettings) (text: Text) : Structure =
     if String.IsNullOrEmpty text.content then
         let cst: Cst.Cst = { elements = [||]; childMap = Map.empty }
-        Structure.ofCst exts cst
+        Structure.ofCst parserSettings cst
     else
         let flatElements = Markdown.scrapeText text
         let cst = Markdown.buildCst text flatElements
-        Structure.ofCst exts cst
+        Structure.ofCst parserSettings cst

Marksman/Server.fs

@@ -637,8 +637,7 @@ type MarksmanServer(client: MarksmanClient) =
             let newState =
                 match State.tryFindFolderAndDoc docPath state with
                 | Some(folder, doc) ->
-                    let newDoc =
-                        Doc.applyLspChange (Folder.configuredMarkdownExts folder) par doc
+                    let newDoc = Doc.applyLspChange (Folder.parserSettings folder) par doc
 
                     let newFolder = Folder.withDoc newDoc folder
 
@@ -677,10 +676,10 @@ type MarksmanServer(client: MarksmanClient) =
             let newState =
                 match State.tryFindFolderEnclosing path state with
                 | None ->
-                    let configuredExts =
-                        (State.userConfigOrDefault state).CoreMarkdownFileExtensions()
+                    let config = State.userConfigOrDefault state
+                    let parserSettings = ParserSettings.OfConfig(config)
 
-                    if isMarkdownFile configuredExts (AbsPath.toSystem path.data) then
+                    if isMarkdownFile parserSettings.mdFileExt (AbsPath.toSystem path.data) then
                         let singletonRoot = UriWith.mkRoot par.TextDocument.Uri
 
                         logger.trace (
@@ -689,17 +688,17 @@ type MarksmanServer(client: MarksmanClient) =
                             >> Log.addContext "root" singletonRoot
                         )
 
-                        let doc = Doc.fromLsp configuredExts singletonRoot par.TextDocument
+                        let doc = Doc.fromLsp parserSettings singletonRoot par.TextDocument
                         let userConfig = (State.workspace state) |> Workspace.userConfig
                         let newFolder = Folder.singleFile doc userConfig
                         State.updateFolder newFolder state
                     else
                         state
                 | Some folder ->
-                    let configuredExts = (Folder.configuredMarkdownExts folder)
+                    let parserSettings = Folder.parserSettings folder
 
-                    if isMarkdownFile configuredExts (AbsPath.toSystem path.data) then
-                        let doc = Doc.fromLsp configuredExts (Folder.id folder) par.TextDocument
+                    if isMarkdownFile parserSettings.mdFileExt (AbsPath.toSystem path.data) then
+                        let doc = Doc.fromLsp parserSettings (Folder.id folder) par.TextDocument
                         let newFolder = Folder.withDoc doc folder
                         State.updateFolder newFolder state
                     else
@@ -732,11 +731,10 @@ type MarksmanServer(client: MarksmanClient) =
                 match State.tryFindFolderEnclosing docUri newState with
                 | None -> ()
                 | Some folder ->
-                    let configuredExts =
-                        (Folder.configOrDefault folder).CoreMarkdownFileExtensions()
+                    let parserSettings = Folder.parserSettings folder
 
-                    if isMarkdownFile configuredExts (AbsPath.toSystem docUri.data) then
-                        match Doc.tryLoad configuredExts (Folder.id folder) (Abs docUri.data) with
+                    if isMarkdownFile parserSettings.mdFileExt (AbsPath.toSystem docUri.data) then
+                        match Doc.tryLoad parserSettings (Folder.id folder) (Abs docUri.data) with
                         | Some doc ->
                             let newFolder = Folder.withDoc doc folder
                             newState <- State.updateFolder newFolder newState

Marksman/Structure.fs

@@ -61,12 +61,12 @@ module Structure =
         findAbstractForSymbol sym structure
         |> Set.fold (fun acc ael -> acc + findConcreteForAbstract ael structure) Set.empty
 
-    let ofCst (exts: seq<string>) (cst: Cst.Cst) : Structure =
+    let ofCst (parserSettings: Config.ParserSettings) (cst: Cst.Cst) : Structure =
         let rec go cst =
             seq {
                 for cel in cst do
                     let ael = Cst.Element.toAbstract cel
-                    let sym = ael |> Option.bind (Ast.Element.toSym exts)
+                    let sym = ael |> Option.bind (Ast.Element.toSym parserSettings)
 
                     match ael with
                     | Some ael -> yield cel, ael, sym

Tests/AstTests.fs

@@ -6,7 +6,7 @@ open Snapper
 open Marksman.Ast
 open Marksman.Structure
 
-let parseString content = Parser.parse Config.defaultMarkdownExtensions (Text.mkText content)
+let parseString content = Parser.parse Config.ParserSettings.Default (Text.mkText content)
 
 let checkInlineSnapshot =
     Helpers.checkInlineSnapshot (fun (el: Element) -> el.CompactFormat())

Tests/Helpers.fs

@@ -63,7 +63,7 @@ type FakeDoc =
             let docId =
                 DocId(UriWith.mkRooted (UriWith.mkRoot rootUri) (LocalPath.ofUri pathUri))
 
-            Doc.mk Config.defaultMarkdownExtensions docId None text
+            Doc.mk Config.ParserSettings.Default docId None text
 
         static member Mk(contentLines: array<string>, ?path: string) : Doc =
             let content = String.concat System.Environment.NewLine contentLines

Tests/ParserTests.fs

@@ -22,7 +22,7 @@ let checkSnapshot (document: array<Element>) =
 let checkInlineSnapshot = checkInlineSnapshot Element.fmt
 
 let scrapeString content =
-    parse Config.defaultMarkdownExtensions (Text.mkText content)
+    parse Config.ParserSettings.Default (Text.mkText content)
     |> Structure.concreteElements
 
 [<StoreSnapshotsPerClass>]

Tests/SematoTests.fs

@@ -22,7 +22,7 @@ Start with [[a-wiki-link]]. Then a [ref-link].
 End with [[wiki-link-no-eol]] and #tag."""
 
     let doc =
-        Doc.mk Config.defaultMarkdownExtensions docPath None (Text.mkText content)
+        Doc.mk Config.ParserSettings.Default docPath None (Text.mkText content)
 
     let data = Token.ofIndexEncoded (Doc.index doc)
     Assert.Equal(5 * 5, data.Length)

Tests/WorkspaceTest.fs

@@ -69,8 +69,7 @@ module DocTest =
         let dummyPath =
             (dummyRootPath [ "dummy.md" ]) |> mkDocId (mkFolderId dummyRoot)
 
-        let empty =
-            Doc.mk defaultMarkdownExtensions dummyPath None (Text.mkText "")
+        let empty = Doc.mk ParserSettings.Default dummyPath None (Text.mkText "")
 
         let insertChange = {
             TextDocument = { Uri = RootedRelPath.toSystem dummyPath.Path; Version = 1 }
@@ -83,8 +82,7 @@ module DocTest =
             |]
         }
 
-        let updated =
-            Doc.applyLspChange defaultMarkdownExtensions insertChange empty
+        let updated = Doc.applyLspChange ParserSettings.Default insertChange empty
 
         Assert.Equal("[", (Doc.text updated).content)
 
@@ -103,7 +101,7 @@ module DocTest =
         }
 
         let singletonRoot = UriWith.mkRoot par.Uri
-        let doc = Doc.fromLsp defaultMarkdownExtensions singletonRoot par
+        let doc = Doc.fromLsp ParserSettings.Default singletonRoot par
         Assert.Equal("file:///a/b/doc.md", (Doc.uri doc).ToString())
         Assert.Equal("AbsPath \"/a/b/doc.md\"", (Doc.path doc).ToString())