core: Add connection graph and incremental reference resolution

Benchmarks/Program.fs

@@ -6,11 +6,13 @@ open BenchmarkDotNet.Running
 open Ionide.LanguageServerProtocol.Types
 
 open Marksman.Misc
+open Marksman.Names
 open Marksman.Paths
-open Marksman.Workspace
+open Marksman.Cst
+open Marksman.Doc
 open Marksman.Index
 open Marksman.Refs
-open Marksman.Cst
+open Marksman.Folder
 
 type ReferenceResolution() =
     let mkFolder size : Folder =
@@ -23,12 +25,14 @@ type ReferenceResolution() =
 
         let docs =
             Array.init size (fun i ->
-                let docId = UriWith.mkRooted folderId (LocalPath.ofSystem $"doc{i}.md")
+                let docId =
+                    DocId(UriWith.mkRooted folderId (LocalPath.ofSystem $"doc{i}.md"))
+
                 let links = List.init size (fun toDoc -> $"[[doc{toDoc}.md]]")
                 let contentLines = $"# Doc {i}" :: links
                 let content = String.concat "\n" contentLines
                 let text = Text.mkText content
-                Doc.mk docId None text)
+                Doc.mk Config.defaultMarkdownExtensions docId None text)
 
         Folder.multiFile "docs" folderId docs None
 
@@ -49,8 +53,7 @@ type ReferenceResolution() =
         let link =
             Doc.index doc |> Index.linkAtPos (Position.Mk(1, 3)) |> Option.get
 
-        let uref = Uref.ofElement [| "md" |] doc.Id link |> Option.get
-        let refs = Dest.tryResolveUref uref doc this.Folder |> Seq.toArray
+        let refs = Dest.tryResolveElement this.Folder doc link |> Seq.toArray
         refs |> ignore
 
     [<Benchmark>]

Marksman.sln.DotSettings

@@ -2,7 +2,12 @@
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=cand/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=clippy/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Compl/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=dests/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=exts/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=imenu/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=noti/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Semato/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=srcs/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Surj/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=syms/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=uref/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

Marksman/Ast.fs

@@ -0,0 +1,162 @@
+module Marksman.Ast
+
+open Marksman.Misc
+open Marksman.Names
+open Marksman.Syms
+
+type Heading =
+    { level: int
+      text: string
+      id: Slug }
+
+    member this.CompactFormat() =
+        let prefix = String.replicate this.level "#"
+        $"{prefix} {this.text} {{{this.id.Raw}}}"
+
+type WikiLink =
+    { doc: option<string>
+      heading: option<string> }
+
+    member this.CompactFormat() =
+        let doc = this.doc |> Option.defaultValue ""
+
+        let heading =
+            this.heading
+            |> Option.map (fun x -> $"#{x}")
+            |> Option.defaultValue ""
+
+        $"[[{doc}{heading}]]"
+
+
+// [text](url "title")
+type MdLink =
+    { text: string
+      url: option<string>
+      anchor: option<string> }
+
+    member this.CompactFormat() =
+        let url =
+            this.url |> Option.map (fun x -> $"{x}") |> Option.defaultValue ""
+
+        let anchor =
+            this.anchor |> Option.map (fun x -> $"#{x}") |> Option.defaultValue ""
+
+        $"[{this.text}]({url}{anchor})"
+
+type MdRef =
+    // [text][dest]
+    | Full of text: string * dest: string
+    // [dest][]
+    | Collapsed of dest: string
+    // [label]
+    | Shortcut of dest: string
+
+    member this.CompactFormat() =
+        match this with
+        | Full (text, dest) -> $"[{text}][{dest}]"
+        | Collapsed dest -> $"[{dest}][]"
+        | Shortcut dest -> $"[{dest}]"
+
+    member this.Dest =
+        match this with
+        | Full (_, dest)
+        | Collapsed dest
+        | Shortcut dest -> dest
+
+    member this.DestLabel = LinkLabel.ofString this.Dest
+
+type MdLinkDef =
+    { label: string
+      url: UrlEncoded }
+
+    member this.Label = LinkLabel.ofString this.label
+    member this.CompactFormat() = $"[{this.label}]: {UrlEncoded.raw this.url}"
+
+[<Struct>]
+type Tag = Tag of string
+
+[<RequireQualifiedAccess>]
+type Element =
+    | H of Heading
+    | WL of WikiLink
+    | ML of MdLink
+    | MR of MdRef
+    | MLD of MdLinkDef
+    | T of Tag
+
+    member this.CompactFormat() =
+        match this with
+        | Element.H heading -> heading.CompactFormat()
+        | Element.WL wikiLink -> wikiLink.CompactFormat()
+        | Element.ML mdLink -> mdLink.CompactFormat()
+        | Element.MR mdRef -> mdRef.CompactFormat()
+        | Element.MLD mdLinkDef -> mdLinkDef.CompactFormat()
+        | Element.T (Tag tag) -> $"#{tag}"
+
+module Element =
+    let asHeading =
+        function
+        | Element.H heading -> Some heading
+        | _ -> None
+
+    let asLinkDef =
+        function
+        | Element.MLD mld -> Some mld
+        | _ -> None
+
+    // TODO: instead of checking for 'all whitespace' symbols all the time, create smart constructors
+    let toSym (exts: seq<string>) (el: Element) : option<Sym> =
+        match el with
+        | Element.H { level = level; id = id } ->
+            if Slug.isEmpty id then
+                None
+            else
+                Syms.Sym.Def(Def.Header(level, Slug.toString id)) |> Some
+        // Wiki-links mapping
+        | Element.WL { doc = None; heading = None } -> None
+        | Element.WL { doc = Some doc; heading = None } ->
+            if doc.IsWhitespace() then
+                None
+            else
+                Syms.Sym.Ref(Ref.CrossRef(CrossDoc doc)) |> Some
+        | Element.WL { doc = Some doc; heading = Some heading } ->
+            if heading.IsWhitespace() then
+                None
+            else
+                Syms.Sym.Ref(Ref.CrossRef(CrossSection(doc, Slug.ofString heading)))
+                |> Some
+        | Element.WL { doc = None; heading = Some heading } ->
+            if heading.IsWhitespace() then
+                None
+            else
+                Syms.Sym.Ref(Ref.IntraRef(IntraSection <| Slug.ofString heading))
+                |> Some
+        // Markdown links mapping
+        | Element.ML { url = url; anchor = anchor } ->
+            let urlIsRef =
+                url |> Option.map (fun url -> url, isPotentiallyInternalRef exts url)
+
+            match urlIsRef, anchor with
+            | None, None -> None
+            | None, Some anchor ->
+                if anchor.IsWhitespace() then
+                    None
+                else
+                    Some(Syms.Sym.Ref(IntraRef(IntraSection <| Slug.ofString anchor)))
+            | Some (_, false), _ -> None
+            | Some (url, true), None ->
+                if url.IsWhitespace() then
+                    None
+                else
+                    Some(Syms.Sym.Ref(CrossRef(CrossDoc url)))
+            | Some (url, true), Some anchor ->
+                if url.IsWhitespace() || anchor.IsWhitespace() then
+                    None
+                else
+                    Some(Syms.Sym.Ref(CrossRef(CrossSection(url, Slug.ofString anchor))))
+        // The rest
+        | Element.MR mdRef -> Some(Syms.Sym.Ref(IntraRef(IntraLinkDef mdRef.DestLabel)))
+        | Element.MLD mdLinkDef -> Some(Syms.Sym.Def(Def.LinkDef(mdLinkDef.Label)))
+        | Element.T (Tag tag) -> Some(Syms.Sym.Tag(Syms.Tag tag))
+
+type Ast = { elements: Element[] }

Marksman/CodeActions.fs

@@ -1,18 +1,19 @@
 module Marksman.CodeActions
 
+open type System.Environment
+
 open FSharpPlus
 open Ionide.LanguageServerProtocol.Types
 open Ionide.LanguageServerProtocol.Logging
 
-open Marksman.Toc
-open Marksman.Workspace
 open Marksman.Misc
-open Marksman.Refs
-open Marksman.Index
-open Marksman.Names
 open Marksman.Paths
-
-open type System.Environment
+open Marksman.Names
+open Marksman.Doc
+open Marksman.Index
+open Marksman.Folder
+open Marksman.Refs
+open Marksman.Toc
 
 let private logger = LogProvider.getLoggerByName "CodeActions"
 
@@ -118,26 +119,23 @@ let createMissingFile
     (doc: Doc)
     (folder: Folder)
     : CreateFileAction option =
-    let configuredExts =
-        (Folder.configOrDefault folder).CoreMarkdownFileExtensions()
+    let configuredExts = (Folder.configuredMarkdownExts folder)
 
     let pos = range.Start
 
     monad' {
         let! atPos = Doc.index doc |> Index.linkAtPos pos
-        let! uref = Uref.ofElement configuredExts (Doc.id doc) atPos
-        let refs = Dest.tryResolveUref uref doc folder
+        let refs = Dest.tryResolveElement folder doc atPos
 
         // Early return if the file exists
         do! guard (Seq.isEmpty refs)
 
         let! name =
-            match uref with
-            | Uref.Doc name -> Some name.data
-            | Uref.Heading (Some name, _) -> Some name.data
+            match doc.Structure |> Structure.Structure.tryFindSymbolForConcrete atPos with
+            | Some (Syms.Sym.Ref (Syms.CrossRef r)) -> Some r.Doc
             | _ -> None
 
-        let! internPath = InternName.tryAsPath name
+        let! internPath = InternName.tryAsPath { name = name; src = doc.Id }
 
         let relPath =
             InternPath.toRel internPath

Marksman/Compl.fs

@@ -5,15 +5,16 @@ open FSharpPlus.GenericBuilders
 open Ionide.LanguageServerProtocol.Logging
 open Ionide.LanguageServerProtocol.Types
 
-open Marksman.Config
-open Marksman.Cst
-open Marksman.Index
 open Marksman.Misc
 open Marksman.Paths
 open Marksman.Names
 open Marksman.Refs
 open Marksman.Text
-open Marksman.Workspace
+open Marksman.Cst
+open Marksman.Doc
+open Marksman.Index
+open Marksman.Config
+open Marksman.Folder
 
 let private logger = LogProvider.getLoggerByName "Compl"
 

Marksman/Config.fs

@@ -127,20 +127,26 @@ type Config =
       caCreateMissingFileEnable: option<bool>
       coreMarkdownFileExtensions: option<array<string>>
       coreTextSync: option<TextSync>
+      coreIncrementalReferences: option<bool>
+      coreParanoid: option<bool>
       complWikiStyle: option<ComplWikiStyle> }
 
     static member Default =
         { caTocEnable = Some true
           caCreateMissingFileEnable = Some true
           coreMarkdownFileExtensions = Some [| "md"; "markdown" |]
           coreTextSync = Some Full
+          coreIncrementalReferences = Some false
+          coreParanoid = Some false
           complWikiStyle = Some TitleSlug }
 
     static member Empty =
         { caTocEnable = None
           caCreateMissingFileEnable = None
           coreMarkdownFileExtensions = None
           coreTextSync = None
+          coreIncrementalReferences = None
+          coreParanoid = None
           complWikiStyle = None }
 
     member this.CaTocEnable() =
@@ -163,6 +169,16 @@ type Config =
         |> Option.orElse Config.Default.coreTextSync
         |> Option.get
 
+    member this.CoreIncrementalReferences() =
+        this.coreIncrementalReferences
+        |> Option.orElse Config.Default.coreIncrementalReferences
+        |> Option.get
+
+    member this.CoreParanoid() =
+        this.coreParanoid
+        |> Option.orElse Config.Default.coreParanoid
+        |> Option.get
+
     member this.ComplWikiStyle() =
         this.complWikiStyle
         |> Option.orElse Config.Default.complWikiStyle
@@ -181,6 +197,11 @@ let private configOfTable (table: TomlTable) : LookupResult<Config> =
         let! coreTextSync = getFromTableOpt<string> table [] [ "core"; "text_sync" ]
         let coreTextSync = coreTextSync |> Option.bind TextSync.ofStringOpt
 
+        let! coreIncrementalReferences =
+            getFromTableOpt<bool> table [] [ "core"; "incremental_references" ]
+
+        let! coreParanoid = getFromTableOpt<bool> table [] [ "core"; "paranoid" ]
+
         let! complWikiStyle = getFromTableOpt<string> table [] [ "completion"; "wiki"; "style" ]
 
         let complWikiStyle =
@@ -190,6 +211,8 @@ let private configOfTable (table: TomlTable) : LookupResult<Config> =
           caCreateMissingFileEnable = caCreateMissingFileEnable
           coreMarkdownFileExtensions = coreMarkdownFileExtensions
           coreTextSync = coreTextSync
+          coreIncrementalReferences = coreIncrementalReferences
+          coreParanoid = coreParanoid
           complWikiStyle = complWikiStyle }
     }
 
@@ -205,6 +228,10 @@ module Config =
             hi.coreMarkdownFileExtensions
             |> Option.orElse low.coreMarkdownFileExtensions
           coreTextSync = hi.coreTextSync |> Option.orElse low.coreTextSync
+          coreIncrementalReferences =
+            hi.coreIncrementalReferences
+            |> Option.orElse low.coreIncrementalReferences
+          coreParanoid = hi.coreParanoid |> Option.orElse low.coreParanoid
           complWikiStyle = hi.complWikiStyle |> Option.orElse low.complWikiStyle }
 
     let mergeOpt hi low =
@@ -247,3 +274,6 @@ module Config =
     let userConfigFile = Path.Join(userConfigDir, "config.toml")
 
     let orDefault configOpt = Option.defaultValue Config.Default configOpt
+
+let defaultMarkdownExtensions =
+    Config.Default.CoreMarkdownFileExtensions() |> Seq.ofArray