escape # and & in markdown if necessary

... and don't escape > because it's never necessary.

Needs 3b/3bmd#60 from 3BMD.

README

@@ -1225,15 +1225,14 @@ The [Markdown][markdown] in docstrings is processed with the
 
 Docstrings can be indented in any of the usual styles. PAX
 normalizes indentation by stripping the longest run of leading
-spaces common to all non-blank lines except the first:
+spaces common to all non-blank lines except the first. The following
+two docstrings are equivalent:
 
     (defun foo ()
       "This is
       indented
       differently")
-
-to
-
+
     (defun foo ()
       "This is
     indented
@@ -2489,14 +2488,14 @@ Transcription support in Emacs can be enabled by loading
 
     The default value of TRANSCRIBE's CHECK-CONSISTENCY argument.
 
-- [variable] *TRANSCRIBE-SYNTAXES* ((:DEFAULT (:OUTPUT "..") (:NO-VALUE "=\> ; No value") (:READABLE "=\>")\
-  (:UNREADABLE "==\>") (:UNREADABLE-CONTINUATION "--\>"))\
- (:COMMENTED-1 (:OUTPUT ";..") (:NO-VALUE ";=\> ; No value") (:READABLE ";=\>")\
-  (:READABLE-CONTINUATION ";-\>") (:UNREADABLE ";==\>")\
-  (:UNREADABLE-CONTINUATION ";--\>"))\
- (:COMMENTED-2 (:OUTPUT ";;..") (:NO-VALUE ";;=\> ; No value")\
-  (:READABLE ";;=\>") (:READABLE-CONTINUATION ";;-\>") (:UNREADABLE ";;==\>")\
-  (:UNREADABLE-CONTINUATION ";;--\>")))
+- [variable] *TRANSCRIBE-SYNTAXES* ((:DEFAULT (:OUTPUT "..") (:NO-VALUE "=> ; No value") (:READABLE "=>")
+  (:UNREADABLE "==>") (:UNREADABLE-CONTINUATION "-->"))
+ (:COMMENTED-1 (:OUTPUT ";..") (:NO-VALUE ";=> ; No value") (:READABLE ";=>")
+  (:READABLE-CONTINUATION ";->") (:UNREADABLE ";==>")
+  (:UNREADABLE-CONTINUATION ";-->"))
+ (:COMMENTED-2 (:OUTPUT ";;..") (:NO-VALUE ";;=> ; No value")
+  (:READABLE ";;=>") (:READABLE-CONTINUATION ";;->") (:UNREADABLE ";;==>")
+  (:UNREADABLE-CONTINUATION ";;-->")))
 
     The default syntaxes used by TRANSCRIBE for reading and writing
     lines containing output and values of an evaluated form.
@@ -2843,13 +2842,14 @@ new DOCUMENT-DREF methods, which emit markdown.
     prefix is not added to the first line if EXCLUDE-FIRST-LINE-P. If
     PARAGRAPHP, then add a newline before and after the output.
 
-- [function] ESCAPE-MARKDOWN STRING &KEY (ESCAPE-NEWLINE T)
+- [function] ESCAPE-MARKDOWN STRING &KEY (ESCAPE-NEWLINE T) (ESCAPE-BLOCK T)
 
     Construct a new string from STRING by adding a backslash before
-    the special markdown characters ``*_`<>[]`` and newline if
-    ESCAPE-NEWLINE.
+    the special markdown characters ``*_`[]`` when necessary, escaping
+    HTML characters `<&`, and also newlines if ESCAPE-NEWLINE, and also
+    `#` when ESCAPE-BLOCK.
 
-- [function] PRIN1-TO-MARKDOWN OBJECT &KEY (ESCAPE-NEWLINE T)
+- [function] PRIN1-TO-MARKDOWN OBJECT &KEY (ESCAPE-NEWLINE T) (ESCAPE-BLOCK T)
 
     Like PRIN1-TO-STRING, but bind *PRINT-CASE* depending on
     *DOCUMENT-DOWNCASE-UPPERCASE-CODE* and *FORMAT*, and

README.md

@@ -1379,15 +1379,14 @@ The [Markdown][markdown] in docstrings is processed with the
 
 Docstrings can be indented in any of the usual styles. PAX
 normalizes indentation by stripping the longest run of leading
-spaces common to all non-blank lines except the first:
+spaces common to all non-blank lines except the first. The following
+two docstrings are equivalent:
 
     (defun foo ()
       "This is
       indented
       differently")
-
-to
-
+
     (defun foo ()
       "This is
     indented
@@ -2763,14 +2762,14 @@ Transcription support in Emacs can be enabled by loading
 
 <a id="x-28MGL-PAX-3A-2ATRANSCRIBE-SYNTAXES-2A-20VARIABLE-29"></a>
 
-- [variable] **\*TRANSCRIBE-SYNTAXES\*** *((:DEFAULT (:OUTPUT "..") (:NO-VALUE "=\> ; No value") (:READABLE "=\>")\
-  (:UNREADABLE "==\>") (:UNREADABLE-CONTINUATION "--\>"))\
- (:COMMENTED-1 (:OUTPUT ";..") (:NO-VALUE ";=\> ; No value") (:READABLE ";=\>")\
-  (:READABLE-CONTINUATION ";-\>") (:UNREADABLE ";==\>")\
-  (:UNREADABLE-CONTINUATION ";--\>"))\
- (:COMMENTED-2 (:OUTPUT ";;..") (:NO-VALUE ";;=\> ; No value")\
-  (:READABLE ";;=\>") (:READABLE-CONTINUATION ";;-\>") (:UNREADABLE ";;==\>")\
-  (:UNREADABLE-CONTINUATION ";;--\>")))*
+- [variable] **\*TRANSCRIBE-SYNTAXES\*** *((:DEFAULT (:OUTPUT "..") (:NO-VALUE "=> ; No value") (:READABLE "=>")
+  (:UNREADABLE "==>") (:UNREADABLE-CONTINUATION "-->"))
+ (:COMMENTED-1 (:OUTPUT ";..") (:NO-VALUE ";=> ; No value") (:READABLE ";=>")
+  (:READABLE-CONTINUATION ";->") (:UNREADABLE ";==>")
+  (:UNREADABLE-CONTINUATION ";-->"))
+ (:COMMENTED-2 (:OUTPUT ";;..") (:NO-VALUE ";;=> ; No value")
+  (:READABLE ";;=>") (:READABLE-CONTINUATION ";;->") (:UNREADABLE ";;==>")
+  (:UNREADABLE-CONTINUATION ";;-->")))*
 
     The default syntaxes used by [`TRANSCRIBE`][f1f0] for reading and writing
     lines containing output and values of an evaluated form.
@@ -3165,15 +3164,16 @@ new `DOCUMENT-DREF` methods, which emit markdown.
 
 <a id="x-28MGL-PAX-3AESCAPE-MARKDOWN-20FUNCTION-29"></a>
 
-- [function] **ESCAPE-MARKDOWN** *STRING &KEY (ESCAPE-NEWLINE T)*
+- [function] **ESCAPE-MARKDOWN** *STRING &KEY (ESCAPE-NEWLINE T) (ESCAPE-BLOCK T)*
 
     Construct a new string from `STRING` by adding a backslash before
-    the special markdown characters ``*_`<>[]`` and newline if
-    `ESCAPE-NEWLINE`.
+    the special markdown characters ``*_`[]`` when necessary, escaping
+    HTML characters `<&`, and also newlines if `ESCAPE-NEWLINE`, and also
+    `#` when `ESCAPE-BLOCK`.
 
 <a id="x-28MGL-PAX-3APRIN1-TO-MARKDOWN-20FUNCTION-29"></a>
 
-- [function] **PRIN1-TO-MARKDOWN** *OBJECT &KEY (ESCAPE-NEWLINE T)*
+- [function] **PRIN1-TO-MARKDOWN** *OBJECT &KEY (ESCAPE-NEWLINE T) (ESCAPE-BLOCK T)*
 
     Like [`PRIN1-TO-STRING`][18e1], but bind [`*PRINT-CASE*`][443b] depending on
     [`*DOCUMENT-DOWNCASE-UPPERCASE-CODE*`][a5ee] and [`*FORMAT*`][3da8], and

src/document/docstring.lisp

@@ -22,15 +22,14 @@
 (defun strip-docstring-indent (docstring indentation first-line-special-p)
   """Docstrings can be indented in any of the usual styles. PAX
   normalizes indentation by stripping the longest run of leading
-  spaces common to all non-blank lines except the first:
+  spaces common to all non-blank lines except the first. The following
+  two docstrings are equivalent:
 
       (defun foo ()
         "This is
         indented
         differently")
 
-  to
-
       (defun foo ()
         "This is
       indented

src/document/document.lisp

@@ -2825,11 +2825,13 @@
             (italic string stream))
         (format stream "~A" string))))
 
-(defun/autoloaded prin1-to-markdown (object &key (escape-newline t))
+(defun/autoloaded prin1-to-markdown (object &key (escape-newline t)
+                                            (escape-block t))
   "Like PRIN1-TO-STRING, but bind *PRINT-CASE* depending on
   *DOCUMENT-DOWNCASE-UPPERCASE-CODE* and *FORMAT*, and
   ESCAPE-MARKDOWN."
-  (escape-markdown (prin1-to-string* object) :escape-newline escape-newline))
+  (escape-markdown (prin1-to-string* object) :escape-newline escape-newline
+                                             :escape-block escape-block))
 
 ;;; Print arg names without the package prefix to a string. The
 ;;; default value with prefix. Works for macro arglists too.

src/document/markdown.lisp

@@ -61,22 +61,53 @@
       ""
       (format stream "*~A*" string)))
 
-(defun markdown-special-char-p (char)
-  (member char '(#\* #\_ #\` #\< #\> #\[ #\])))
+(defun markdown-special-inline-char-p (char)
+  (member char '(#\* #\_ #\` #\[ #\])))
 
-(defun/autoloaded escape-markdown (string &key (escape-newline t))
+(defun markdown-special-block-char-p (char)
+  (member char '(#\#)))
+
+(defun/autoloaded escape-markdown (string &key (escape-newline t)
+                                          (escape-block t))
   "Construct a new string from STRING by adding a backslash before
-  the special markdown characters ``*_`<>[]`` and newline if
-  ESCAPE-NEWLINE."
-  (with-output-to-string (stream)
-    (dotimes (i (length string))
-      (let ((char (aref string i)))
-        (when (or (markdown-special-char-p char)
-                  (and escape-newline
-                       (or (char= char #\Newline)
-                           (char= char #\Return))))
-          (write-char #\\ stream))
-        (write-char char stream)))))
+  the special markdown characters ``*_`[]`` when necessary, escaping
+  HTML characters `<&`, and also newlines if ESCAPE-NEWLINE, and also
+  `#` when ESCAPE-BLOCK."
+  (flet ((blank-line-until-p (pos)
+           (loop for i downfrom (1- pos) downto 0
+                 for char = (aref string i)
+                 do (when (char= char #\Newline)
+                      (return t))
+                 do (unless (whitespacep char)
+                      (return nil))
+                 finally (return t))))
+    (with-output-to-string (stream)
+      (dotimes (i (length string))
+        (let ((char (aref string i)))
+          (case char
+            ((#\<) (write-string "&lt;" stream))
+            ((#\&)
+             (let ((semicolon-pos (position #\; string :start (1+ i)))
+                   (whitespace-pos (position-if #'whitespacep string
+                                                :start (1+ i))))
+               ;; If there is no semicolon or there is no whitespace
+               ;; between & and ;, then it's not parsed as an entity, so
+               ;; don't escape it to reduce clutter.
+               (if (and semicolon-pos (or (null whitespace-pos)
+                                          (< semicolon-pos whitespace-pos)))
+                   (write-string "&amp;" stream)
+                   (write-char char stream))))
+            (t
+             (when (or (markdown-special-inline-char-p char)
+                       (and escape-newline
+                            (or (char= char #\Return)
+                                (char= char #\Newline))
+                            (blank-line-until-p i))
+                       (and escape-block
+                            (markdown-special-block-char-p char)
+                            (blank-line-until-p i)))
+               (write-char #\\ stream))
+             (write-char char stream))))))))
 
 (defun unescape-markdown (string)
   (let ((escaping nil))

test/test-document.lisp

@@ -787,7 +787,7 @@ This is [Self-referencing][e042].
                     "<p><a id=\"MGL-PAX-TEST:*NASTY-VAR*%20VARIABLE\"></a></p>
 
 <ul>
-<li><p><span class=reference-bullet><span class=reference><span class=\"locative-type\">[variable]</span> <span class=\"reference-object\"><a href=\"#MGL-PAX-TEST:*NASTY-VAR*%20VARIABLE\" >*NASTY-VAR*</a></span></span> <span class=\"locative-args\">&quot; \\
+<li><p><span class=reference-bullet><span class=reference><span class=\"locative-type\">[variable]</span> <span class=\"reference-object\"><a href=\"#MGL-PAX-TEST:*NASTY-VAR*%20VARIABLE\" >*NASTY-VAR*</a></span></span> <span class=\"locative-args\">&quot; 
 \\
 \\&quot;</span></span></p>