Merge pull request #1237 from BetterThanTomorrow/defun-range-experiments

improve performance of rangeForDefun (cause of repl slowness) #942

CHANGELOG.md

@@ -3,6 +3,7 @@
 Changes to Calva.
 
 ## [Unreleased]
+- Performance improvement [REPL is Slow and Performance Degrades as the Output Grows](https://github.com/BetterThanTomorrow/calva/issues/942) 
 
 ## [2.0.205] - 2021-07-14
 - [Use new custom LSP method for server info command and print info in "Calva says" output channel ](https://github.com/BetterThanTomorrow/calva/issues/1211)

src/cursor-doc/paredit.ts

@@ -90,8 +90,8 @@ export function selectOpenList(doc: EditableDocument) {
  * Gets the range for the ”current” top level form
  * @see ListTokenCursor.rangeForDefun
  */
-export function rangeForDefun(doc: EditableDocument, offset: number = doc.selectionLeft, start: number = 0): [number, number] {
-    const cursor = doc.getTokenCursor(start);
+export function rangeForDefun(doc: EditableDocument, offset: number = doc.selection.active): [number, number] {
+    const cursor = doc.getTokenCursor(offset);
     return cursor.rangeForDefun(offset);
 }
 

src/cursor-doc/token-cursor.ts

@@ -388,8 +388,11 @@ export class LispTokenCursor extends TokenCursor {
         let cursor = this.clone();
         while (cursor.forwardSexp()) { }
         if (cursor.getToken().type === "close") {
-            this.set(cursor);
-            return true;
+            const backCursor = cursor.clone();
+            if (backCursor.backwardList()) {
+                this.set(cursor);
+                return true;
+            }
         }
         return false;
     }
@@ -602,35 +605,19 @@ export class LispTokenCursor extends TokenCursor {
             }
         }
         return undefined; // 8.
-    }
-
-    /**
-     * Gets the range for the ”current” top level form, visiting forms from the cursor towards `offset`
-     * With `commentCreatesTopLevel` as true (default): If the current top level form is a `(comment ...)`, consider it creating a new top level and continue the search.
-     * @param offset The ”current” position
-     * @param depth Controls if the cursor should consider `comment` top level (if > 0, it will not)
-     * @param commentIsTopLevel? Controls
-     */
-    rangeForDefun(offset: number, depth = 0, commentCreatesTopLevel = true): [number, number] {
-        const cursor = this.clone();
-        while (cursor.forwardSexp()) {
-            if (cursor.offsetEnd >= offset) {
-                if (depth < 1 && cursor.getPrevToken().raw === ')') {
-                    const commentCursor = cursor.clone();
-                    commentCursor.previous();
-                    if (commentCursor.getFunctionName() === 'comment' && commentCreatesTopLevel) {
-                        commentCursor.backwardList();
-                        commentCursor.forwardWhitespace();
-                        commentCursor.forwardSexp();
-                        return commentCursor.rangeForDefun(offset, depth + 1);
-                    }
-                }
-                const end = cursor.offsetStart;
-                cursor.backwardSexp();
-                return [cursor.offsetStart, end];
+    }  
+
+    rangeForDefun(offset: number, commentCreatesTopLevel = true): [number, number] {
+        const cursor = this.doc.getTokenCursor(offset);
+        let lastCandidateRange: [number, number] = cursor.rangeForCurrentForm(offset);
+        while (cursor.forwardList() && cursor.upList()) {
+            const commentCursor = cursor.clone();
+            commentCursor.backwardDownList();
+            if (!commentCreatesTopLevel || commentCursor.getToken().raw !== ')' || commentCursor.getFunctionName() !== 'comment') {
+                lastCandidateRange = cursor.rangeForCurrentForm(cursor.offsetStart);
             }
         }
-        return [offset, offset]
+        return lastCandidateRange;
     }
 
     rangesForTopLevelForms(): [number, number][] {

src/extension-test/unit/cursor-doc/token-cursor-test.ts

@@ -203,12 +203,35 @@ describe('Token Cursor', () => {
         });
     });
 
-    it('forwardList', () => {
-        const a = docFromTextNotation('(a(b(c•|#f•(#b •[:f :b :z])•#z•1)))');
-        const b = docFromTextNotation('(a(b(c•#f•(#b •[:f :b :z])•#z•1|)))');
-        const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
-        cursor.forwardList();
-        expect(cursor.offsetStart).toBe(b.selectionLeft);
+    describe('forwardList', () => {
+        it('Moves to closing end of list', () => {
+            const a = docFromTextNotation('(a(b(c•|#f•(#b •[:f :b :z])•#z•1)))');
+            const b = docFromTextNotation('(a(b(c•#f•(#b •[:f :b :z])•#z•1|)))');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
+        it('Does not move at top level', () => {
+            const a = docFromTextNotation('|foo (bar baz)');
+            const b = docFromTextNotation('|foo (bar baz)');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
+        it('Does not move at top level when unbalanced document from extra closings', () => {
+            const a = docFromTextNotation('|foo (bar baz))');
+            const b = docFromTextNotation('|foo (bar baz))');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
+        it('Does not move at top level when unbalanced document from extra opens', () => {
+            const a = docFromTextNotation('|foo ((bar baz)');
+            const b = docFromTextNotation('|foo ((bar baz)');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
     });
     it('upList', () => {
         const a = docFromTextNotation('(a(b(c•#f•(#b •[:f :b :z])•#z•1|)))');
@@ -217,12 +240,35 @@ describe('Token Cursor', () => {
         cursor.upList();
         expect(cursor.offsetStart).toBe(b.selectionLeft);
     });
-    it('backwardList', () => {
-        const a = docFromTextNotation('(a(b(c•#f•(#b •[:f :b :z])•#z•|1)))');
-        const b = docFromTextNotation('(a(b(|c•#f•(#b •[:f :b :z])•#z•1)))');
-        const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
-        cursor.backwardList();
-        expect(cursor.offsetStart).toBe(b.selectionLeft);
+    describe('backwardList', () => {
+        it('backwardList', () => {
+            const a = docFromTextNotation('(a(b(c•#f•(#b •[:f :b :z])•#z•|1)))');
+            const b = docFromTextNotation('(a(b(|c•#f•(#b •[:f :b :z])•#z•1)))');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.backwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
+            it('Does not move at top level', () => {
+            const a = docFromTextNotation('foo (bar baz)|');
+            const b = docFromTextNotation('foo (bar baz)|');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
+        it('Does not move at top level when unbalanced document from extra closings', () => {
+            const a = docFromTextNotation('foo (bar baz))|');
+            const b = docFromTextNotation('foo (bar baz))|');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
+        it('Does not move at top level when unbalanced document from extra opens', () => {
+            const a = docFromTextNotation('foo ((bar baz)|');
+            const b = docFromTextNotation('foo ((bar baz)|');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.forwardList();
+            expect(cursor.offsetStart).toBe(b.selectionLeft);
+        });
     });
 
     it('backwardUpList', () => {
@@ -256,7 +302,24 @@ describe('Token Cursor', () => {
             cursor.backwardSexp();
             expect(cursor.offsetStart).toEqual(b.selectionLeft);
         });
-    })
+    });
+
+    describe('The REPL prompt', () => {
+        it('Backward sexp bypasses prompt', () => {
+            const a = docFromTextNotation('foo•clj꞉foo꞉> |');
+            const b = docFromTextNotation('|foo•clj꞉foo꞉> ');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.backwardSexp();
+            expect(cursor.offsetStart).toEqual(b.selection.active);
+        });
+        it('Backward sexp not skipping comments bypasses prompt finding its start', () => {
+            const a = docFromTextNotation('foo•clj꞉foo꞉> |');
+            const b = docFromTextNotation('foo•|clj꞉foo꞉> ');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selectionLeft);
+            cursor.backwardSexp(false);
+            expect(cursor.offsetStart).toEqual(b.selection.active);
+        });
+    });
 
     describe('Current Form', () => {
         it('0: selects from within non-list form', () => {
@@ -346,44 +409,99 @@ describe('Token Cursor', () => {
     });
 
     describe('Top Level Form', () => {
-        it('Finds range for a regular top level form', () => {
+        it('Finds range when nested down a some forms', () => {
             const a = docFromTextNotation('aaa (bbb (ccc •#foo•(#bar •#baz•[:a :b| :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) (ddd eee)');
             const b = docFromTextNotation('aaa |(bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar))| (ddd eee)');
-            const cursor: LispTokenCursor = a.getTokenCursor(0);
-            expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds range when in current form is top level', () => {
+            const a = docFromTextNotation('aaa (bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) |(ddd eee)');
+            const b = docFromTextNotation('aaa (bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) |(ddd eee)|');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds range when in ”solid” top level form', () => {
+            const a = docFromTextNotation('a|aa (bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) (ddd eee)');
+            const b = docFromTextNotation('|aaa| (bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) (ddd eee)');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
         });
         it('Finds range for a top level form inside a comment', () => {
+            const a = docFromTextNotation('aaa (comment (comment [bbb cc|c]  ddd))');
+            const b = docFromTextNotation('aaa (comment (comment |[bbb ccc]|  ddd))');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds top level comment range if comment special treatment is disabled', () => {
             const a = docFromTextNotation('aaa (comment (ccc •#foo•(#bar •#baz•[:a :b| :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) (ddd eee)');
-            const b = docFromTextNotation('aaa (comment |(ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)|) (ddd eee)');
-            const cursor: LispTokenCursor = a.getTokenCursor(0);
-            expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
+            const b = docFromTextNotation('aaa |(comment (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar))| (ddd eee)');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active, false)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds comment range for empty comment form', () => { // Unimportant use case, just documenting how it behaves
+            const a = docFromTextNotation('aaa (comment |  ) bbb');
+            const b = docFromTextNotation('aaa (|comment|   ) bbb');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
         });
-        it('Finds comment range when comments are nested', () => { // TODO: Consider changing this behavior
+        it('Does not find comment range when comments are nested', () => {
             const a = docFromTextNotation('aaa (comment (comment [bbb ccc] | ddd))');
-            const b = docFromTextNotation('aaa (comment |(comment [bbb ccc]  ddd)|)');
-            const cursor: LispTokenCursor = a.getTokenCursor(0);
-            expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
+            const b = docFromTextNotation('aaa (comment (comment |[bbb ccc]|  ddd))');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds comment range when current form is top level comment form', () => {
+            const a = docFromTextNotation('aaa (bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) |(comment eee)');
+            const b = docFromTextNotation('aaa (bbb (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) |(comment eee)|');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
         });
         it('Includes reader tag', () => {
             const a = docFromTextNotation('aaa (comment #r [bbb ccc|]  ddd)');
             const b = docFromTextNotation('aaa (comment |#r [bbb ccc]|  ddd)');
-            const cursor: LispTokenCursor = a.getTokenCursor(0);
-            expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
         });
         it('Finds the preceding range when cursor is between to forms on the same line', () => {
             const a = docFromTextNotation('aaa (comment [bbb ccc] | ddd)');
             const b = docFromTextNotation('aaa (comment |[bbb ccc]|  ddd)');
-            const cursor: LispTokenCursor = a.getTokenCursor(0);
-            expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
         });
         it('Finds the succeeding range when cursor is at the start of the line', () => {
             const a = docFromTextNotation('aaa (comment [bbb ccc]• | ddd)');
             const b = docFromTextNotation('aaa (comment [bbb ccc]•  |ddd|)');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds the preceding comment symbol range when cursor is between that and something else on the same line', () => {
+            // This is a bit funny, but is not an important use case
+            const a = docFromTextNotation('aaa (comment  | [bbb ccc]  ddd)');
+            const b = docFromTextNotation('aaa (|comment|   [bbb ccc]  ddd)');
+            const cursor: LispTokenCursor = a.getTokenCursor(a.selection.active);
+            expect(cursor.rangeForDefun(a.selection.active)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Can find the comment range for a top level form inside a comment', () => {
+            const a = docFromTextNotation('aaa (comment (ccc •#foo•(#bar •#baz•[:a :b| :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar)) (ddd eee)');
+            const b = docFromTextNotation('aaa |(comment (ccc •#foo•(#bar •#baz•[:a :b :c]•x•#(a b c))•#baz•yyy•   z z z   •foo•   •   bar))| (ddd eee)');
+            const cursor: LispTokenCursor = a.getTokenCursor(0);
+            expect(cursor.rangeForDefun(a.selectionLeft, false)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds closest form inside multiple nested comments', () => {
+            const a = docFromTextNotation('aaa (comment (comment [bbb ccc] | ddd))');
+            const b = docFromTextNotation('aaa (comment (comment |[bbb ccc]|  ddd))');
+            const cursor: LispTokenCursor = a.getTokenCursor(0);
+            expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
+        });
+        it('Finds the preceding range when cursor is between two forms on the same line', () => {
+            const a = docFromTextNotation('aaa (comment [bbb ccc] | ddd)');
+            const b = docFromTextNotation('aaa (comment |[bbb ccc]|  ddd)');
             const cursor: LispTokenCursor = a.getTokenCursor(0);
             expect(cursor.rangeForDefun(a.selectionLeft)).toEqual(textAndSelection(b)[1]);
         });
-
     });
+
     describe('Location State', () => {
         it('Knows when inside string', () => {
             const doc = docFromTextNotation('(str [] "", "foo" "f   b  b"   "   f b b   " "\\"" \\")');

src/highlight/src/extension.ts

@@ -216,11 +216,11 @@ function updateRainbowBrackets() {
     const startOffset = doc.offsetAt(range.start),
       endOffset = doc.offsetAt(range.end),
       startCursor: LispTokenCursor = mirrorDoc.getTokenCursor(0),
-      startRange = startCursor.rangeForDefun(startOffset, 1),
-      endCursor: LispTokenCursor = mirrorDoc.getTokenCursor(startRange[1]),
-      endRange = endCursor.rangeForDefun(endOffset, 1),
-      rangeStart = startRange[0],
-      rangeEnd = endRange[1];
+      startRange = startCursor.rangeForDefun(startOffset, false),
+      endCursor: LispTokenCursor = mirrorDoc.getTokenCursor(endOffset),
+      endRange = endCursor.rangeForDefun(endOffset, false),
+      rangeStart = startRange ? startRange[0] : startOffset,
+      rangeEnd = endRange ? endRange[1] : endOffset;
     // Look for top level ignores, and adjust starting point if found
     const topLevelSentinelCursor = mirrorDoc.getTokenCursor(rangeStart);
     let startPaintingFrom = rangeStart;