From 34cc9e88b351bae76dbf092e7a39b820773a5982 Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Fri, 5 Aug 2022 16:03:29 -0700 Subject: [PATCH 1/7] refactor: avoid regexp parsing for expression continuations --- .changeset/tame-geese-drop.md | 5 + .../attr-complex-instanceof.expected.txt | 21 +- .../attr-complex-instanceof/input.marko | 6 +- .../attr-operator-whitespace-eof.expected.txt | 9 + .../attr-operator-whitespace-eof/input.marko | 1 + .../attr-operators-space-between.expected.txt | 66 ++-- .../attr-operators-space-between/input.marko | 2 + src/states/EXPRESSION.ts | 347 ++++++++++++++---- src/util/constants.ts | 5 + 9 files changed, 358 insertions(+), 104 deletions(-) create mode 100644 .changeset/tame-geese-drop.md create mode 100644 src/__tests__/fixtures/attr-operator-whitespace-eof/__snapshots__/attr-operator-whitespace-eof.expected.txt create mode 100644 src/__tests__/fixtures/attr-operator-whitespace-eof/input.marko diff --git a/.changeset/tame-geese-drop.md b/.changeset/tame-geese-drop.md new file mode 100644 index 00000000..3e4967be --- /dev/null +++ b/.changeset/tame-geese-drop.md @@ -0,0 +1,5 @@ +--- +"htmljs-parser": patch +--- + +Switch from regexp based parsing for the expression continuations. This slightly improves performance and more importantly fixes usage of the parser in safari. diff --git a/src/__tests__/fixtures/attr-complex-instanceof/__snapshots__/attr-complex-instanceof.expected.txt b/src/__tests__/fixtures/attr-complex-instanceof/__snapshots__/attr-complex-instanceof.expected.txt index af8f1d09..dc74e8d0 100644 --- a/src/__tests__/fixtures/attr-complex-instanceof/__snapshots__/attr-complex-instanceof.expected.txt +++ b/src/__tests__/fixtures/attr-complex-instanceof/__snapshots__/attr-complex-instanceof.expected.txt @@ -73,4 +73,23 @@ │ ││ │ ╰─ attrValue "= 'foo'" │ ││ ╰─ attrName │ │╰─ tagName "tag" - ╰─ ╰─ openTagStart \ No newline at end of file + ╰─ ╰─ openTagStart +12├─ +13╭─ tag a = 'foo' instanceofthing String + │ │ │ │ │ │ ╰─ attrName "String" + │ │ │ │ │ ╰─ attrName "instanceofthing" + │ │ │ │ ╰─ attrValue.value "'foo'" + │ │ │ ╰─ attrValue "= 'foo'" + │ │ ╰─ attrName + ╰─ ╰─ tagName "tag" +14╭─ + ╰─ ╰─ openTagEnd +15╭─ tag a = 'foo' instanceof + │ │ │ │ │ │ ├─ closeTagEnd(tag) + │ │ │ │ │ │ ╰─ openTagEnd + │ │ │ │ │ ╰─ attrName "instanceof" + │ │ │ │ ╰─ attrValue.value "'foo'" + │ │ │ ╰─ attrValue "= 'foo'" + │ │ ╰─ attrName + │ ├─ closeTagEnd(tag) + ╰─ ╰─ tagName "tag" \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-complex-instanceof/input.marko b/src/__tests__/fixtures/attr-complex-instanceof/input.marko index 5bb5fcf5..12dc9920 100644 --- a/src/__tests__/fixtures/attr-complex-instanceof/input.marko +++ b/src/__tests__/fixtures/attr-complex-instanceof/input.marko @@ -8,4 +8,8 @@ tag a = 'foo' instanceof; tag a = 'foo' instanceof, b - \ No newline at end of file + + +tag a = 'foo' instanceofthing String + +tag a = 'foo' instanceof \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-operator-whitespace-eof/__snapshots__/attr-operator-whitespace-eof.expected.txt b/src/__tests__/fixtures/attr-operator-whitespace-eof/__snapshots__/attr-operator-whitespace-eof.expected.txt new file mode 100644 index 00000000..5f832363 --- /dev/null +++ b/src/__tests__/fixtures/attr-operator-whitespace-eof/__snapshots__/attr-operator-whitespace-eof.expected.txt @@ -0,0 +1,9 @@ +1╭─ tag a = 'foo' instanceof + │ │ │ │ │ ╰─ attrName "instanceof" + │ │ │ │ ╰─ attrValue.value "'foo'" + │ │ │ ╰─ attrValue "= 'foo'" + │ │ ╰─ attrName + ╰─ ╰─ tagName "tag" +2╭─ + │ ├─ openTagEnd + ╰─ ╰─ closeTagEnd(tag) \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-operator-whitespace-eof/input.marko b/src/__tests__/fixtures/attr-operator-whitespace-eof/input.marko new file mode 100644 index 00000000..8bb913cc --- /dev/null +++ b/src/__tests__/fixtures/attr-operator-whitespace-eof/input.marko @@ -0,0 +1 @@ +tag a = 'foo' instanceof diff --git a/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt b/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt index 65d484a0..5c8b9eb3 100644 --- a/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt +++ b/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt @@ -384,11 +384,10 @@ │ ├─ openTagEnd ╰─ ╰─ tagName 49╭─ a=x ++ y a - │ │││ │ │ ╰─ attrName - │ │││ │ ╰─ attrName - │ │││ ╰─ attrName "++" - │ ││╰─ attrValue.value - │ │├─ attrValue "=x" + │ │││ │ ╰─ attrName + │ │││ ╰─ attrName + │ ││╰─ attrValue.value "x ++" + │ │├─ attrValue "=x ++" │ │╰─ attrName │ ├─ closeTagEnd(a) │ ├─ openTagEnd @@ -879,17 +878,32 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -111╭─ - │ ││││ │ │ │╰─ openTagEnd:selfClosed "/>" - │ ││││ │ │ ╰─ attrName - │ ││││ │ ╰─ attrName - │ ││││ ╰─ attrName "++" - │ │││╰─ attrValue.value - │ ││├─ attrValue "=x" +111╭─ + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "typeof ++ y" + │ ││├─ attrValue "=typeof ++ y" + │ ││╰─ attrName + │ │╰─ tagName + ╰─ ╰─ openTagStart +112╭─ + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "typeof y ++" + │ ││├─ attrValue "=typeof y ++" + │ ││╰─ attrName + │ │╰─ tagName + ╰─ ╰─ openTagStart +113╭─ + │ ││││ │ │╰─ openTagEnd:selfClosed "/>" + │ ││││ │ ╰─ attrName + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x ++" + │ ││├─ attrValue "=x ++" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -112╭─ y +114╭─ y │ ││││ ││ ││╰─ openTagEnd:selfClosed "/>" │ ││││ ││ │╰─ tagName │ ││││ ││ ╰─ openTagStart @@ -901,7 +915,7 @@ │ │╰─ tagName │ ├─ error(MISSING_END_TAG:Missing ending "a" tag) "" ╰─ ╰─ openTagStart -113╭─ > y a/> +115╭─ > y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x >> y" @@ -910,7 +924,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -114╭─ >> y a/> +116╭─ >> y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x >>> y" @@ -919,7 +933,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -115╭─ +117╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x ( y )" @@ -928,7 +942,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -116╭─ +118╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x { y }" @@ -937,7 +951,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -117╭─ y a/> +119╭─ y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x => y" @@ -946,7 +960,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -118╭─ ( y ) a/> +120╭─ ( y ) a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x => ( y )" @@ -955,7 +969,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -119╭─ { y } a/> +121╭─ { y } a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "x => { y }" @@ -964,7 +978,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -120╭─ +122╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" │ ││││ ╰─ attrName │ │││╰─ attrValue.value "( x ) { y }" @@ -973,7 +987,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -121╭─ { console.log("y") } a/> +123╭─ { console.log("y") } a/> │ │││ │ │╰─ openTagEnd:selfClosed "/>" │ │││ │ ╰─ attrName │ │││ ╰─ attrValue.value "(x) => { console.log(\"y\") }" @@ -982,7 +996,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -122╭─ { console.log("y") } a/> +124╭─ { console.log("y") } a/> │ │││ │ │╰─ openTagEnd:selfClosed "/>" │ │││ │ ╰─ attrName │ │││ ╰─ attrValue.value "async x => { console.log(\"y\") }" @@ -991,7 +1005,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -123╭─ +125╭─ │ │││ │ │╰─ openTagEnd:selfClosed "/>" │ │││ │ ╰─ attrName │ │││ ╰─ attrValue.value "function (x) { console.log(\"y\") }" @@ -1000,7 +1014,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -124╭─ { console.log("y") } a/> +126╭─ { console.log("y") } a/> │ │││ │ │╰─ openTagEnd:selfClosed "/>" │ │││ │ ╰─ attrName │ │││ ╰─ attrValue.value "x => { console.log(\"y\") }" @@ -1009,7 +1023,7 @@ │ │╰─ tagName │ ├─ text "\n" ╰─ ╰─ openTagStart -125╭─ +127╭─ │ │││ │ │╰─ openTagEnd:selfClosed "/>" │ │││ │ ╰─ attrName │ │││ ╰─ attrValue.value "async function (x) { console.log(\"y\") }" diff --git a/src/__tests__/fixtures/attr-operators-space-between/input.marko b/src/__tests__/fixtures/attr-operators-space-between/input.marko index f960d5ca..31fb88e0 100644 --- a/src/__tests__/fixtures/attr-operators-space-between/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-between/input.marko @@ -108,6 +108,8 @@ a = async function (x) { console.log("y") } a + + y > y a/> diff --git a/src/states/EXPRESSION.ts b/src/states/EXPRESSION.ts index 5fa20467..f3cac17a 100644 --- a/src/states/EXPRESSION.ts +++ b/src/states/EXPRESSION.ts @@ -8,12 +8,6 @@ import { ErrorCode, } from "../internal"; -const enum PatternType { - HTML_ATTRS, - CONCISE_ATTRS, - CONCISE_ATTRS_GROUP, -} - export interface ExpressionMeta extends Meta { groupStack: number[]; terminator: number | (number | number[])[]; @@ -22,9 +16,18 @@ export interface ExpressionMeta extends Meta { terminatedByWhitespace: boolean; } -const htmlAttrsPattern = buildPattern(PatternType.HTML_ATTRS); -const conciseAttrsPattern = buildPattern(PatternType.CONCISE_ATTRS); -const conciseAttrsGroupPattern = buildPattern(PatternType.CONCISE_ATTRS_GROUP); +const unaryKeywords = [ + "async", + "await", + "keyof", + "class", + "function", + "new", + "typeof", + "void", +] as const; + +const binaryKeywords = ["instanceof", "in", "as", "extends"] as const; export const EXPRESSION: StateDefinition = { name: "EXPRESSION", @@ -48,7 +51,7 @@ export const EXPRESSION: StateDefinition = { char(code, expression) { if (!expression.groupStack.length) { if (expression.terminatedByWhitespace && isWhitespaceCode(code)) { - if (!checkForOperators(this, expression)) { + if (!checkForOperators(this, expression, false)) { this.exitState(); } return; @@ -86,11 +89,7 @@ export const EXPRESSION: StateDefinition = { this.pos++; break; default: { - if ( - canCharCodeBeFollowedByDivision( - this.getPreviousNonWhitespaceCharCode() - ) - ) { + if (canFollowDivision(this.getPreviousNonWhitespaceCharCode())) { this.pos++; this.consumeWhitespace(); } else { @@ -145,7 +144,7 @@ export const EXPRESSION: StateDefinition = { if ( !expression.groupStack.length && (expression.terminatedByEOL || expression.terminatedByWhitespace) && - !checkForOperators(this, expression) + !checkForOperators(this, expression, true) ) { this.exitState(); } @@ -212,67 +211,31 @@ export const EXPRESSION: StateDefinition = { return() {}, }; -function buildPattern(type: PatternType) { - const space = type === PatternType.CONCISE_ATTRS ? "[ \\t]" : "\\s"; - const binary = - "(?:[!~*%&^|?<]+=*)+" + // Any of these characters can always continue an expression - "|:+(?!=)" + // Match a colon without matching a bound attribute - "|[>/+=-]+=|=>" + // Match equality and multi char assignment operators w/o matching equals by itself - `|(?${type === PatternType.HTML_ATTRS ? "{2,}" : "+"}` + // in html mode only consume closing angle brackets if it is >> - "|[ \\t]+(?:in(?:stanceof)?|as|extends)(?=[ \\t]+[^=/,;:>])"; // We only continue after word operators (instanceof/in) when they are not followed by a terminator - const unary = - "\\b(?])`; // if we have spaces followed by an opening bracket, we'll consume the spaces and let the expression state handle the brackets - const lookBehindPattern = `(?<=${unary}|${binary})`; - return new RegExp(`${lookAheadPattern}|${lookBehindPattern}`, "ym"); -} +function checkForOperators( + parser: Parser, + expression: ExpressionMeta, + eol: boolean +) { + if (expression.skipOperators) return false; -function checkForOperators(parser: Parser, expression: ExpressionMeta) { - if (expression.skipOperators) { - return false; + const { pos, data } = parser; + const lookBehindPos = lookBehindForOperator(data, pos); + if (lookBehindPos !== -1) { + parser.consumeWhitespace(); + parser.forward = 0; + return true; } - - const pattern = parser.isConcise - ? parser.activeTag?.stage === STATE.TAG_STAGE.ATTR_GROUP - ? conciseAttrsGroupPattern - : conciseAttrsPattern - : expression.terminatedByEOL - ? conciseAttrsPattern - : htmlAttrsPattern; - pattern.lastIndex = parser.pos; - const matches = pattern.exec(parser.data); - - if (matches) { - const [match] = matches; - if (match.length === 0) { - // We matched a look behind. - parser.consumeWhitespace(); - } else { - // We matched a look ahead. - parser.pos += match.length; + const terminatedByEOL = expression.terminatedByEOL || parser.isConcise; + if (!(terminatedByEOL && eol)) { + const lookAheadPos = lookAheadForOperator(data, pos, terminatedByEOL); + if (lookAheadPos !== -1) { + parser.pos = lookAheadPos; + parser.forward = 0; + return true; } - - // After this point we should be on the character we want to process next - // so we don't want to move forward and miss that character. - parser.forward = 0; - } else { - return false; } - return true; + return false; } function checkForTerminators( @@ -296,11 +259,173 @@ function checkForTerminators( } } -function canCharCodeBeFollowedByDivision(code: number) { +function lookBehindForOperator(data: string, pos: number): number { + const curPos = pos - 1; + const code = data.charCodeAt(curPos); + + switch (code) { + case CODE.AMPERSAND: + case CODE.ASTERISK: + case CODE.CARET: + case CODE.COLON: + case CODE.EQUAL: + case CODE.EXCLAMATION: + case CODE.OPEN_ANGLE_BRACKET: + case CODE.CLOSE_ANGLE_BRACKET: + case CODE.PERCENT: + case CODE.PERIOD: + case CODE.PIPE: + case CODE.QUESTION: + case CODE.TILDE: + return curPos; + + case CODE.PLUS: + case CODE.HYPHEN: { + // special case -- and ++ + if (data.charCodeAt(curPos - 1) === code) { + // Check if we should continue for another reason. + // eg "typeof++ x" + return lookBehindForOperator( + data, + lookBehindWhile(isWhitespaceCode, data, curPos - 2) + ); + } + + return curPos; + } + + default: { + for (const keyword of unaryKeywords) { + const keywordPos = lookBehindFor(data, curPos, keyword); + if (keywordPos !== -1) { + return keywordPos === 0 || + data.charCodeAt(keywordPos - 1) !== CODE.PERIOD + ? keywordPos + : -1; + } + } + return -1; + } + } +} + +function lookAheadForOperator( + data: string, + pos: number, + terminatedByEOL: boolean +): number { + const isSpace = terminatedByEOL ? isIndentCode : isWhitespaceCode; + const curPos = lookAheadWhile(isSpace, data, pos + 1); + const code = data.charCodeAt(curPos); + + switch (code) { + case CODE.AMPERSAND: + case CODE.ASTERISK: + case CODE.CARET: + case CODE.EXCLAMATION: + case CODE.OPEN_ANGLE_BRACKET: + case CODE.PERCENT: + case CODE.PIPE: + case CODE.QUESTION: + case CODE.TILDE: + case CODE.PLUS: + return curPos + 1; + + case CODE.HYPHEN: { + // Can't match -- when in concise mode. + return terminatedByEOL && data.charCodeAt(curPos + 1) === CODE.HYPHEN + ? -1 + : curPos + 1; + } + + case CODE.OPEN_CURLY_BRACE: + case CODE.OPEN_PAREN: + return curPos; // defers to base expression state to track block groups. + + case CODE.FORWARD_SLASH: + return terminatedByEOL || + // Only match a / in html mode if it's not `/>` + data.charCodeAt(curPos + 1) !== CODE.CLOSE_ANGLE_BRACKET + ? curPos // defers to base expression state to track regexp groups. + : -1; + + case CODE.COLON: + // Only match a colon if its not := + return data.charCodeAt(curPos + 1) === CODE.EQUAL ? -1 : curPos + 1; + + case CODE.PERIOD: + // Only match a dot if its not ... + return data.charCodeAt(curPos + 1) === CODE.PERIOD ? -1 : curPos + 1; + + case CODE.CLOSE_ANGLE_BRACKET: + // In concise mode a > is a normal operator. + if (terminatedByEOL) return curPos + 1; + + // Otherwise we match >=, >> and >>> + switch (data.charCodeAt(curPos + 1)) { + case CODE.EQUAL: + return curPos + 2; + case CODE.CLOSE_ANGLE_BRACKET: + return ( + curPos + + (data.charCodeAt(curPos + 2) === CODE.CLOSE_ANGLE_BRACKET ? 3 : 2) + ); + default: + return -1; + } + case CODE.EQUAL: + // We'll match ==, ===, and => + // This is primarily to support => since otherwise it'd end some expressions. + switch (data.charCodeAt(curPos + 1)) { + case CODE.CLOSE_ANGLE_BRACKET: + return curPos + 2; + case CODE.EQUAL: + return curPos + (data.charCodeAt(curPos + 2) === CODE.EQUAL ? 3 : 2); + default: + return -1; + } + + default: { + for (const keyword of binaryKeywords) { + let nextPos = lookAheadFor(data, curPos, keyword); + if (nextPos === -1) continue; + + const max = data.length - 1; + if (nextPos === max) return -1; + + let nextCode = data.charCodeAt(nextPos + 1); + if (isWhitespaceCode(nextCode)) { + // skip any whitespace after the operator + nextPos = lookAheadWhile(isWhitespaceCode, data, nextPos + 2); + if (nextPos === max) return -1; + nextCode = data.charCodeAt(nextPos); + } else if (isWordCode(nextCode)) { + // bail if we are continuing a word, eg "**in**teger" + return -1; + } + + // finally check that this is not followed by a terminator. + switch (nextCode) { + case CODE.COLON: + case CODE.COMMA: + case CODE.EQUAL: + case CODE.FORWARD_SLASH: + case CODE.CLOSE_ANGLE_BRACKET: + case CODE.SEMICOLON: + return -1; + default: + return nextPos; + } + } + + return -1; + } + } +} + +function canFollowDivision(code: number) { return ( - (code >= CODE.NUMBER_0 && code <= CODE.NUMBER_9) || - (code >= CODE.UPPER_A && code <= CODE.UPPER_Z) || - (code >= CODE.LOWER_A && code <= CODE.LOWER_Z) || + isWordCode(code) || code === CODE.PERCENT || code === CODE.CLOSE_PAREN || code === CODE.PERIOD || @@ -309,3 +434,73 @@ function canCharCodeBeFollowedByDivision(code: number) { code === CODE.CLOSE_CURLY_BRACE ); } + +function isWordCode(code: number) { + return ( + (code >= CODE.UPPER_A && code <= CODE.UPPER_Z) || + (code >= CODE.LOWER_A && code <= CODE.LOWER_Z) || + (code >= CODE.NUMBER_0 && code <= CODE.NUMBER_9) || + code === CODE.UNDERSCORE + ); +} + +function isIndentCode(code: number) { + return code === CODE.TAB || code === CODE.SPACE; +} + +function lookAheadWhile( + match: (code: number) => boolean, + data: string, + pos: number +) { + const max = data.length; + for (let i = pos; i < max; i++) { + if (!match(data.charCodeAt(i))) return i; + } + + return max - 1; +} + +function lookBehindWhile( + match: (code: number) => boolean, + data: string, + pos: number +) { + let i = pos; + + do { + if (!match(data.charCodeAt(i))) { + return i + 1; + } + } while (i--); + + return 0; +} + +function lookBehindFor(data: string, pos: number, str: string) { + let i = str.length; + const endPos = pos - i + 1; + if (endPos < 0) return -1; + + while (i--) { + if (data.charCodeAt(endPos + i) !== str.charCodeAt(i)) { + return -1; + } + } + + return endPos; +} + +function lookAheadFor(data: string, pos: number, str: string) { + let i = str.length; + const endPos = pos + i; + if (endPos > data.length) return -1; + + while (i--) { + if (data.charCodeAt(pos + i) !== str.charCodeAt(i)) { + return -1; + } + } + + return endPos - 1; +} diff --git a/src/util/constants.ts b/src/util/constants.ts index 3ed47da0..9ac8a85d 100644 --- a/src/util/constants.ts +++ b/src/util/constants.ts @@ -27,6 +27,7 @@ export const enum CODE { DOLLAR = 36, PERCENT = 37, PERIOD = 46, + PLUS = 43, COMMA = 44, COLON = 58, SEMICOLON = 59, @@ -36,6 +37,10 @@ export const enum CODE { CARRIAGE_RETURN = 13, SPACE = 32, TAB = 9, + AMPERSAND = 38, + CARET = 94, + TILDE = 126, + UNDERSCORE = 95, } // Same format as https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#position From f886c2716a6e9095be9e9ed63fa6c4db57fcc38d Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Fri, 5 Aug 2022 21:49:30 -0700 Subject: [PATCH 2/7] feat: improve continue expressions with equals --- .../attr-complex-functions.expected.txt | 28 +++++- .../attr-complex-functions/input.marko | 3 + src/states/ATTRIBUTE.ts | 13 ++- src/states/EXPRESSION.ts | 98 +++++++++++++------ src/states/OPEN_TAG.ts | 16 ++- 5 files changed, 118 insertions(+), 40 deletions(-) diff --git a/src/__tests__/fixtures/attr-complex-functions/__snapshots__/attr-complex-functions.expected.txt b/src/__tests__/fixtures/attr-complex-functions/__snapshots__/attr-complex-functions.expected.txt index 8a4dd549..4452261c 100644 --- a/src/__tests__/fixtures/attr-complex-functions/__snapshots__/attr-complex-functions.expected.txt +++ b/src/__tests__/fixtures/attr-complex-functions/__snapshots__/attr-complex-functions.expected.txt @@ -20,7 +20,31 @@ │ ├─ closeTagEnd(tag) │ ├─ openTagEnd ╰─ ╰─ tagName "tag" -4╭─ tag a = async x => { console.log("y") } b +4╭─ tag a = x => y b + │ │ │ │ │ ╰─ attrName + │ │ │ │ ╰─ attrValue.value "x => y" + │ │ │ ╰─ attrValue "= x => y" + │ │ ╰─ attrName + │ ├─ closeTagEnd(tag) + │ ├─ openTagEnd + ╰─ ╰─ tagName "tag" +5╭─ tag a = x => y + 1 b + │ │ │ │ │ ╰─ attrName + │ │ │ │ ╰─ attrValue.value "x => y + 1" + │ │ │ ╰─ attrValue "= x => y + 1" + │ │ ╰─ attrName + │ ├─ closeTagEnd(tag) + │ ├─ openTagEnd + ╰─ ╰─ tagName "tag" +6╭─ tag a = x => y = 1 b + │ │ │ │ │ ╰─ attrName + │ │ │ │ ╰─ attrValue.value "x => y = 1" + │ │ │ ╰─ attrValue "= x => y = 1" + │ │ ╰─ attrName + │ ├─ closeTagEnd(tag) + │ ├─ openTagEnd + ╰─ ╰─ tagName "tag" +7╭─ tag a = async x => { console.log("y") } b │ │ │ │ │ ╰─ attrName │ │ │ │ ╰─ attrValue.value "async x => { console.log(\"y\") }" │ │ │ ╰─ attrValue "= async x => { console.log(\"y\") }" @@ -28,7 +52,7 @@ │ ├─ closeTagEnd(tag) │ ├─ openTagEnd ╰─ ╰─ tagName "tag" -5╭─ tag a = async function (x) { console.log("y") } b +8╭─ tag a = async function (x) { console.log("y") } b │ │ │ │ │ │├─ closeTagEnd(tag) │ │ │ │ │ │╰─ openTagEnd │ │ │ │ │ ╰─ attrName diff --git a/src/__tests__/fixtures/attr-complex-functions/input.marko b/src/__tests__/fixtures/attr-complex-functions/input.marko index 148f6ccd..2d061747 100644 --- a/src/__tests__/fixtures/attr-complex-functions/input.marko +++ b/src/__tests__/fixtures/attr-complex-functions/input.marko @@ -1,5 +1,8 @@ tag a = function (x) { console.log("y") } b tag a = (x) => { console.log("y") } b tag a = x => { console.log("y") } b +tag a = x => y b +tag a = x => y + 1 b +tag a = x => y = 1 b tag a = async x => { console.log("y") } b tag a = async function (x) { console.log("y") } b \ No newline at end of file diff --git a/src/states/ATTRIBUTE.ts b/src/states/ATTRIBUTE.ts index 869f6ee5..d908ea63 100644 --- a/src/states/ATTRIBUTE.ts +++ b/src/states/ATTRIBUTE.ts @@ -9,6 +9,7 @@ import { Meta, ErrorCode, } from "../internal"; +import { OPERATOR_TERMINATOR } from "./EXPRESSION"; const enum ATTR_STAGE { UNKNOWN, @@ -109,9 +110,14 @@ export const ATTRIBUTE: StateDefinition = { attr.stage = ATTR_STAGE.VALUE; const expr = this.enterState(STATE.EXPRESSION); expr.terminatedByWhitespace = true; - expr.terminator = this.isConcise - ? CONCISE_VALUE_TERMINATORS - : HTML_VALUE_TERMINATORS; + + if (this.isConcise) { + expr.terminator = CONCISE_VALUE_TERMINATORS; + expr.operatorTerminator = OPERATOR_TERMINATOR.Hyphens; + } else { + expr.terminator = HTML_VALUE_TERMINATORS; + expr.operatorTerminator = OPERATOR_TERMINATOR.CloseAngleBracket; + } } else if (code === CODE.OPEN_PAREN) { ensureAttrName(this, attr); attr.stage = ATTR_STAGE.ARGUMENT; @@ -124,7 +130,6 @@ export const ATTRIBUTE: StateDefinition = { this.pos++; // skip { this.forward = 0; const expr = this.enterState(STATE.EXPRESSION); - expr.terminatedByWhitespace = false; expr.terminator = CODE.CLOSE_CURLY_BRACE; } else if (attr.stage === ATTR_STAGE.UNKNOWN) { attr.stage = ATTR_STAGE.NAME; diff --git a/src/states/EXPRESSION.ts b/src/states/EXPRESSION.ts index f3cac17a..8ed573c6 100644 --- a/src/states/EXPRESSION.ts +++ b/src/states/EXPRESSION.ts @@ -8,12 +8,24 @@ import { ErrorCode, } from "../internal"; +export enum OPERATOR_TERMINATOR { + // All operators are ok. + None = 0, + // Prevents double hyphens (set when in concise mode attrs). + Hyphens = 1, + // Prevent equals continuing an expression (set for tag variable expression) + AttrValue = 1 << 1, + // Prevent close angle bracket continuing an expression (set for html mode attrs) + CloseAngleBracket = 1 << 2, +} + export interface ExpressionMeta extends Meta { groupStack: number[]; terminator: number | (number | number[])[]; skipOperators: boolean; terminatedByEOL: boolean; terminatedByWhitespace: boolean; + operatorTerminator: OPERATOR_TERMINATOR; } const unaryKeywords = [ @@ -43,6 +55,7 @@ export const EXPRESSION: StateDefinition = { skipOperators: false, terminatedByEOL: false, terminatedByWhitespace: false, + operatorTerminator: OPERATOR_TERMINATOR.None, }; }, @@ -227,7 +240,15 @@ function checkForOperators( } const terminatedByEOL = expression.terminatedByEOL || parser.isConcise; if (!(terminatedByEOL && eol)) { - const lookAheadPos = lookAheadForOperator(data, pos, terminatedByEOL); + const lookAheadPos = lookAheadForOperator( + data, + lookAheadWhile( + terminatedByEOL ? isIndentCode : isWhitespaceCode, + data, + pos + 1 + ), + expression.operatorTerminator + ); if (lookAheadPos !== -1) { parser.pos = lookAheadPos; parser.forward = 0; @@ -312,13 +333,9 @@ function lookBehindForOperator(data: string, pos: number): number { function lookAheadForOperator( data: string, pos: number, - terminatedByEOL: boolean + terminator: OPERATOR_TERMINATOR ): number { - const isSpace = terminatedByEOL ? isIndentCode : isWhitespaceCode; - const curPos = lookAheadWhile(isSpace, data, pos + 1); - const code = data.charCodeAt(curPos); - - switch (code) { + switch (data.charCodeAt(pos)) { case CODE.AMPERSAND: case CODE.ASTERISK: case CODE.CARET: @@ -329,65 +346,86 @@ function lookAheadForOperator( case CODE.QUESTION: case CODE.TILDE: case CODE.PLUS: - return curPos + 1; + return pos + 1; case CODE.HYPHEN: { - // Can't match -- when in concise mode. - return terminatedByEOL && data.charCodeAt(curPos + 1) === CODE.HYPHEN + return (terminator & OPERATOR_TERMINATOR.Hyphens) === + OPERATOR_TERMINATOR.Hyphens && data.charCodeAt(pos + 1) === CODE.HYPHEN ? -1 - : curPos + 1; + : pos + 1; } case CODE.OPEN_CURLY_BRACE: case CODE.OPEN_PAREN: - return curPos; // defers to base expression state to track block groups. + return pos; // defers to base expression state to track block groups. case CODE.FORWARD_SLASH: - return terminatedByEOL || - // Only match a / in html mode if it's not `/>` - data.charCodeAt(curPos + 1) !== CODE.CLOSE_ANGLE_BRACKET - ? curPos // defers to base expression state to track regexp groups. - : -1; + return (terminator & OPERATOR_TERMINATOR.CloseAngleBracket) === + OPERATOR_TERMINATOR.CloseAngleBracket && + data.charCodeAt(pos + 1) === CODE.CLOSE_ANGLE_BRACKET + ? -1 + : pos; // defers to base expression state to track regexp groups. case CODE.COLON: - // Only match a colon if its not := - return data.charCodeAt(curPos + 1) === CODE.EQUAL ? -1 : curPos + 1; + return (terminator & OPERATOR_TERMINATOR.AttrValue) === + OPERATOR_TERMINATOR.AttrValue && data.charCodeAt(pos + 1) === CODE.EQUAL + ? -1 + : pos + 1; case CODE.PERIOD: // Only match a dot if its not ... - return data.charCodeAt(curPos + 1) === CODE.PERIOD ? -1 : curPos + 1; + return data.charCodeAt(pos + 1) === CODE.PERIOD ? -1 : pos + 1; case CODE.CLOSE_ANGLE_BRACKET: // In concise mode a > is a normal operator. - if (terminatedByEOL) return curPos + 1; + if ( + (terminator & OPERATOR_TERMINATOR.CloseAngleBracket) !== + OPERATOR_TERMINATOR.CloseAngleBracket + ) { + return pos + 1; + } // Otherwise we match >=, >> and >>> - switch (data.charCodeAt(curPos + 1)) { + switch (data.charCodeAt(pos + 1)) { case CODE.EQUAL: - return curPos + 2; + return pos + 2; case CODE.CLOSE_ANGLE_BRACKET: return ( - curPos + - (data.charCodeAt(curPos + 2) === CODE.CLOSE_ANGLE_BRACKET ? 3 : 2) + pos + + (data.charCodeAt(pos + 2) === CODE.CLOSE_ANGLE_BRACKET ? 3 : 2) ); default: return -1; } case CODE.EQUAL: + if ( + (terminator & OPERATOR_TERMINATOR.AttrValue) === + OPERATOR_TERMINATOR.AttrValue + ) { + return -1; + } + + if ( + (terminator & OPERATOR_TERMINATOR.CloseAngleBracket) !== + OPERATOR_TERMINATOR.CloseAngleBracket + ) { + return pos + 1; + } + // We'll match ==, ===, and => - // This is primarily to support => since otherwise it'd end some expressions. - switch (data.charCodeAt(curPos + 1)) { + // This is primarily to support => since otherwise it'd end a CloseAngleBracket terminator. + switch (data.charCodeAt(pos + 1)) { case CODE.CLOSE_ANGLE_BRACKET: - return curPos + 2; + return pos + 2; case CODE.EQUAL: - return curPos + (data.charCodeAt(curPos + 2) === CODE.EQUAL ? 3 : 2); + return pos + (data.charCodeAt(pos + 2) === CODE.EQUAL ? 3 : 2); default: return -1; } default: { for (const keyword of binaryKeywords) { - let nextPos = lookAheadFor(data, curPos, keyword); + let nextPos = lookAheadFor(data, pos, keyword); if (nextPos === -1) continue; const max = data.length - 1; diff --git a/src/states/OPEN_TAG.ts b/src/states/OPEN_TAG.ts index e8ae3d0d..ebf47142 100644 --- a/src/states/OPEN_TAG.ts +++ b/src/states/OPEN_TAG.ts @@ -8,8 +8,9 @@ import { TagType, ErrorCode, } from "../internal"; +import { OPERATOR_TERMINATOR } from "./EXPRESSION"; -export const enum TAG_STAGE { +const enum TAG_STAGE { UNKNOWN, VAR, ARGUMENT, @@ -310,9 +311,16 @@ export const OPEN_TAG: StateDefinition = { const expr = this.enterState(STATE.EXPRESSION); expr.terminatedByWhitespace = true; - expr.terminator = this.isConcise - ? CONCISE_TAG_VAR_TERMINATORS - : HTML_TAG_VAR_TERMINATORS; + + if (this.isConcise) { + expr.terminator = CONCISE_TAG_VAR_TERMINATORS; + expr.operatorTerminator = + OPERATOR_TERMINATOR.AttrValue | OPERATOR_TERMINATOR.Hyphens; + } else { + expr.terminator = HTML_TAG_VAR_TERMINATORS; + expr.operatorTerminator = + OPERATOR_TERMINATOR.AttrValue | OPERATOR_TERMINATOR.CloseAngleBracket; + } } else if (code === CODE.OPEN_PAREN && !tag.hasAttrs) { if (tag.hasArgs) { this.emitError( From eb9f52cd76778d145cb7e342e4e9e87ec4ec60eb Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Fri, 5 Aug 2022 21:59:46 -0700 Subject: [PATCH 3/7] fix: allow hyphen continuation inside concise attr groups --- .../attr-concise-hyphens.expected.txt | 18 ++++++++++++++++++ .../fixtures/attr-concise-hyphens/input.marko | 3 +++ src/states/ATTRIBUTE.ts | 6 +++++- src/states/OPEN_TAG.ts | 2 +- 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 src/__tests__/fixtures/attr-concise-hyphens/__snapshots__/attr-concise-hyphens.expected.txt create mode 100644 src/__tests__/fixtures/attr-concise-hyphens/input.marko diff --git a/src/__tests__/fixtures/attr-concise-hyphens/__snapshots__/attr-concise-hyphens.expected.txt b/src/__tests__/fixtures/attr-concise-hyphens/__snapshots__/attr-concise-hyphens.expected.txt new file mode 100644 index 00000000..5795dd7a --- /dev/null +++ b/src/__tests__/fixtures/attr-concise-hyphens/__snapshots__/attr-concise-hyphens.expected.txt @@ -0,0 +1,18 @@ +1╭─ a b=c -- y + │ │ │││ │ ╰─ text + │ │ │││ ╰─ openTagEnd + │ │ ││╰─ attrValue.value + │ │ │╰─ attrValue "=c" + │ │ ╰─ attrName + ╰─ ╰─ tagName +2├─ +3╭─ a [b=c -- y] + │ │ │││ ╰─ attrName + │ │ ││╰─ attrValue.value "c --" + │ │ │╰─ attrValue "=c --" + │ │ ╰─ attrName + │ ├─ closeTagEnd(a) + ╰─ ╰─ tagName +4╭─ + │ ├─ openTagEnd + ╰─ ╰─ closeTagEnd(a) \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-concise-hyphens/input.marko b/src/__tests__/fixtures/attr-concise-hyphens/input.marko new file mode 100644 index 00000000..04201acf --- /dev/null +++ b/src/__tests__/fixtures/attr-concise-hyphens/input.marko @@ -0,0 +1,3 @@ +a b=c -- y + +a [b=c -- y] diff --git a/src/states/ATTRIBUTE.ts b/src/states/ATTRIBUTE.ts index d908ea63..f586ffa7 100644 --- a/src/states/ATTRIBUTE.ts +++ b/src/states/ATTRIBUTE.ts @@ -10,6 +10,7 @@ import { ErrorCode, } from "../internal"; import { OPERATOR_TERMINATOR } from "./EXPRESSION"; +import { TAG_STAGE } from "./OPEN_TAG"; const enum ATTR_STAGE { UNKNOWN, @@ -113,7 +114,10 @@ export const ATTRIBUTE: StateDefinition = { if (this.isConcise) { expr.terminator = CONCISE_VALUE_TERMINATORS; - expr.operatorTerminator = OPERATOR_TERMINATOR.Hyphens; + + if (this.activeTag!.stage !== TAG_STAGE.ATTR_GROUP) { + expr.operatorTerminator = OPERATOR_TERMINATOR.Hyphens; + } } else { expr.terminator = HTML_VALUE_TERMINATORS; expr.operatorTerminator = OPERATOR_TERMINATOR.CloseAngleBracket; diff --git a/src/states/OPEN_TAG.ts b/src/states/OPEN_TAG.ts index ebf47142..b4e757c1 100644 --- a/src/states/OPEN_TAG.ts +++ b/src/states/OPEN_TAG.ts @@ -10,7 +10,7 @@ import { } from "../internal"; import { OPERATOR_TERMINATOR } from "./EXPRESSION"; -const enum TAG_STAGE { +export enum TAG_STAGE { UNKNOWN, VAR, ARGUMENT, From 00cce1f3607395d8737ea0c2547a2ee29cfd501c Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Mon, 8 Aug 2022 08:49:21 -0700 Subject: [PATCH 4/7] refactor: move terminator logic out of expressions state --- .../attr-operators-space-after.expected.txt | 93 ++++------ .../attr-operators-space-after/input.marko | 2 +- .../attr-operators-space-before.expected.txt | 20 +- .../attr-operators-space-before/input.marko | 2 +- .../attr-operators-space-between.expected.txt | 22 +-- .../attr-operators-space-between/input.marko | 2 +- src/states/ATTRIBUTE.ts | 174 +++++++++++++----- src/states/EXPRESSION.ts | 164 ++++------------- src/states/INLINE_SCRIPT.ts | 11 +- src/states/OPEN_TAG.ts | 70 +++---- src/states/PLACEHOLDER.ts | 6 +- src/states/TAG_NAME.ts | 6 +- src/states/TEMPLATE_STRING.ts | 6 +- src/util/util.ts | 12 ++ 14 files changed, 285 insertions(+), 305 deletions(-) diff --git a/src/__tests__/fixtures/attr-operators-space-after/__snapshots__/attr-operators-space-after.expected.txt b/src/__tests__/fixtures/attr-operators-space-after/__snapshots__/attr-operators-space-after.expected.txt index b0617a95..1fdb542c 100644 --- a/src/__tests__/fixtures/attr-operators-space-after/__snapshots__/attr-operators-space-after.expected.txt +++ b/src/__tests__/fixtures/attr-operators-space-after/__snapshots__/attr-operators-space-after.expected.txt @@ -512,10 +512,10 @@ │ │╰─ tagName ╰─ ╰─ openTagStart 66╭─ = y a/> - │ │││││╰─ text "= y a/>\n" - │ ││││╰─ openTagEnd - │ │││╰─ attrValue.value - │ ││├─ attrValue "=x" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x>= y" + │ ││├─ attrValue "=x>= y" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart @@ -534,7 +534,6 @@ │ ││├─ attrValue "=x&&= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 69╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -543,7 +542,6 @@ │ ││├─ attrValue "=x|= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 70╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -552,7 +550,6 @@ │ ││├─ attrValue "=x||= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 71╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -561,7 +558,6 @@ │ ││├─ attrValue "=x^= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 72╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -570,22 +566,20 @@ │ ││├─ attrValue "=x~= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 73╭─ >= y a/> - │ │││││╰─ text ">= y a/>\n" - │ ││││╰─ openTagEnd - │ │││╰─ attrValue.value - │ ││├─ attrValue "=x" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x>>= y" + │ ││├─ attrValue "=x>>= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 74╭─ >>= y a/> - │ │││││╰─ text ">>= y a/>\n" - │ ││││╰─ openTagEnd - │ │││╰─ attrValue.value - │ ││├─ attrValue "=x" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x>>>= y" + │ ││├─ attrValue "=x>>>= y" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart @@ -604,7 +598,6 @@ │ ││├─ attrValue "=x/= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 77╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -613,7 +606,6 @@ │ ││├─ attrValue "=x*= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 78╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -622,7 +614,6 @@ │ ││├─ attrValue "=x**= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 79╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -631,7 +622,6 @@ │ ││├─ attrValue "=x%= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 80╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -640,7 +630,6 @@ │ ││├─ attrValue "=x+= y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 81╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -649,7 +638,6 @@ │ ││├─ attrValue "=x++ + y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 82╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -658,34 +646,31 @@ │ ││├─ attrValue "=x+ ++ y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart -83╭─ y - │ ││││││ ││╰─ openTagEnd:selfClosed "/>" - │ ││││││ │╰─ tagName - │ ││││││ ╰─ openTagStart +83╭─ y + │ ││││││ │ │╰─ closeTagEnd(a) + │ ││││││ │ ╰─ closeTagName + │ ││││││ ╰─ closeTagStart "> y a/> - │ │││││╰─ text "> y a/>\n" - │ ││││╰─ openTagEnd - │ │││╰─ attrValue.value - │ ││├─ attrValue "=x" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x>> y" + │ ││├─ attrValue "=x>> y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 85╭─ >> y a/> - │ │││││╰─ text ">> y a/>\n" - │ ││││╰─ openTagEnd - │ │││╰─ attrValue.value - │ ││├─ attrValue "=x" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x>>> y" + │ ││├─ attrValue "=x>>> y" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart @@ -704,33 +689,30 @@ │ ││├─ attrValue "=z {y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 88╭─ y a/> - │ ││││ │╰─ text " y a/>\n" - │ ││││ ╰─ openTagEnd - │ │││╰─ attrValue.value "x=" - │ ││├─ attrValue "=x=" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x=> y" + │ ││├─ attrValue "=x=> y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 89╭─ (y ) a/> - │ ││││ │╰─ text " (y ) a/>\n" - │ ││││ ╰─ openTagEnd - │ │││╰─ attrValue.value "x=" - │ ││├─ attrValue "=x=" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x=> (y )" + │ ││├─ attrValue "=x=> (y )" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart 90╭─ {y } a/> - │ ││││ │╰─ text " {y } a/>\n" - │ ││││ ╰─ openTagEnd - │ │││╰─ attrValue.value "x=" - │ ││├─ attrValue "=x=" + │ ││││ │╰─ openTagEnd:selfClosed "/>" + │ ││││ ╰─ attrName + │ │││╰─ attrValue.value "x=> {y }" + │ ││├─ attrValue "=x=> {y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ error(MISSING_END_TAG:Missing ending "a" tag) "" ╰─ ╰─ openTagStart 91╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -740,5 +722,4 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -92╭─ - ╰─ ╰─ text "\n" \ No newline at end of file +92╰─ \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-operators-space-after/input.marko b/src/__tests__/fixtures/attr-operators-space-after/input.marko index 70a18e02..8814d976 100644 --- a/src/__tests__/fixtures/attr-operators-space-after/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-after/input.marko @@ -80,7 +80,7 @@ a=(x ) {y } a - y + y > y a/> >> y a/> diff --git a/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt b/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt index 9290fab3..b6909124 100644 --- a/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt +++ b/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt @@ -647,17 +647,16 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -83╭─ y - │ ││││ ││ ││╰─ openTagEnd:selfClosed "/>" - │ ││││ ││ │╰─ tagName - │ ││││ ││ ╰─ openTagStart +83╭─ y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart "" ╰─ ╰─ openTagStart 84╭─ >y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -666,7 +665,6 @@ │ ││├─ attrValue "=x >>y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 85╭─ >>y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -675,7 +673,6 @@ │ ││├─ attrValue "=x >>>y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 86╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -684,7 +681,6 @@ │ ││├─ attrValue "=x (y )" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 87╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -693,7 +689,6 @@ │ ││├─ attrValue "=x {y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 88╭─ y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -702,7 +697,6 @@ │ ││├─ attrValue "=x =>y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 89╭─ (y ) a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -711,7 +705,6 @@ │ ││├─ attrValue "=x => (y )" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 90╭─ {y } a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -720,7 +713,6 @@ │ ││├─ attrValue "=x => {y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 91╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -729,7 +721,5 @@ │ ││├─ attrValue "=( x ) {y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart -92╭─ - ╰─ ╰─ text "\n" \ No newline at end of file +92╰─ \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-operators-space-before/input.marko b/src/__tests__/fixtures/attr-operators-space-before/input.marko index f87ef5aa..a482cb49 100644 --- a/src/__tests__/fixtures/attr-operators-space-before/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-before/input.marko @@ -80,7 +80,7 @@ a=( x ) {y } a -y +y >y a/> >>y a/> diff --git a/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt b/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt index 5c8b9eb3..ceb4c149 100644 --- a/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt +++ b/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt @@ -903,17 +903,16 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -114╭─ y - │ ││││ ││ ││╰─ openTagEnd:selfClosed "/>" - │ ││││ ││ │╰─ tagName - │ ││││ ││ ╰─ openTagStart +114╭─ y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart "" ╰─ ╰─ openTagStart 115╭─ > y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -922,7 +921,6 @@ │ ││├─ attrValue "=x >> y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 116╭─ >> y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -931,7 +929,6 @@ │ ││├─ attrValue "=x >>> y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 117╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -940,7 +937,6 @@ │ ││├─ attrValue "=x ( y )" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 118╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -949,7 +945,6 @@ │ ││├─ attrValue "=x { y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 119╭─ y a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -958,7 +953,6 @@ │ ││├─ attrValue "=x => y" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 120╭─ ( y ) a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -967,7 +961,6 @@ │ ││├─ attrValue "=x => ( y )" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 121╭─ { y } a/> │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -976,7 +969,6 @@ │ ││├─ attrValue "=x => { y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 122╭─ │ ││││ │╰─ openTagEnd:selfClosed "/>" @@ -985,7 +977,6 @@ │ ││├─ attrValue "=( x ) { y }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 123╭─ { console.log("y") } a/> │ │││ │ │╰─ openTagEnd:selfClosed "/>" @@ -994,7 +985,6 @@ │ ││├─ attrValue "= (x) => { console.log(\"y\") }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 124╭─ { console.log("y") } a/> │ │││ │ │╰─ openTagEnd:selfClosed "/>" @@ -1003,7 +993,6 @@ │ ││├─ attrValue "= async x => { console.log(\"y\") }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 125╭─ │ │││ │ │╰─ openTagEnd:selfClosed "/>" @@ -1012,7 +1001,6 @@ │ ││├─ attrValue "= function (x) { console.log(\"y\") }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 126╭─ { console.log("y") } a/> │ │││ │ │╰─ openTagEnd:selfClosed "/>" @@ -1021,7 +1009,6 @@ │ ││├─ attrValue "= x => { console.log(\"y\") }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart 127╭─ │ │││ │ │╰─ openTagEnd:selfClosed "/>" @@ -1030,5 +1017,4 @@ │ ││├─ attrValue "= async function (x) { console.log(\"y\") }" │ ││╰─ attrName │ │╰─ tagName - │ ├─ text "\n" ╰─ ╰─ openTagStart \ No newline at end of file diff --git a/src/__tests__/fixtures/attr-operators-space-between/input.marko b/src/__tests__/fixtures/attr-operators-space-between/input.marko index 31fb88e0..4147ebd9 100644 --- a/src/__tests__/fixtures/attr-operators-space-between/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-between/input.marko @@ -111,7 +111,7 @@ a = async function (x) { console.log("y") } a - y + y > y a/> >> y a/> diff --git a/src/states/ATTRIBUTE.ts b/src/states/ATTRIBUTE.ts index f586ffa7..39f5b141 100644 --- a/src/states/ATTRIBUTE.ts +++ b/src/states/ATTRIBUTE.ts @@ -8,8 +8,9 @@ import { Ranges, Meta, ErrorCode, + matchesCloseCurlyBrace, + matchesCloseParen, } from "../internal"; -import { OPERATOR_TERMINATOR } from "./EXPRESSION"; import { TAG_STAGE } from "./OPEN_TAG"; const enum ATTR_STAGE { @@ -29,36 +30,6 @@ export interface AttrMeta extends Meta { bound: boolean; } -const HTML_VALUE_TERMINATORS = [ - CODE.CLOSE_ANGLE_BRACKET, - CODE.COMMA, - [CODE.FORWARD_SLASH, CODE.CLOSE_ANGLE_BRACKET], -]; - -const CONCISE_VALUE_TERMINATORS = [ - CODE.CLOSE_SQUARE_BRACKET, - CODE.SEMICOLON, - CODE.COMMA, -]; - -const HTML_NAME_TERMINATORS = [ - CODE.CLOSE_ANGLE_BRACKET, - CODE.COMMA, - CODE.OPEN_PAREN, - CODE.EQUAL, - [CODE.COLON, CODE.EQUAL], - [CODE.FORWARD_SLASH, CODE.CLOSE_ANGLE_BRACKET], -]; - -const CONCISE_NAME_TERMINATORS = [ - CODE.CLOSE_SQUARE_BRACKET, - CODE.SEMICOLON, - CODE.EQUAL, - CODE.COMMA, - CODE.OPEN_PAREN, - [CODE.COLON, CODE.EQUAL], -]; - // We enter STATE.ATTRIBUTE when we see a non-whitespace // character after reading the tag name export const ATTRIBUTE: StateDefinition = { @@ -111,39 +82,35 @@ export const ATTRIBUTE: StateDefinition = { attr.stage = ATTR_STAGE.VALUE; const expr = this.enterState(STATE.EXPRESSION); expr.terminatedByWhitespace = true; - - if (this.isConcise) { - expr.terminator = CONCISE_VALUE_TERMINATORS; - - if (this.activeTag!.stage !== TAG_STAGE.ATTR_GROUP) { - expr.operatorTerminator = OPERATOR_TERMINATOR.Hyphens; - } - } else { - expr.terminator = HTML_VALUE_TERMINATORS; - expr.operatorTerminator = OPERATOR_TERMINATOR.CloseAngleBracket; - } + expr.shouldTerminate = this.isConcise + ? this.activeTag!.stage === TAG_STAGE.ATTR_GROUP + ? shouldTerminateConciseGroupedAttrValue + : shouldTerminateConciseAttrValue + : shouldTerminateHtmlAttrValue; } else if (code === CODE.OPEN_PAREN) { ensureAttrName(this, attr); attr.stage = ATTR_STAGE.ARGUMENT; this.pos++; // skip ( this.forward = 0; - this.enterState(STATE.EXPRESSION).terminator = CODE.CLOSE_PAREN; + this.enterState(STATE.EXPRESSION).shouldTerminate = matchesCloseParen; } else if (code === CODE.OPEN_CURLY_BRACE && attr.args) { ensureAttrName(this, attr); attr.stage = ATTR_STAGE.BLOCK; this.pos++; // skip { this.forward = 0; const expr = this.enterState(STATE.EXPRESSION); - expr.terminator = CODE.CLOSE_CURLY_BRACE; + expr.shouldTerminate = matchesCloseCurlyBrace; } else if (attr.stage === ATTR_STAGE.UNKNOWN) { attr.stage = ATTR_STAGE.NAME; this.forward = 0; const expr = this.enterState(STATE.EXPRESSION); expr.terminatedByWhitespace = true; expr.skipOperators = true; - expr.terminator = this.isConcise - ? CONCISE_NAME_TERMINATORS - : HTML_NAME_TERMINATORS; + expr.shouldTerminate = this.isConcise + ? this.activeTag!.stage === TAG_STAGE.ATTR_GROUP + ? shouldTerminateConciseGroupedAttrName + : shouldTerminateConciseAttrName + : shouldTerminateHtmlAttrName; } else { this.exitState(); } @@ -282,3 +249,116 @@ function ensureAttrName(parser: Parser, attr: AttrMeta) { }); } } + +function shouldTerminateHtmlAttrName(code: number, data: string, pos: number) { + switch (code) { + case CODE.COMMA: + case CODE.EQUAL: + case CODE.OPEN_PAREN: + case CODE.CLOSE_ANGLE_BRACKET: + return true; + case CODE.COLON: + return data.charCodeAt(pos + 1) === CODE.EQUAL; + case CODE.FORWARD_SLASH: + return data.charCodeAt(pos + 1) === CODE.CLOSE_ANGLE_BRACKET; + default: + return false; + } +} + +function shouldTerminateHtmlAttrValue(code: number, data: string, pos: number) { + switch (code) { + case CODE.COMMA: + return true; + case CODE.FORWARD_SLASH: + return data.charCodeAt(pos + 1) === CODE.CLOSE_ANGLE_BRACKET; + + // Add special cases for >> >= => + case CODE.CLOSE_ANGLE_BRACKET: + switch (data.charCodeAt(pos + 1)) { + case CODE.CLOSE_ANGLE_BRACKET: + case CODE.EQUAL: + return false; + default: + switch (data.charCodeAt(pos - 1)) { + case CODE.CLOSE_ANGLE_BRACKET: + case CODE.EQUAL: + return false; + default: + return true; + } + } + default: + return false; + } +} + +function shouldTerminateConciseAttrName( + code: number, + data: string, + pos: number +) { + switch (code) { + case CODE.COMMA: + case CODE.EQUAL: + case CODE.OPEN_PAREN: + case CODE.SEMICOLON: + return true; + case CODE.COLON: + return data.charCodeAt(pos + 1) === CODE.EQUAL; + case CODE.HYPHEN: + return ( + data.charCodeAt(pos + 1) === CODE.HYPHEN && + isWhitespaceCode(data.charCodeAt(pos - 1)) + ); + default: + return false; + } +} + +function shouldTerminateConciseAttrValue( + code: number, + data: string, + pos: number +) { + switch (code) { + case CODE.COMMA: + case CODE.SEMICOLON: + return true; + case CODE.HYPHEN: + return ( + data.charCodeAt(pos + 1) === CODE.HYPHEN && + isWhitespaceCode(data.charCodeAt(pos - 1)) + ); + default: + return false; + } +} + +function shouldTerminateConciseGroupedAttrName( + code: number, + data: string, + pos: number +) { + switch (code) { + case CODE.COMMA: + case CODE.EQUAL: + case CODE.OPEN_PAREN: + case CODE.CLOSE_SQUARE_BRACKET: + return true; + case CODE.COLON: + return data.charCodeAt(pos + 1) === CODE.EQUAL; + default: + return false; + } +} + +function shouldTerminateConciseGroupedAttrValue(code: number) { + switch (code) { + case CODE.COMMA: + case CODE.CLOSE_SQUARE_BRACKET: + return true; + default: + return false; + } +} diff --git a/src/states/EXPRESSION.ts b/src/states/EXPRESSION.ts index 8ed573c6..d91e6fcf 100644 --- a/src/states/EXPRESSION.ts +++ b/src/states/EXPRESSION.ts @@ -8,26 +8,17 @@ import { ErrorCode, } from "../internal"; -export enum OPERATOR_TERMINATOR { - // All operators are ok. - None = 0, - // Prevents double hyphens (set when in concise mode attrs). - Hyphens = 1, - // Prevent equals continuing an expression (set for tag variable expression) - AttrValue = 1 << 1, - // Prevent close angle bracket continuing an expression (set for html mode attrs) - CloseAngleBracket = 1 << 2, -} - export interface ExpressionMeta extends Meta { groupStack: number[]; - terminator: number | (number | number[])[]; skipOperators: boolean; terminatedByEOL: boolean; terminatedByWhitespace: boolean; - operatorTerminator: OPERATOR_TERMINATOR; + shouldTerminate(code: number, data: string, pos: number): boolean; } +// Never terminate early by default. +const shouldTerminate = () => false; + const unaryKeywords = [ "async", "await", @@ -51,11 +42,10 @@ export const EXPRESSION: StateDefinition = { start, end: start, groupStack: [], - terminator: -1, + shouldTerminate, skipOperators: false, terminatedByEOL: false, terminatedByWhitespace: false, - operatorTerminator: OPERATOR_TERMINATOR.None, }; }, @@ -70,11 +60,7 @@ export const EXPRESSION: StateDefinition = { return; } - if ( - typeof expression.terminator === "number" - ? expression.terminator === code - : checkForTerminators(this, code, expression.terminator) - ) { + if (expression.shouldTerminate(code, this.data, this.pos)) { this.exitState(); return; } @@ -232,52 +218,37 @@ function checkForOperators( if (expression.skipOperators) return false; const { pos, data } = parser; - const lookBehindPos = lookBehindForOperator(data, pos); - if (lookBehindPos !== -1) { + if (lookBehindForOperator(data, pos) !== -1) { parser.consumeWhitespace(); parser.forward = 0; return true; } + const terminatedByEOL = expression.terminatedByEOL || parser.isConcise; if (!(terminatedByEOL && eol)) { - const lookAheadPos = lookAheadForOperator( + const nextNonSpace = lookAheadWhile( + terminatedByEOL ? isIndentCode : isWhitespaceCode, data, - lookAheadWhile( - terminatedByEOL ? isIndentCode : isWhitespaceCode, - data, - pos + 1 - ), - expression.operatorTerminator + pos + 1 ); - if (lookAheadPos !== -1) { - parser.pos = lookAheadPos; - parser.forward = 0; - return true; - } - } - - return false; -} - -function checkForTerminators( - parser: Parser, - code: number, - terminators: (number | number[])[] -) { - outer: for (const terminator of terminators) { - if (typeof terminator === "number") { - if (code === terminator) return true; - } else { - if (terminator[0] === code) { - for (let i = terminator.length; i-- > 1; ) { - if (parser.data.charCodeAt(parser.pos + i) !== terminator[i]) - continue outer; - } + if ( + !expression.shouldTerminate( + data.charCodeAt(nextNonSpace), + data, + nextNonSpace + ) + ) { + const lookAheadPos = lookAheadForOperator(data, nextNonSpace); + if (lookAheadPos !== -1) { + parser.pos = lookAheadPos; + parser.forward = 0; return true; } } } + + return false; } function lookBehindForOperator(data: string, pos: number): number { @@ -300,9 +271,9 @@ function lookBehindForOperator(data: string, pos: number): number { case CODE.TILDE: return curPos; + // special case -- and ++ case CODE.PLUS: case CODE.HYPHEN: { - // special case -- and ++ if (data.charCodeAt(curPos - 1) === code) { // Check if we should continue for another reason. // eg "typeof++ x" @@ -319,10 +290,9 @@ function lookBehindForOperator(data: string, pos: number): number { for (const keyword of unaryKeywords) { const keywordPos = lookBehindFor(data, curPos, keyword); if (keywordPos !== -1) { - return keywordPos === 0 || - data.charCodeAt(keywordPos - 1) !== CODE.PERIOD - ? keywordPos - : -1; + return data.charCodeAt(keywordPos - 1) === CODE.PERIOD + ? -1 + : keywordPos; } } return -1; @@ -330,11 +300,7 @@ function lookBehindForOperator(data: string, pos: number): number { } } -function lookAheadForOperator( - data: string, - pos: number, - terminator: OPERATOR_TERMINATOR -): number { +function lookAheadForOperator(data: string, pos: number): number { switch (data.charCodeAt(pos)) { case CODE.AMPERSAND: case CODE.ASTERISK: @@ -346,83 +312,21 @@ function lookAheadForOperator( case CODE.QUESTION: case CODE.TILDE: case CODE.PLUS: + case CODE.HYPHEN: + case CODE.COLON: + case CODE.CLOSE_ANGLE_BRACKET: + case CODE.EQUAL: return pos + 1; - case CODE.HYPHEN: { - return (terminator & OPERATOR_TERMINATOR.Hyphens) === - OPERATOR_TERMINATOR.Hyphens && data.charCodeAt(pos + 1) === CODE.HYPHEN - ? -1 - : pos + 1; - } - + case CODE.FORWARD_SLASH: case CODE.OPEN_CURLY_BRACE: case CODE.OPEN_PAREN: return pos; // defers to base expression state to track block groups. - case CODE.FORWARD_SLASH: - return (terminator & OPERATOR_TERMINATOR.CloseAngleBracket) === - OPERATOR_TERMINATOR.CloseAngleBracket && - data.charCodeAt(pos + 1) === CODE.CLOSE_ANGLE_BRACKET - ? -1 - : pos; // defers to base expression state to track regexp groups. - - case CODE.COLON: - return (terminator & OPERATOR_TERMINATOR.AttrValue) === - OPERATOR_TERMINATOR.AttrValue && data.charCodeAt(pos + 1) === CODE.EQUAL - ? -1 - : pos + 1; - case CODE.PERIOD: // Only match a dot if its not ... return data.charCodeAt(pos + 1) === CODE.PERIOD ? -1 : pos + 1; - case CODE.CLOSE_ANGLE_BRACKET: - // In concise mode a > is a normal operator. - if ( - (terminator & OPERATOR_TERMINATOR.CloseAngleBracket) !== - OPERATOR_TERMINATOR.CloseAngleBracket - ) { - return pos + 1; - } - - // Otherwise we match >=, >> and >>> - switch (data.charCodeAt(pos + 1)) { - case CODE.EQUAL: - return pos + 2; - case CODE.CLOSE_ANGLE_BRACKET: - return ( - pos + - (data.charCodeAt(pos + 2) === CODE.CLOSE_ANGLE_BRACKET ? 3 : 2) - ); - default: - return -1; - } - case CODE.EQUAL: - if ( - (terminator & OPERATOR_TERMINATOR.AttrValue) === - OPERATOR_TERMINATOR.AttrValue - ) { - return -1; - } - - if ( - (terminator & OPERATOR_TERMINATOR.CloseAngleBracket) !== - OPERATOR_TERMINATOR.CloseAngleBracket - ) { - return pos + 1; - } - - // We'll match ==, ===, and => - // This is primarily to support => since otherwise it'd end a CloseAngleBracket terminator. - switch (data.charCodeAt(pos + 1)) { - case CODE.CLOSE_ANGLE_BRACKET: - return pos + 2; - case CODE.EQUAL: - return pos + (data.charCodeAt(pos + 2) === CODE.EQUAL ? 3 : 2); - default: - return -1; - } - default: { for (const keyword of binaryKeywords) { let nextPos = lookAheadFor(data, pos, keyword); diff --git a/src/states/INLINE_SCRIPT.ts b/src/states/INLINE_SCRIPT.ts index 2b8c462d..f8e21193 100644 --- a/src/states/INLINE_SCRIPT.ts +++ b/src/states/INLINE_SCRIPT.ts @@ -1,4 +1,11 @@ -import { CODE, Range, STATE, StateDefinition, Meta } from "../internal"; +import { + CODE, + Range, + STATE, + StateDefinition, + Meta, + matchesCloseCurlyBrace, +} from "../internal"; interface ScriptletMeta extends Meta { block: boolean; @@ -44,7 +51,7 @@ export const INLINE_SCRIPT: StateDefinition = { inlineScript.block = true; this.pos++; // skip { const expr = this.enterState(STATE.EXPRESSION); - expr.terminator = CODE.CLOSE_CURLY_BRACE; + expr.shouldTerminate = matchesCloseCurlyBrace; expr.skipOperators = true; } else { const expr = this.enterState(STATE.EXPRESSION); diff --git a/src/states/OPEN_TAG.ts b/src/states/OPEN_TAG.ts index b4e757c1..c2cf89c3 100644 --- a/src/states/OPEN_TAG.ts +++ b/src/states/OPEN_TAG.ts @@ -7,8 +7,9 @@ import { Meta, TagType, ErrorCode, + matchesPipe, + matchesCloseParen, } from "../internal"; -import { OPERATOR_TERMINATOR } from "./EXPRESSION"; export enum TAG_STAGE { UNKNOWN, @@ -33,24 +34,6 @@ export interface OpenTagMeta extends Meta { nestedIndent: string | undefined; parentTag: OpenTagMeta | undefined; } -const CONCISE_TAG_VAR_TERMINATORS = [ - CODE.SEMICOLON, - CODE.OPEN_PAREN, - CODE.PIPE, - CODE.EQUAL, - CODE.COMMA, - [CODE.COLON, CODE.EQUAL], -]; - -const HTML_TAG_VAR_TERMINATORS = [ - CODE.CLOSE_ANGLE_BRACKET, - CODE.OPEN_PAREN, - CODE.PIPE, - CODE.EQUAL, - CODE.COMMA, - [CODE.COLON, CODE.EQUAL], - [CODE.FORWARD_SLASH, CODE.CLOSE_ANGLE_BRACKET], -]; export const OPEN_TAG: StateDefinition = { name: "OPEN_TAG", @@ -311,16 +294,9 @@ export const OPEN_TAG: StateDefinition = { const expr = this.enterState(STATE.EXPRESSION); expr.terminatedByWhitespace = true; - - if (this.isConcise) { - expr.terminator = CONCISE_TAG_VAR_TERMINATORS; - expr.operatorTerminator = - OPERATOR_TERMINATOR.AttrValue | OPERATOR_TERMINATOR.Hyphens; - } else { - expr.terminator = HTML_TAG_VAR_TERMINATORS; - expr.operatorTerminator = - OPERATOR_TERMINATOR.AttrValue | OPERATOR_TERMINATOR.CloseAngleBracket; - } + expr.shouldTerminate = this.isConcise + ? shouldTerminateConciseTagVar + : shouldTerminateHtmlTagVar; } else if (code === CODE.OPEN_PAREN && !tag.hasAttrs) { if (tag.hasArgs) { this.emitError( @@ -335,14 +311,14 @@ export const OPEN_TAG: StateDefinition = { this.forward = 0; const expr = this.enterState(STATE.EXPRESSION); expr.skipOperators = true; - expr.terminator = CODE.CLOSE_PAREN; + expr.shouldTerminate = matchesCloseParen; } else if (code === CODE.PIPE && !tag.hasAttrs) { tag.stage = TAG_STAGE.PARAMS; this.pos++; // skip | this.forward = 0; const expr = this.enterState(STATE.EXPRESSION); expr.skipOperators = true; - expr.terminator = CODE.PIPE; + expr.shouldTerminate = matchesPipe; } else { this.forward = 0; @@ -424,3 +400,35 @@ export const OPEN_TAG: StateDefinition = { } }, }; + +function shouldTerminateConciseTagVar(code: number, data: string, pos: number) { + switch (code) { + case CODE.COMMA: + case CODE.EQUAL: + case CODE.PIPE: + case CODE.OPEN_PAREN: + case CODE.SEMICOLON: + return true; + case CODE.COLON: + return data.charCodeAt(pos + 1) === CODE.EQUAL; + default: + return false; + } +} + +function shouldTerminateHtmlTagVar(code: number, data: string, pos: number) { + switch (code) { + case CODE.PIPE: + case CODE.COMMA: + case CODE.EQUAL: + case CODE.OPEN_PAREN: + case CODE.CLOSE_ANGLE_BRACKET: + return true; + case CODE.COLON: + return data.charCodeAt(pos + 1) === CODE.EQUAL; + case CODE.FORWARD_SLASH: + return data.charCodeAt(pos + 1) === CODE.CLOSE_ANGLE_BRACKET; + default: + return false; + } +} diff --git a/src/states/PLACEHOLDER.ts b/src/states/PLACEHOLDER.ts index c3a887db..d650460c 100644 --- a/src/states/PLACEHOLDER.ts +++ b/src/states/PLACEHOLDER.ts @@ -94,10 +94,14 @@ export function checkForPlaceholder(parser: Parser, code: number) { parser.enterState(PLACEHOLDER).escape = escape; parser.pos += escape ? 2 : 3; // skip ${ or $!{ parser.forward = 0; - parser.enterState(STATE.EXPRESSION).terminator = CODE.CLOSE_CURLY_BRACE; + parser.enterState(STATE.EXPRESSION).shouldTerminate = shouldTerminate; return true; } } return false; } + +function shouldTerminate(code: number) { + return code === CODE.CLOSE_CURLY_BRACE; +} diff --git a/src/states/TAG_NAME.ts b/src/states/TAG_NAME.ts index e45ff421..197dc608 100644 --- a/src/states/TAG_NAME.ts +++ b/src/states/TAG_NAME.ts @@ -110,7 +110,7 @@ export const TAG_NAME: StateDefinition = { ) { this.pos += 2; // skip ${ this.forward = 0; - this.enterState(STATE.EXPRESSION).terminator = CODE.CLOSE_CURLY_BRACE; + this.enterState(STATE.EXPRESSION).shouldTerminate = shouldTerminate; } else if ( isWhitespaceCode(code) || code === CODE.EQUAL || @@ -168,3 +168,7 @@ export const TAG_NAME: StateDefinition = { quasis.push({ start: nextStart, end: nextStart }); }, }; + +function shouldTerminate(code: number) { + return code === CODE.CLOSE_CURLY_BRACE; +} diff --git a/src/states/TEMPLATE_STRING.ts b/src/states/TEMPLATE_STRING.ts index 95893a3a..e63467e6 100644 --- a/src/states/TEMPLATE_STRING.ts +++ b/src/states/TEMPLATE_STRING.ts @@ -22,7 +22,7 @@ export const TEMPLATE_STRING: StateDefinition = { this.pos++; // skip { const expr = this.enterState(STATE.EXPRESSION); expr.skipOperators = true; - expr.terminator = CODE.CLOSE_CURLY_BRACE; + expr.shouldTerminate = shouldTerminate; } else { if (code === CODE.BACK_SLASH) { this.pos++; // skip \ @@ -55,3 +55,7 @@ export const TEMPLATE_STRING: StateDefinition = { this.pos++; // skip closing } }, }; + +function shouldTerminate(code: number) { + return code === CODE.CLOSE_CURLY_BRACE; +} diff --git a/src/util/util.ts b/src/util/util.ts index 65a96bcf..090d568f 100644 --- a/src/util/util.ts +++ b/src/util/util.ts @@ -68,6 +68,18 @@ export function htmlEOF(this: Parser) { } } +export function matchesCloseParen(code: number) { + return code === CODE.CLOSE_PAREN; +} + +export function matchesCloseCurlyBrace(code: number) { + return code === CODE.CLOSE_CURLY_BRACE; +} + +export function matchesPipe(code: number) { + return code === CODE.PIPE; +} + function getPosAfterLine( lines: number[], startLine: number, From 9bc1010b726c020a576a262d5d93e9a46c06eddc Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Mon, 8 Aug 2022 09:09:30 -0700 Subject: [PATCH 5/7] refactor: make the expression operators config opt-in instead of opt-out --- src/states/ATTRIBUTE.ts | 6 +++--- src/states/EXPRESSION.ts | 6 +++--- src/states/INLINE_SCRIPT.ts | 6 +++--- src/states/OPEN_TAG.ts | 9 +++------ src/states/PLACEHOLDER.ts | 8 +++----- src/states/TAG_NAME.ts | 12 ++++++------ src/states/TEMPLATE_STRING.ts | 17 +++++++++-------- 7 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/states/ATTRIBUTE.ts b/src/states/ATTRIBUTE.ts index 39f5b141..b6e6adfa 100644 --- a/src/states/ATTRIBUTE.ts +++ b/src/states/ATTRIBUTE.ts @@ -81,6 +81,7 @@ export const ATTRIBUTE: StateDefinition = { attr.stage = ATTR_STAGE.VALUE; const expr = this.enterState(STATE.EXPRESSION); + expr.operators = true; expr.terminatedByWhitespace = true; expr.shouldTerminate = this.isConcise ? this.activeTag!.stage === TAG_STAGE.ATTR_GROUP @@ -98,14 +99,13 @@ export const ATTRIBUTE: StateDefinition = { attr.stage = ATTR_STAGE.BLOCK; this.pos++; // skip { this.forward = 0; - const expr = this.enterState(STATE.EXPRESSION); - expr.shouldTerminate = matchesCloseCurlyBrace; + this.enterState(STATE.EXPRESSION).shouldTerminate = + matchesCloseCurlyBrace; } else if (attr.stage === ATTR_STAGE.UNKNOWN) { attr.stage = ATTR_STAGE.NAME; this.forward = 0; const expr = this.enterState(STATE.EXPRESSION); expr.terminatedByWhitespace = true; - expr.skipOperators = true; expr.shouldTerminate = this.isConcise ? this.activeTag!.stage === TAG_STAGE.ATTR_GROUP ? shouldTerminateConciseGroupedAttrName diff --git a/src/states/EXPRESSION.ts b/src/states/EXPRESSION.ts index d91e6fcf..3c8a8dce 100644 --- a/src/states/EXPRESSION.ts +++ b/src/states/EXPRESSION.ts @@ -10,7 +10,7 @@ import { export interface ExpressionMeta extends Meta { groupStack: number[]; - skipOperators: boolean; + operators: boolean; terminatedByEOL: boolean; terminatedByWhitespace: boolean; shouldTerminate(code: number, data: string, pos: number): boolean; @@ -43,7 +43,7 @@ export const EXPRESSION: StateDefinition = { end: start, groupStack: [], shouldTerminate, - skipOperators: false, + operators: false, terminatedByEOL: false, terminatedByWhitespace: false, }; @@ -215,7 +215,7 @@ function checkForOperators( expression: ExpressionMeta, eol: boolean ) { - if (expression.skipOperators) return false; + if (!expression.operators) return false; const { pos, data } = parser; if (lookBehindForOperator(data, pos) !== -1) { diff --git a/src/states/INLINE_SCRIPT.ts b/src/states/INLINE_SCRIPT.ts index f8e21193..83e2d3cb 100644 --- a/src/states/INLINE_SCRIPT.ts +++ b/src/states/INLINE_SCRIPT.ts @@ -50,11 +50,11 @@ export const INLINE_SCRIPT: StateDefinition = { if (code === CODE.OPEN_CURLY_BRACE) { inlineScript.block = true; this.pos++; // skip { - const expr = this.enterState(STATE.EXPRESSION); - expr.shouldTerminate = matchesCloseCurlyBrace; - expr.skipOperators = true; + this.enterState(STATE.EXPRESSION).shouldTerminate = + matchesCloseCurlyBrace; } else { const expr = this.enterState(STATE.EXPRESSION); + expr.operators = true; expr.terminatedByEOL = true; } }, diff --git a/src/states/OPEN_TAG.ts b/src/states/OPEN_TAG.ts index c2cf89c3..ca844836 100644 --- a/src/states/OPEN_TAG.ts +++ b/src/states/OPEN_TAG.ts @@ -293,6 +293,7 @@ export const OPEN_TAG: StateDefinition = { } const expr = this.enterState(STATE.EXPRESSION); + expr.operators = true; expr.terminatedByWhitespace = true; expr.shouldTerminate = this.isConcise ? shouldTerminateConciseTagVar @@ -309,16 +310,12 @@ export const OPEN_TAG: StateDefinition = { tag.stage = TAG_STAGE.ARGUMENT; this.pos++; // skip ( this.forward = 0; - const expr = this.enterState(STATE.EXPRESSION); - expr.skipOperators = true; - expr.shouldTerminate = matchesCloseParen; + this.enterState(STATE.EXPRESSION).shouldTerminate = matchesCloseParen; } else if (code === CODE.PIPE && !tag.hasAttrs) { tag.stage = TAG_STAGE.PARAMS; this.pos++; // skip | this.forward = 0; - const expr = this.enterState(STATE.EXPRESSION); - expr.skipOperators = true; - expr.shouldTerminate = matchesPipe; + this.enterState(STATE.EXPRESSION).shouldTerminate = matchesPipe; } else { this.forward = 0; diff --git a/src/states/PLACEHOLDER.ts b/src/states/PLACEHOLDER.ts index d650460c..3e0929ae 100644 --- a/src/states/PLACEHOLDER.ts +++ b/src/states/PLACEHOLDER.ts @@ -5,6 +5,7 @@ import { StateDefinition, Meta, ErrorCode, + matchesCloseCurlyBrace, } from "../internal"; interface PlaceholderMeta extends Meta { @@ -94,14 +95,11 @@ export function checkForPlaceholder(parser: Parser, code: number) { parser.enterState(PLACEHOLDER).escape = escape; parser.pos += escape ? 2 : 3; // skip ${ or $!{ parser.forward = 0; - parser.enterState(STATE.EXPRESSION).shouldTerminate = shouldTerminate; + parser.enterState(STATE.EXPRESSION).shouldTerminate = + matchesCloseCurlyBrace; return true; } } return false; } - -function shouldTerminate(code: number) { - return code === CODE.CLOSE_CURLY_BRACE; -} diff --git a/src/states/TAG_NAME.ts b/src/states/TAG_NAME.ts index 197dc608..183e1746 100644 --- a/src/states/TAG_NAME.ts +++ b/src/states/TAG_NAME.ts @@ -7,6 +7,7 @@ import { Meta, TagType, ErrorCode, + matchesCloseCurlyBrace, } from "../internal"; export interface TagNameMeta extends Meta, Ranges.Template { @@ -94,7 +95,9 @@ export const TAG_NAME: StateDefinition = { ); } - this.enterState(STATE.EXPRESSION).terminatedByEOL = true; + const expr = this.enterState(STATE.EXPRESSION); + expr.operators = true; + expr.terminatedByEOL = true; } } @@ -110,7 +113,8 @@ export const TAG_NAME: StateDefinition = { ) { this.pos += 2; // skip ${ this.forward = 0; - this.enterState(STATE.EXPRESSION).shouldTerminate = shouldTerminate; + this.enterState(STATE.EXPRESSION).shouldTerminate = + matchesCloseCurlyBrace; } else if ( isWhitespaceCode(code) || code === CODE.EQUAL || @@ -168,7 +172,3 @@ export const TAG_NAME: StateDefinition = { quasis.push({ start: nextStart, end: nextStart }); }, }; - -function shouldTerminate(code: number) { - return code === CODE.CLOSE_CURLY_BRACE; -} diff --git a/src/states/TEMPLATE_STRING.ts b/src/states/TEMPLATE_STRING.ts index e63467e6..d501932e 100644 --- a/src/states/TEMPLATE_STRING.ts +++ b/src/states/TEMPLATE_STRING.ts @@ -1,4 +1,10 @@ -import { CODE, ErrorCode, STATE, StateDefinition } from "../internal"; +import { + CODE, + ErrorCode, + STATE, + StateDefinition, + matchesCloseCurlyBrace, +} from "../internal"; export const TEMPLATE_STRING: StateDefinition = { name: "TEMPLATE_STRING", @@ -20,9 +26,8 @@ export const TEMPLATE_STRING: StateDefinition = { this.lookAtCharCodeAhead(1) === CODE.OPEN_CURLY_BRACE ) { this.pos++; // skip { - const expr = this.enterState(STATE.EXPRESSION); - expr.skipOperators = true; - expr.shouldTerminate = shouldTerminate; + this.enterState(STATE.EXPRESSION).shouldTerminate = + matchesCloseCurlyBrace; } else { if (code === CODE.BACK_SLASH) { this.pos++; // skip \ @@ -55,7 +60,3 @@ export const TEMPLATE_STRING: StateDefinition = { this.pos++; // skip closing } }, }; - -function shouldTerminate(code: number) { - return code === CODE.CLOSE_CURLY_BRACE; -} From 65a29358f729dfa3b05fa5818fff62586a47f344 Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Mon, 8 Aug 2022 14:32:41 -0700 Subject: [PATCH 6/7] fix: remove special cases for binary operators not supported in older versions of the parser --- .../attr-operators-newline-after.expected.txt | 65 ++++++++++++------- .../attr-operators-newline-after/input.marko | 10 +-- ...attr-operators-newline-before.expected.txt | 65 ++++++++++++------- .../attr-operators-newline-before/input.marko | 10 +-- .../attr-operators-space-after.expected.txt | 65 ++++++++++++------- .../attr-operators-space-after/input.marko | 10 +-- .../attr-operators-space-before.expected.txt | 65 ++++++++++++------- .../attr-operators-space-before/input.marko | 10 +-- .../attr-operators-space-between.expected.txt | 65 ++++++++++++------- .../attr-operators-space-between/input.marko | 10 +-- src/states/ATTRIBUTE.ts | 17 +---- 11 files changed, 227 insertions(+), 165 deletions(-) diff --git a/src/__tests__/fixtures/attr-operators-newline-after/__snapshots__/attr-operators-newline-after.expected.txt b/src/__tests__/fixtures/attr-operators-newline-after/__snapshots__/attr-operators-newline-after.expected.txt index 3b7daf30..7558659a 100644 --- a/src/__tests__/fixtures/attr-operators-newline-after/__snapshots__/attr-operators-newline-after.expected.txt +++ b/src/__tests__/fixtures/attr-operators-newline-after/__snapshots__/attr-operators-newline-after.expected.txt @@ -683,14 +683,17 @@ │ │╰─ openTagEnd:selfClosed "/>" ╰─ ╰─ attrName 159╭─ = - │ │││╰─ attrValue.value "x >=\ny" - │ ││├─ attrValue "=x >=\ny" + │ ││││ │╰─ text "=\ny " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -160╭─ y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +160╭─ y + │ │ │╰─ closeTagEnd(a) + │ │ ╰─ closeTagName + ╰─ ╰─ closeTagStart "" ╰─ ╰─ attrName 173╭─ >= - │ │││╰─ attrValue.value "x >>=\ny" - │ ││├─ attrValue "=x >>=\ny" + │ ││││ │╰─ text ">=\ny " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -174╭─ y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +174╭─ y + │ │ │╰─ closeTagEnd(a) + │ │ ╰─ closeTagName + ╰─ ╰─ closeTagStart ">>= - │ │││╰─ attrValue.value "x >>>=\ny" - │ ││├─ attrValue "=x >>>=\ny" + │ ││││ │╰─ text ">>=\ny " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -176╭─ y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +176╭─ y + │ │ │╰─ closeTagEnd(a) + │ │ ╰─ closeTagName + ╰─ ╰─ closeTagStart "" ╰─ ╰─ attrName 193╭─ > - │ │││╰─ attrValue.value "x >>\ny" - │ ││├─ attrValue "=x >>\ny" + │ ││││ │╰─ text ">\ny " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -194╭─ y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +194╭─ y + │ │ │╰─ closeTagEnd(a) + │ │ ╰─ closeTagName + ╰─ ╰─ closeTagStart ">> - │ │││╰─ attrValue.value "x >>>\ny" - │ ││├─ attrValue "=x >>>\ny" + │ ││││ │╰─ text ">>\ny " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -196╭─ y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +196╭─ y + │ │ │╰─ closeTagEnd(a) + │ │ ╰─ closeTagName + ╰─ ╰─ closeTagStart " = -y a/> +y >= -y a/> +y >>= -y a/> +y > -y a/> +y >> -y a/> +y " ╰─ ╰─ attrName 45╭─ = y" - │ ││├─ attrValue "=x\n>= y" + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -46╭─ >= y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +46╭─ >= y + │ ││ │ │╰─ closeTagEnd(a) + │ ││ │ ╰─ closeTagName + │ ││ ╰─ closeTagStart "" ╰─ ╰─ attrName 59╭─ >= y" - │ ││├─ attrValue "=x\n>>= y" + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -60╭─ >>= y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +60╭─ >>= y + │ ││ │ │╰─ closeTagEnd(a) + │ ││ │ ╰─ closeTagName + │ ││ ╰─ closeTagStart "= y " + ╰─ ╰─ openTagEnd 61╭─ >>= y" - │ ││├─ attrValue "=x\n>>>= y" + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -62╭─ >>>= y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +62╭─ >>>= y + │ ││ │ │╰─ closeTagEnd(a) + │ ││ │ ╰─ closeTagName + │ ││ ╰─ closeTagStart ">= y " + ╰─ ╰─ openTagEnd 63╭─ " ╰─ ╰─ attrName 79╭─ >y" - │ ││├─ attrValue "=x\n>>y" + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -80╭─ >>y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +80╭─ >>y + │ ││ │ │╰─ closeTagEnd(a) + │ ││ │ ╰─ closeTagName + │ ││ ╰─ closeTagStart "y " + ╰─ ╰─ openTagEnd 81╭─ >> y" - │ ││├─ attrValue "=x\n>>> y" + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -82╭─ >>> y a/> - │ │╰─ openTagEnd:selfClosed "/>" - ╰─ ╰─ attrName +82╭─ >>> y + │ ││ │ │╰─ closeTagEnd(a) + │ ││ │ ╰─ closeTagName + │ ││ ╰─ closeTagStart "> y " + ╰─ ╰─ openTagEnd 83╭─ = y a/> +>= y >= y a/> +>>= y >>= y a/> +>>>= y >y a/> +>>y >> y a/> +>>> y = y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x>= y" - │ ││├─ attrValue "=x>= y" +66╭─ = y + │ ││││││ │ │╰─ closeTagEnd(a) + │ ││││││ │ ╰─ closeTagName + │ ││││││ ╰─ closeTagStart ">= y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x>>= y" - │ ││├─ attrValue "=x>>= y" +73╭─ >= y + │ ││││││ │ │╰─ closeTagEnd(a) + │ ││││││ │ ╰─ closeTagName + │ ││││││ ╰─ closeTagStart "= y " + │ ││││╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -74╭─ >>= y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x>>>= y" - │ ││├─ attrValue "=x>>>= y" +74╭─ >>= y + │ ││││││ │ │╰─ closeTagEnd(a) + │ ││││││ │ ╰─ closeTagName + │ ││││││ ╰─ closeTagStart ">= y " + │ ││││╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart @@ -658,19 +667,25 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -84╭─ > y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x>> y" - │ ││├─ attrValue "=x>> y" +84╭─ > y + │ ││││││ │ │╰─ closeTagEnd(a) + │ ││││││ │ ╰─ closeTagName + │ ││││││ ╰─ closeTagStart " y " + │ ││││╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -85╭─ >> y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x>>> y" - │ ││├─ attrValue "=x>>> y" +85╭─ >> y + │ ││││││ │ │╰─ closeTagEnd(a) + │ ││││││ │ ╰─ closeTagName + │ ││││││ ╰─ closeTagStart "> y " + │ ││││╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart diff --git a/src/__tests__/fixtures/attr-operators-space-after/input.marko b/src/__tests__/fixtures/attr-operators-space-after/input.marko index 8814d976..4accea87 100644 --- a/src/__tests__/fixtures/attr-operators-space-after/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-after/input.marko @@ -63,15 +63,15 @@ a=(x ) {y } a -= y a/> += y ->= y a/> ->>= y a/> +>= y +>>= y @@ -81,8 +81,8 @@ a=(x ) {y } a y -> y a/> ->> y a/> +> y +>> y y a/> diff --git a/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt b/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt index b6909124..72b9be8c 100644 --- a/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt +++ b/src/__tests__/fixtures/attr-operators-space-before/__snapshots__/attr-operators-space-before.expected.txt @@ -511,11 +511,14 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -66╭─ =y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >=y" - │ ││├─ attrValue "=x >=y" +66╭─ =y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart ">=y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>=y" - │ ││├─ attrValue "=x >>=y" +73╭─ >=y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart "=y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -74╭─ >>=y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>>=y" - │ ││├─ attrValue "=x >>>=y" +74╭─ >>=y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart ">=y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart @@ -658,19 +667,25 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -84╭─ >y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>y" - │ ││├─ attrValue "=x >>y" +84╭─ >y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart "y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -85╭─ >>y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>>y" - │ ││├─ attrValue "=x >>>y" +85╭─ >>y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart ">y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart diff --git a/src/__tests__/fixtures/attr-operators-space-before/input.marko b/src/__tests__/fixtures/attr-operators-space-before/input.marko index a482cb49..7c1d7a8d 100644 --- a/src/__tests__/fixtures/attr-operators-space-before/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-before/input.marko @@ -63,15 +63,15 @@ a=( x ) {y } a -=y a/> +=y ->=y a/> ->>=y a/> +>=y +>>=y @@ -81,8 +81,8 @@ a=( x ) {y } a y ->y a/> ->>y a/> +>y +>>y y a/> diff --git a/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt b/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt index ceb4c149..3c041919 100644 --- a/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt +++ b/src/__tests__/fixtures/attr-operators-space-between/__snapshots__/attr-operators-space-between.expected.txt @@ -699,11 +699,14 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -89╭─ = y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >= y" - │ ││├─ attrValue "=x >= y" +89╭─ = y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart ">= y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>= y" - │ ││├─ attrValue "=x >>= y" +96╭─ >= y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart "= y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -97╭─ >>= y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>>= y" - │ ││├─ attrValue "=x >>>= y" +97╭─ >>= y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart ">= y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart @@ -914,19 +923,25 @@ │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -115╭─ > y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >> y" - │ ││├─ attrValue "=x >> y" +115╭─ > y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart " y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart -116╭─ >> y a/> - │ ││││ │╰─ openTagEnd:selfClosed "/>" - │ ││││ ╰─ attrName - │ │││╰─ attrValue.value "x >>> y" - │ ││├─ attrValue "=x >>> y" +116╭─ >> y + │ ││││ ││ │ │╰─ closeTagEnd(a) + │ ││││ ││ │ ╰─ closeTagName + │ ││││ ││ ╰─ closeTagStart "> y " + │ ││││ ╰─ openTagEnd + │ │││╰─ attrValue.value + │ ││├─ attrValue "=x" │ ││╰─ attrName │ │╰─ tagName ╰─ ╰─ openTagStart diff --git a/src/__tests__/fixtures/attr-operators-space-between/input.marko b/src/__tests__/fixtures/attr-operators-space-between/input.marko index 4147ebd9..af2fb653 100644 --- a/src/__tests__/fixtures/attr-operators-space-between/input.marko +++ b/src/__tests__/fixtures/attr-operators-space-between/input.marko @@ -86,15 +86,15 @@ a = async function (x) { console.log("y") } a -= y a/> += y ->= y a/> ->>= y a/> +>= y +>>= y @@ -112,8 +112,8 @@ a = async function (x) { console.log("y") } a y -> y a/> ->> y a/> +> y +>> y y a/> diff --git a/src/states/ATTRIBUTE.ts b/src/states/ATTRIBUTE.ts index b6e6adfa..a5adfab6 100644 --- a/src/states/ATTRIBUTE.ts +++ b/src/states/ATTRIBUTE.ts @@ -272,22 +272,9 @@ function shouldTerminateHtmlAttrValue(code: number, data: string, pos: number) { return true; case CODE.FORWARD_SLASH: return data.charCodeAt(pos + 1) === CODE.CLOSE_ANGLE_BRACKET; - - // Add special cases for >> >= => + // Add special case for => case CODE.CLOSE_ANGLE_BRACKET: - switch (data.charCodeAt(pos + 1)) { - case CODE.CLOSE_ANGLE_BRACKET: - case CODE.EQUAL: - return false; - default: - switch (data.charCodeAt(pos - 1)) { - case CODE.CLOSE_ANGLE_BRACKET: - case CODE.EQUAL: - return false; - default: - return true; - } - } + return data.charCodeAt(pos - 1) !== CODE.EQUAL; default: return false; } From f923c5e9e1af8cda5c2e55df25ba0e370465faf1 Mon Sep 17 00:00:00 2001 From: Dylan Piercey Date: Mon, 8 Aug 2022 14:37:54 -0700 Subject: [PATCH 7/7] refactor: cleanup template literal state --- src/states/TEMPLATE_STRING.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/states/TEMPLATE_STRING.ts b/src/states/TEMPLATE_STRING.ts index d501932e..7e4bf7f3 100644 --- a/src/states/TEMPLATE_STRING.ts +++ b/src/states/TEMPLATE_STRING.ts @@ -21,20 +21,21 @@ export const TEMPLATE_STRING: StateDefinition = { exit() {}, char(code) { - if ( - code === CODE.DOLLAR && - this.lookAtCharCodeAhead(1) === CODE.OPEN_CURLY_BRACE - ) { - this.pos++; // skip { - this.enterState(STATE.EXPRESSION).shouldTerminate = - matchesCloseCurlyBrace; - } else { - if (code === CODE.BACK_SLASH) { + switch (code) { + case CODE.DOLLAR: + if (this.lookAtCharCodeAhead(1) === CODE.OPEN_CURLY_BRACE) { + this.pos++; // skip { + this.enterState(STATE.EXPRESSION).shouldTerminate = + matchesCloseCurlyBrace; + } + break; + case CODE.BACK_SLASH: this.pos++; // skip \ - } else if (code === CODE.BACKTICK) { + break; + case CODE.BACKTICK: this.pos++; // skip ` this.exitState(); - } + break; } },