refactor: Remove Tokenizer statics

parse5 · Nov 22, 2021 · ff5fac8 · ff5fac8
1 parent 6f57fec
commit ff5fac8
Show file tree

Hide file tree

Showing 15 changed files with 302 additions and 331 deletions.
diff --git a/packages/parse5/lib/common/foreign-content.ts b/packages/parse5/lib/common/foreign-content.ts
@@ -1,4 +1,3 @@
-import { Tokenizer } from '../tokenizer/index.js';
 import { TAG_NAMES as $, NAMESPACES as NS, ATTRS } from './html.js';
 import type { TagToken, Attribute } from './token.js';
 
@@ -180,9 +179,7 @@ export function causesExit(startTagToken: TagToken) {
     const tn = startTagToken.tagName;
     const isFontWithAttrs =
         tn === $.FONT &&
-        (Tokenizer.getTokenAttr(startTagToken, ATTRS.COLOR) !== null ||
-            Tokenizer.getTokenAttr(startTagToken, ATTRS.SIZE) !== null ||
-            Tokenizer.getTokenAttr(startTagToken, ATTRS.FACE) !== null);
+        startTagToken.attrs.some(({ name }) => name === ATTRS.COLOR || name === ATTRS.SIZE || name === ATTRS.FACE);
 
     return isFontWithAttrs || EXITS_FOREIGN_CONTENT.has(tn);
 }

diff --git a/packages/parse5/lib/common/token.ts b/packages/parse5/lib/common/token.ts
@@ -62,6 +62,16 @@ export interface TagToken extends TokenBase {
     location?: LocationWithAttributes;
 }
 
+export function getTokenAttr(token: TagToken, attrName: string) {
+    for (let i = token.attrs.length - 1; i >= 0; i--) {
+        if (token.attrs[i].name === attrName) {
+            return token.attrs[i].value;
+        }
+    }
+
+    return null;
+}
+
 export interface CommentToken extends TokenBase {
     readonly type: TokenType.COMMENT;
     data: string;

diff --git a/packages/parse5/lib/extensions/location-info/parser-mixin.ts b/packages/parse5/lib/extensions/location-info/parser-mixin.ts
@@ -1,12 +1,11 @@
 import { CommentToken, DoctypeToken, CharacterToken } from '../../common/token';
 import { Mixin } from '../../utils/mixin.js';
-import { Tokenizer } from '../../tokenizer/index.js';
 import { LocationInfoTokenizerMixin } from './tokenizer-mixin.js';
 import { TAG_NAMES as $, NAMESPACES as NS } from '../../common/html.js';
 import type { TreeAdapter, TreeAdapterTypeMap, ElementLocation } from '../../tree-adapters/interface';
 import type { Parser } from '../../parser/index.js';
 import type { PositionTrackingPreprocessorMixin } from '../position-tracking/preprocessor-mixin';
-import type { Token, TagToken } from '../../common/token.js';
+import { TokenType, Token, TagToken } from '../../common/token.js';
 
 export class LocationInfoParserMixin<T extends TreeAdapterTypeMap> extends Mixin<Parser<T>> {
     treeAdapter: TreeAdapter<T>;
@@ -44,7 +43,7 @@ export class LocationInfoParserMixin<T extends TreeAdapterTypeMap> extends Mixin
 
                 // NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing
                 // tag and for cases like <td> <p> </td> - 'p' closes without a closing tag.
-                const isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN && tn === closingToken.tagName;
+                const isClosingEndTag = closingToken.type === TokenType.END_TAG && tn === closingToken.tagName;
                 const endLoc: Partial<ElementLocation> = {};
                 if (isClosingEndTag) {
                     endLoc.endTag = { ...ctLoc };
@@ -100,7 +99,7 @@ export class LocationInfoParserMixin<T extends TreeAdapterTypeMap> extends Mixin
                 //NOTE: <body> and <html> are never popped from the stack, so we need to updated
                 //their end location explicitly.
                 const requireExplicitUpdate =
-                    token.type === Tokenizer.END_TAG_TOKEN &&
+                    token.type === TokenType.END_TAG &&
                     (token.tagName === $.HTML || (token.tagName === $.BODY && this.openElements.hasInScope($.BODY)));
 
                 if (requireExplicitUpdate) {

diff --git a/packages/parse5/lib/extensions/location-info/tokenizer-mixin.test.ts b/packages/parse5/lib/extensions/location-info/tokenizer-mixin.test.ts
@@ -1,13 +1,14 @@
 import * as assert from 'assert';
-import { Tokenizer } from '../../tokenizer/index.js';
+import { Tokenizer, TokenizerMode } from '../../tokenizer/index.js';
 import { LocationInfoTokenizerMixin } from './tokenizer-mixin.js';
 import { Mixin } from '../../utils/mixin.js';
+import { TokenType } from './../../common/token.js';
 import { getSubstringByLineCol, normalizeNewLine } from '../../../../../test/utils/common.js';
 
 it('Location Info (Tokenizer)', () => {
     const testCases = [
         {
-            initialMode: Tokenizer.MODE.DATA,
+            initialMode: TokenizerMode.DATA,
             lastStartTagName: '',
             htmlChunks: [
                 '\r\n',
@@ -59,22 +60,22 @@ it('Location Info (Tokenizer)', () => {
             ],
         },
         {
-            initialMode: Tokenizer.MODE.RCDATA,
+            initialMode: TokenizerMode.RCDATA,
             lastStartTagName: 'title',
             htmlChunks: ['<div>Test', ' \n   ', 'hey', ' ', 'ya!', '</title>', '<!--Yo-->'],
         },
         {
-            initialMode: Tokenizer.MODE.RAWTEXT,
+            initialMode: TokenizerMode.RAWTEXT,
             lastStartTagName: 'style',
             htmlChunks: ['.header{', ' \n   ', 'color:red;', '\n', '}', '</style>', 'Some', ' ', 'text'],
         },
         {
-            initialMode: Tokenizer.MODE.SCRIPT_DATA,
+            initialMode: TokenizerMode.SCRIPT_DATA,
             lastStartTagName: 'script',
             htmlChunks: ['var', ' ', 'a=c', ' ', '-', ' ', 'd;', '\n', 'a<--d;', '</script>', '<div>'],
         },
         {
-            initialMode: Tokenizer.MODE.PLAINTEXT,
+            initialMode: TokenizerMode.PLAINTEXT,
             lastStartTagName: 'plaintext',
             htmlChunks: ['Text', ' \n', 'Test</plaintext><div>'],
         },
@@ -97,8 +98,8 @@ it('Location Info (Tokenizer)', () => {
         tokenizer.state = testCase.initialMode;
         tokenizer.lastStartTagName = testCase.lastStartTagName;
 
-        for (let token = tokenizer.getNextToken(), j = 0; token.type !== Tokenizer.EOF_TOKEN; ) {
-            if (token.type === Tokenizer.HIBERNATION_TOKEN) {
+        for (let token = tokenizer.getNextToken(), j = 0; token.type !== TokenType.EOF; ) {
+            if (token.type === TokenType.HIBERNATION) {
                 continue;
             }
 

diff --git a/packages/parse5/lib/extensions/location-info/tokenizer-mixin.ts b/packages/parse5/lib/extensions/location-info/tokenizer-mixin.ts
@@ -1,7 +1,7 @@
 import { Mixin } from '../../utils/mixin.js';
 import { Tokenizer } from '../../tokenizer/index.js';
 import { PositionTrackingPreprocessorMixin } from '../position-tracking/preprocessor-mixin.js';
-import { Location, LocationWithAttributes } from '../../common/token.js';
+import { TokenType, Location, LocationWithAttributes } from '../../common/token.js';
 
 export class LocationInfoTokenizerMixin extends Mixin<Tokenizer> {
     posTracker: PositionTrackingPreprocessorMixin;
@@ -97,7 +97,7 @@ export class LocationInfoTokenizerMixin extends Mixin<Tokenizer> {
                     currentCharacterToken.location!.endOffset = ctLoc.startOffset;
                 }
 
-                if (this.currentToken!.type === Tokenizer.EOF_TOKEN) {
+                if (this.currentToken!.type === TokenType.EOF) {
                     ctLoc.endLine = ctLoc.startLine;
                     ctLoc.endCol = ctLoc.startCol;
                     ctLoc.endOffset = ctLoc.startOffset;

diff --git a/packages/parse5/lib/parser/index.test.ts b/packages/parse5/lib/parser/index.test.ts
@@ -7,7 +7,7 @@ import { NAMESPACES as NS } from '../common/html.js';
 
 const origParseFragment = Parser.prototype.parseFragment;
 
-generateParsingTests('parser', 'Parser', { skipFragments: false }, (test, opts) => ({
+generateParsingTests('parser', 'Parser', {}, (test, opts) => ({
     node: test.fragmentContext
         ? parse5.parseFragment(test.fragmentContext, test.input, opts)
         : parse5.parse(test.input, opts),