Lorem ipsum dolor sit amet, falli detraxit facilisis sit ex. Nec vidit nihil ut, vis utamur definitionem in. Ancillae phaedrum ut eos. Ne eam errem cotidieque. +
++ Sit amet vitae referrentur ad, oportere necessitatibus vituperatoribus vim ne. Audiam quaeque nusquam ei eos, mel ex adhuc prompta complectitur. Decore voluptua cotidieque vel ei, veri novum persecuti te sea. Nisl novum sea cu. Dicta delicatissimi ut est. Et has eripuit impedit mediocrem. +
++ Eu vim evertitur argumentum, pro legendos iudicabit voluptatibus ne. Ne alii nullam deserunt duo, vis no animal verterem. Ne quo veniam euripidis. Pri discere mentitum vituperatoribus ne. Meliore corrumpit vim at, has ea admodum convenire. +
++ Ne mea alii detraxit electram, his ne epicurei gloriatur consetetur. Ex pro aeque omnes appareat, possit integre accommodare eu quo. Qui in porro molestiae, no sit equidem petentium patrioque, omnes voluptaria vim te. Odio quot rebum ad sed, ei sale reque inani sea. +
++ At mea vidit petentium, mei lorem pertinacia signiferumque ne. Per eu dicam postea probatus, sea pericula accusamus omittantur ad. Id mei munere ullamcorper. Oportere ocurreret cu vel. Pri ne dicant soluta graecis, duis ponderum corrumpit sed eu, pro ei rebum vivendum. +
++ Eum ea exerci sententiae constituam, ut lorem putant volutpat pro. Vix ex iudicabit salutatus principes, pri nisl erat ut, ne cum persius verterem phaedrum. Vix mundi vitae eu. Vel summo vocent albucius in. Qui omnis partem possim ex, wisi doctus efficiantur cu has. Aperiri denique ea pri, cum dissentias signiferumque eu. Per posse dissentiet necessitatibus id, omnis eleifend te mel. +
++ Phaedrum erroribus adolescens ut nec, duo mutat viris id. In tale purto dolores sit, ei consul inermis est, his ea movet expetenda. Cibo scribentur id eam, per an quem iracundia, has erat zril solet eu. Falli fierent ne cum, eam placerat facilisis suscipiantur te. Id quo dicat tractatos definiebas, est copiosae splendide id. +
++ Constituto neglegentur qui ne, eum ne putent phaedrum, no quod nominati consulatu per. Mei cu accusam principes interesset, vim fugit abhorreant eu, ludus feugait an sed. Clita expetendis definitionem has in, meis pericula no mea, soluta nemore cotidieque eos cu. Summo populo signiferumque pri eu, eum no labores democritum posidonium, pro ignota adipisci in. Tollit molestie id mei. +
++ Ipsum impetus et eam, ei his epicuri instructior. Impedit ullamcorper vim an. Impedit ullamcorper vim an. Et eum senserit sententiae reprimique, quem detracto sententiae at sed, no inermis ocurreret rationibus sea pericula no. + + diff --git a/rollup.config.ts b/rollup.config.ts index 9dd08e1e9f..b25dea474d 100644 --- a/rollup.config.ts +++ b/rollup.config.ts @@ -71,6 +71,11 @@ const umd = { delimiters: ['', ''], './fs/node': './fs/browser' }), + replace({ + include: './src/parser/tokenizer.ts', + delimiters: ['', ''], + './flatten/node': './flatten/browser' + }), typescript({ tsconfigOverride: { include: [ 'src' ], @@ -99,6 +104,11 @@ const min = { delimiters: ['', ''], './fs/node': './fs/browser' }), + replace({ + include: './src/parser/tokenizer.ts', + delimiters: ['', ''], + './flatten/node': './flatten/browser' + }), typescript({ tsconfigOverride: { include: [ 'src' ], diff --git a/src/parser/flatten/browser.ts b/src/parser/flatten/browser.ts new file mode 100644 index 0000000000..6ffb38bcec --- /dev/null +++ b/src/parser/flatten/browser.ts @@ -0,0 +1,3 @@ +export function flatten (str: string) { + return str +} diff --git a/src/parser/flatten/node.ts b/src/parser/flatten/node.ts new file mode 100644 index 0000000000..e14c2dccd9 --- /dev/null +++ b/src/parser/flatten/node.ts @@ -0,0 +1,10 @@ +/** + * This function forces a string to be re-instantiated in memory using a flat representation instead of a graph + * of concatenated strings. This is an optimization to reduce the memory footprint of token string fragments after + * they are parsed. + * This optimization targets the V8 javascript engine and only works on Node.js. + * @param {string} str + */ +export function flatten (str: string) { + return Buffer.from(str).toString() +} diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 070f7b3de6..c601a1dc09 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -5,6 +5,7 @@ import { Token } from './token' import { OutputToken } from './output-token' import { TokenizationError } from '../util/error' import { NormalizedFullOptions, applyDefault } from '../liquid-options' +import { flatten } from './flatten/node' enum ParseState { HTML, OUTPUT, TAG } @@ -36,7 +37,7 @@ export class Tokenizer { } if (state === ParseState.HTML) { if (input.substr(p, outputDelimiterLeft.length) === outputDelimiterLeft) { - if (buffer) tokens.push(new HTMLToken(buffer, input, line, col, file)) + if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file)) buffer = outputDelimiterLeft line = curLine col = p - lineBegin + 1 @@ -44,7 +45,7 @@ export class Tokenizer { state = ParseState.OUTPUT continue } else if (input.substr(p, tagDelimiterLeft.length) === tagDelimiterLeft) { - if (buffer) tokens.push(new HTMLToken(buffer, input, line, col, file)) + if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file)) buffer = tagDelimiterLeft line = curLine col = p - lineBegin + 1 @@ -57,7 +58,7 @@ export class Tokenizer { input.substr(p, outputDelimiterRight.length) === outputDelimiterRight ) { buffer += outputDelimiterRight - tokens.push(new OutputToken(buffer, buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length), input, line, col, this.options, file)) + tokens.push(new OutputToken(flatten(buffer), buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length), input, line, col, this.options, file)) p += outputDelimiterRight.length buffer = '' line = curLine @@ -66,7 +67,7 @@ export class Tokenizer { continue } else if (input.substr(p, tagDelimiterRight.length) === tagDelimiterRight) { buffer += tagDelimiterRight - tokens.push(new TagToken(buffer, buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length), input, line, col, this.options, file)) + tokens.push(new TagToken(flatten(buffer), buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length), input, line, col, this.options, file)) p += tagDelimiterRight.length buffer = '' line = curLine @@ -81,10 +82,10 @@ export class Tokenizer { const str = buffer.length > 16 ? buffer.slice(0, 13) + '...' : buffer throw new TokenizationError( `${t} "${str}" not closed`, - new Token(buffer, input, line, col, file) + new Token(flatten(buffer), input, line, col, file) ) } - if (buffer) tokens.push(new HTMLToken(buffer, input, line, col, file)) + if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file)) whiteSpaceCtrl(tokens, this.options) return tokens