diff --git a/README.md b/README.md index 132d9ae6..c922477a 100644 --- a/README.md +++ b/README.md @@ -60,22 +60,26 @@ var content = MarkupIt.DraftUtils.decode(rawContent); var text = markdown.toText(content); ``` -#### Custom Syntax +### Extend Syntax This module contains the [markdown syntax](./syntaxes/markdown), but you can write your custom syntax or extend the existing ones. +#### Create rules + ```js var myRule = MarkupIt.Rule(DraftMarkup.BLOCKS.HEADING_1) - .regExp(/^

(\S+)<\/h1>/, function(match) { + .regExp(/^

(\S+)<\/h1>/, function(state, match) { return { - text: match[1] + tokens: state.parseAsInline(match[1]) }; }) - .toText(function(innerText) { - return '

' + innerText+ '

'; + .toText(function(state, token) { + return '

' + state.renderAsInline(token) + '

'; }); ``` +#### Custom Syntax + Create a new syntax inherited from the markdown one: ```js diff --git a/bin/toProseMirror.js b/bin/toProseMirror.js new file mode 100755 index 00000000..69fd0eb1 --- /dev/null +++ b/bin/toProseMirror.js @@ -0,0 +1,14 @@ +#! /usr/bin/env node +/* eslint-disable no-console */ + +var MarkupIt = require('../'); +var utils = require('./utils'); + +utils.command(function(content) { + console.log( + JSON.stringify( + MarkupIt.ProseMirrorUtils.encode(content), + null, 4 + ) + ); +}); diff --git a/lib/__tests__/syntax.js b/lib/__tests__/syntax.js index 56cdc09a..ace913fe 100644 --- a/lib/__tests__/syntax.js +++ b/lib/__tests__/syntax.js @@ -4,7 +4,7 @@ describe('Custom Syntax', function() { var syntax = MarkupIt.Syntax('mysyntax', { inline: [ MarkupIt.Rule(MarkupIt.STYLES.BOLD) - .regExp(/^\*\*([\s\S]+?)\*\*/, function(match) { + .regExp(/^\*\*([\s\S]+?)\*\*/, function(state, match) { return { text: match[1] }; @@ -22,24 +22,27 @@ describe('Custom Syntax', function() { it('should parse as unstyled', function() { var content = markup.toContent('Hello World'); - var tokens = content.getTokens(); - tokens.size.should.equal(1); - var p = tokens.get(0); + var doc = content.getToken(); + var blocks = doc.getTokens(); + + blocks.size.should.equal(1); + var p = blocks.get(0); p.getType().should.equal(MarkupIt.BLOCKS.TEXT); - p.getText().should.equal('Hello World'); + p.getAsPlainText().should.equal('Hello World'); }); it('should parse inline', function() { var content = markup.toContent('Hello **World**'); - var tokens = content.getTokens(); + var doc = content.getToken(); + var blocks = doc.getTokens(); - tokens.size.should.equal(1); - var p = tokens.get(0); + blocks.size.should.equal(1); + var p = blocks.get(0); p.getType().should.equal(MarkupIt.BLOCKS.TEXT); - p.getText().should.equal('Hello World'); + p.getAsPlainText().should.equal('Hello World'); }); }); @@ -47,22 +50,24 @@ describe('Custom Syntax', function() { it('should output correct string', function() { var content = MarkupIt.JSONUtils.decode({ syntax: 'mysyntax', - tokens: [ - { - type: MarkupIt.BLOCKS.PARAGRAPH, - text: 'Hello World', - tokens: [ - { - type: MarkupIt.STYLES.TEXT, - text: 'Hello ' - }, - { - type: MarkupIt.STYLES.BOLD, - text: 'World' - } - ] - } - ] + token: { + type: MarkupIt.BLOCKS.DOCUMENT, + tokens: [ + { + type: MarkupIt.BLOCKS.PARAGRAPH, + tokens: [ + { + type: MarkupIt.STYLES.TEXT, + text: 'Hello ' + }, + { + type: MarkupIt.STYLES.BOLD, + text: 'World' + } + ] + } + ] + } }); var text = markup.toText(content); text.should.equal('Hello **World**\n'); diff --git a/lib/constants/blocks.js b/lib/constants/blocks.js index 4fa29a8a..c6d6484f 100644 --- a/lib/constants/blocks.js +++ b/lib/constants/blocks.js @@ -1,32 +1,31 @@ module.exports = { - IGNORE: 'ignore', - TEXT: 'text', + DOCUMENT: 'doc', + TEXT: 'unstyled', - CODE: 'code-block', + CODE: 'code_block', BLOCKQUOTE: 'blockquote', - PARAGRAPH: 'unstyled', + PARAGRAPH: 'paragraph', FOOTNOTE: 'footnote', - DEFINITION: 'definition', - HTML: 'html-block', - HR: 'atomic', + HTML: 'html_block', + HR: 'hr', - HEADING_1: 'header-one', - HEADING_2: 'header-two', - HEADING_3: 'header-three', - HEADING_4: 'header-four', - HEADING_5: 'header-five', - HEADING_6: 'header-six', + HEADING_1: 'header_one', + HEADING_2: 'header_two', + HEADING_3: 'header_three', + HEADING_4: 'header_four', + HEADING_5: 'header_five', + HEADING_6: 'header_six', TABLE: 'table', - TABLE_HEADER: 'table-header', - TABLE_BODY: 'table-body', - TABLE_ROW: 'table-row', - TABLE_CELL: 'table-cell', + TABLE_ROW: 'table_row', + TABLE_CELL: 'table_cell', - OL_ITEM: 'ordered-list-item', - UL_ITEM: 'unordered-list-item', + OL_LIST: 'ordered_list', + UL_LIST: 'unordered_list', + + LIST_ITEM: 'list_item', // GitBook specifics TEMPLATE: 'template', - MATH: 'math' + MATH: 'math_block' }; diff --git a/lib/constants/defaultRules.js b/lib/constants/defaultRules.js index efe0687d..2b62ad1e 100644 --- a/lib/constants/defaultRules.js +++ b/lib/constants/defaultRules.js @@ -3,14 +3,34 @@ var Rule = require('../models/rule'); var BLOCKS = require('./blocks'); var STYLES = require('./styles'); +var defaultDocumentRule = Rule(BLOCKS.DOCUMENT) + .match(function(state, text) { + return { + tokens: state.parseAsBlock(text) + }; + }) + .toText(function(state, token) { + return state.renderAsBlock(token); + }); + var defaultBlockRule = Rule(BLOCKS.TEXT) + .match(function(state, text) { + return { + tokens: state.parseAsInline(text) + }; + }) .toText('%s\n'); var defaultInlineRule = Rule(STYLES.TEXT) - .setOption('parse', false) + .match(function(state, text) { + return { + text: text + }; + }) .toText('%s'); module.exports = { - blockRule: defaultBlockRule, - inlineRule: defaultInlineRule + documentRule: defaultDocumentRule, + blockRule: defaultBlockRule, + inlineRule: defaultInlineRule }; diff --git a/lib/constants/entities.js b/lib/constants/entities.js index 15a05ec2..b7044f98 100644 --- a/lib/constants/entities.js +++ b/lib/constants/entities.js @@ -1,8 +1,6 @@ module.exports = { LINK: 'link', - LINK_REF: 'link-ref', IMAGE: 'image', - LINK_IMAGE: 'link-image', FOOTNOTE_REF: 'footnote-ref', // GitBook specifics diff --git a/lib/constants/styles.js b/lib/constants/styles.js index 1472d172..e8ac9bf7 100644 --- a/lib/constants/styles.js +++ b/lib/constants/styles.js @@ -1,9 +1,10 @@ module.exports = { + TEXT: 'text', + BOLD: 'BOLD', ITALIC: 'ITALIC', CODE: 'CODE', STRIKETHROUGH: 'STRIKETHROUGH', - TEXT: 'UNSTYLED', HTML: 'HTML', // GitBook specifics diff --git a/lib/draft/__tests__/decode.js b/lib/draft/__tests__/decode.js deleted file mode 100644 index 46f651f0..00000000 --- a/lib/draft/__tests__/decode.js +++ /dev/null @@ -1,78 +0,0 @@ -var decode = require('../decode'); -var ENTITIES = require('../../constants/entities'); -var BLOCKS = require('../../constants/blocks'); -var STYLES = require('../../constants/styles'); - -describe('decode', function() { - var content; - - before(function() { - var rawContent = { - entityMap: { - '1': { - type: ENTITIES.LINK, - mutability: 'MUTABLE', - data: { - href: 'http://google.fr' - } - } - }, - blocks: [ - { - type: BLOCKS.HEADING_1, - text: 'Hello World', - inlineStyleRanges: [], - entityRanges: [] - }, - { - type: BLOCKS.PARAGRAPH, - text: 'This is a link', - inlineStyleRanges: [ - { - offset: 0, - length: 4, - style: STYLES.BOLD - } - ], - entityRanges: [ - { - offset: 10, - length: 4, - key: '1' - } - ] - } - ] - }; - - content = decode(rawContent); - }); - - it('should correctly extract block tokens', function() { - var tokens = content.getTokens(); - - tokens.size.should.equal(2); - tokens.get(0).getType().should.equal(BLOCKS.HEADING_1); - tokens.get(1).getType().should.equal(BLOCKS.PARAGRAPH); - }); - - it('should correctly extract inline styles', function() { - var tokens = content.getTokens(); - var p = tokens.get(1); - var inline = p.getTokens(); - - inline.size.should.equal(3); - - var bold = inline.get(0); - bold.getType().should.equal(STYLES.BOLD); - bold.getText().should.equal('This'); - - var text = inline.get(1); - text.getType().should.equal(STYLES.TEXT); - text.getText().should.equal(' is a '); - - var link = inline.get(2); - link.getType().should.equal(ENTITIES.LINK); - link.getText().should.equal('link'); - }); -}); diff --git a/lib/draft/__tests__/encode.js b/lib/draft/__tests__/encode.js deleted file mode 100644 index bedf96df..00000000 --- a/lib/draft/__tests__/encode.js +++ /dev/null @@ -1,24 +0,0 @@ -var encode = require('../encode'); -var BLOCKS = require('../../constants/blocks'); - -describe('encode', function() { - describe('paragraph + heading', function() { - var rawContent = encode(mock.titleParagraph); - - it('should return empty entityMap', function() { - rawContent.should.have.property('entityMap'); - rawContent.entityMap.should.deepEqual({}); - }); - - it('should return blocks', function() { - rawContent.should.have.property('blocks'); - rawContent.blocks.should.have.lengthOf(2); - - rawContent.blocks[0].should.have.property('type'); - rawContent.blocks[0].should.have.property('text'); - rawContent.blocks[0].type.should.equal(BLOCKS.HEADING_1); - - rawContent.blocks[1].type.should.equal(BLOCKS.PARAGRAPH); - }); - }); -}); diff --git a/lib/draft/decode.js b/lib/draft/decode.js deleted file mode 100644 index b9a8bf6a..00000000 --- a/lib/draft/decode.js +++ /dev/null @@ -1,137 +0,0 @@ -var Immutable = require('immutable'); -var Range = require('range-utils'); - -var Content = require('../models/content'); -var Token = require('../models/token'); - -var STYLES = require('../constants/styles'); - -var decodeEntities = require('./decodeEntities'); - -/** - * Return true if range is an entity - * - * @param {Range} - * @return {Boolean} - */ -function isRangeEntity(range) { - return Boolean(range.entity); -} - -/** - * Convert a RangeTreeNode to a token - * - * @param {String} - * @param {RangeTreeNode} - * @return {Token} - */ -function encodeRangeNodeToToken(baseText, range) { - var innerText = baseText.slice(range.offset, range.offset + range.length); - var isRange = isRangeEntity(range); - - return new Token({ - type: isRange? range.entity.type : range.style, - text: innerText, - data: isRange? range.entity.data : {}, - tokens: range.style == STYLES.TEXT? [] : encodeRangeTreeToTokens(innerText, range.children || []) - }); -} - -/** - * Convert a RangeTree to a list of tokens - * - * @param {String} - * @param {RangeTree} - * @return {List} - */ -function encodeRangeTreeToTokens(baseText, ranges) { - ranges = Range.fill(baseText, ranges, { - style: STYLES.TEXT - }); - - return new Immutable.List( - ranges.map(function(range) { - return encodeRangeNodeToToken(baseText, range); - }) - ); -} - -/** - * Convert a ContentBlock into a token - * - * @param {ContentBlock} block - * @param {Object} entityMap - * @return {Token} - */ -function encodeBlockToToken(block, entityMap) { - var text = block.text; - - var styleRanges = block.inlineStyleRanges; - var entityRanges = block.entityRanges; - var blocks = block.blocks; - var tokens; - - if (blocks && blocks.length > 0) { - tokens = decodeDraftBlocksToTokens(blocks, entityMap); - } else { - // Unmerge link-image - entityRanges = decodeEntities(entityRanges, entityMap); - - var allEntities = [] - .concat(entityRanges) - .concat(styleRanges); - - var tree = Range.toTree(allEntities, isRangeEntity); - tokens = encodeRangeTreeToTokens(text, tree); - } - - return new Token({ - type: block.type, - text: text, - data: block.data, - tokens: tokens - }); -} - -/** - * Transform a an array of blocks to a list of tokens - * - * @param {Array} rawBlocks - * @param {Object} entityMap - * @return {List} - */ -function decodeDraftBlocksToTokens(rawBlocks, entityMap) { - return Immutable.List(rawBlocks) - .map(function(block) { - return encodeBlockToToken(block, entityMap); - }); -} - -/** - * Transform a RawContentState from draft into a list of tokens - * - * @param {RawContentState} rawContentState - * @return {List} - */ -function decodeDraftToTokens(rawContentState) { - var blocks = rawContentState.blocks; - var entityMap = rawContentState.entityMap; - - return decodeDraftBlocksToTokens(blocks, entityMap); -} - -/** - * Transform a RawContentState from draft into a Content instance - * - * @param {RawContentState} rawContentState - * @param {String} syntaxName - * @return {Content} - */ -function decodeDraftToContent(rawContentState, syntaxName) { - syntaxName = syntaxName || 'draft-js'; - var tokens = decodeDraftToTokens(rawContentState); - - return Content.createFromTokens(syntaxName, tokens); -} - -module.exports = decodeDraftToContent; diff --git a/lib/draft/decodeEntities.js b/lib/draft/decodeEntities.js deleted file mode 100644 index b6451ca9..00000000 --- a/lib/draft/decodeEntities.js +++ /dev/null @@ -1,47 +0,0 @@ -var is = require('is'); -var Range = require('range-utils'); -var ENTITIES = require('../constants/entities'); - -/** - * Normalize and decode a list of entity ranges - * - * @param {Array} entityRanges - * @param {Object} entityMap - * @return {Array} - */ -function decodeEntities(entityRanges, entityMap) { - return entityRanges.reduce(function(result, range) { - if (is.defined(range.key)) { - range.entity = entityMap[String(range.key)]; - delete range.key; - } - - if (!range.entity || range.entity.type != ENTITIES.LINK_IMAGE) { - result.push(range); - return result; - } - - result.push(Range(range.offset, range.length, { - entity: { - type: ENTITIES.IMAGE, - data: { - src: range.entity.data.src, - title: range.entity.data.imageTitle - } - } - })); - result.push(Range(range.offset, range.length, { - entity: { - type: ENTITIES.LINK, - data: { - href: range.entity.data.href, - title: range.entity.data.linkTitle - } - } - })); - - return result; - }, []); -} - -module.exports = decodeEntities; diff --git a/lib/draft/encode.js b/lib/draft/encode.js deleted file mode 100644 index 9ee8278b..00000000 --- a/lib/draft/encode.js +++ /dev/null @@ -1,209 +0,0 @@ -var Immutable = require('immutable'); -var Range = require('range-utils'); - -var walk = require('../utils/walk'); -var genKey = require('../utils/genKey'); -var getMutability = require('./getMutability'); - -var ENTITIES = require('../constants/entities'); -var STYLES = require('../constants/styles'); - -/** - * Default option for encoding to draft - */ -var EncodingOption = Immutable.Record({ - // Blacklist some type of tokens - blacklist: [] -}); - - -/** - * Encode an entity from a token - * - * @param {Token} token - * @return {Object} - */ -function encodeTokenToEntity(token) { - return { - type: token.getType(), - mutability: getMutability(token), - data: token.getData().toJS() - }; -} - -/** - * Encode a token into a ContentBlock - * - * @param {Token} token - * @param {Function} registerEntity - * @param {Function} filter - * @return {Object} - */ -function encodeTokenToBlock(token, registerEntity, filter) { - var tokenType = token.getType(); - var inlineStyleRanges = []; - var entityRanges = []; - var blockTokens = []; - - // Add child block tokens as data.innerContent - token.getTokens() - .forEach(function(tok) { - if (!tok.isBlock()) { - return; - } - - blockTokens.push(tok); - }); - - var innerText = walk(token, function(tok, range) { - if (tok.isEntity()) { - if (!filter(tok)) { - return; - } - - var entity = encodeTokenToEntity(tok); - - entityRanges.push( - Range( - range.offset, - range.length, - { - entity: entity - } - ) - ); - - } else if (tok.isStyle() && tok.getType() !== STYLES.TEXT) { - if (!filter(tok)) { - return; - } - - inlineStyleRanges.push( - Range( - range.offset, - range.length, - { - style: tok.getType() - } - ) - ); - } - }); - - // Linearize/Merge ranges (draft-js doesn't support multiple entities on the same range) - entityRanges = Range.merge(entityRanges, function(a, b) { - if ( - (a.entity.type == ENTITIES.IMAGE || a.entity.type == ENTITIES.LINK) && - (b.entity.type == ENTITIES.IMAGE || b.entity.type == ENTITIES.LINK) && - (a.entity.type !== b.entity.type) - ) { - var img = ((a.entity.type == ENTITIES.IMAGE)? a : b).entity.data; - var link = ((a.entity.type == ENTITIES.LINK)? a : b).entity.data; - - return Range(a.offset, a.length, { - entity: { - type: ENTITIES.LINK_IMAGE, - mutability: getMutability(ENTITIES.LINK_IMAGE), - data: { - src: img.src, - href: link.href, - imageTitle: img.title, - linkTitle: link.title - } - } - }); - } - - return a; - }); - - // Register all entities - entityRanges = entityRanges.map(function(range) { - var entityKey = registerEntity(range.entity); - - return Range(range.offset, range.length, { - key: entityKey - }); - }); - - // Metadata for this blocks - var data = token.getData().toJS(); - - // Encode inner tokens - blockTokens = encodeTokensToBlocks(Immutable.List(blockTokens), registerEntity, filter); - - return { - key: genKey(), - type: tokenType, - text: innerText, - data: data, - inlineStyleRanges: inlineStyleRanges, - entityRanges: entityRanges, - blocks: blockTokens - }; -} - - -/** - * Encode a list of Token into a RawContentState for draft - * - * @paran {List} tokens - * @param {Function} registerEntity - * @param {Function} filter - * @return {Object} - */ -function encodeTokensToBlocks(blockTokens, registerEntity, filter) { - return blockTokens - .map(function(token) { - if (!filter(token)) { - return; - } - - return encodeTokenToBlock(token, registerEntity, filter); - }) - .filter(function(blk) { - return Boolean(blk); - }) - .toJS(); -} - - -/** - * Encode a Content instance into a RawContentState for draft - * - * @paran {Content} content - * @return {Object} - */ -function encodeContentToDraft(content, options) { - options = EncodingOption(options || {}); - var blacklist = Immutable.List(options.get('blacklist')); - var blockTokens = content.getTokens(); - var entityKey = 0; - var entityMap = {}; - - // Register an entity and returns its key/ID - function registerEntity(entity) { - entityKey++; - entityMap[entityKey] = entity; - - return entityKey; - } - - // Filter tokens - function filter(token) { - var type = token.getType(); - - if (blacklist.includes(type)) { - throw new Error('Content of type "' + type + '" is not allowed'); - } - - return true; - } - - return { - blocks: encodeTokensToBlocks(blockTokens, registerEntity, filter), - entityMap: entityMap - }; -} - -module.exports = encodeContentToDraft; diff --git a/lib/draft/getMutability.js b/lib/draft/getMutability.js deleted file mode 100644 index 479f3de1..00000000 --- a/lib/draft/getMutability.js +++ /dev/null @@ -1,23 +0,0 @@ -var is = require('is'); -var ENTITIES = require('../constants/entities'); - -var MUTABLE = 'MUTABLE'; -var IMMUTABLE = 'IMMUTABLE'; - -var MUTABLE_TYPES = [ - ENTITIES.LINK, ENTITIES.LINK_REF, ENTITIES.FOOTNOTE_REF -]; - -/** - * Get mutability of a token - * - * @param {Token|String} token - * @return @String - */ -function getMutability(token) { - var tokenType = is.string(token)? token : token.getType(); - return (MUTABLE_TYPES.indexOf(tokenType) >= 0)? MUTABLE : IMMUTABLE; -} - - -module.exports = getMutability; diff --git a/lib/draft/index.js b/lib/draft/index.js deleted file mode 100644 index 53f9a69c..00000000 --- a/lib/draft/index.js +++ /dev/null @@ -1,7 +0,0 @@ -var encode = require('./encode'); -var decode = require('./decode'); - -module.exports = { - encode: encode, - decode: decode -}; diff --git a/lib/index.js b/lib/index.js index dabbd22f..4feb67aa 100644 --- a/lib/index.js +++ b/lib/index.js @@ -13,32 +13,32 @@ var Token = require('./models/token'); var parse = require('./parse'); var render = require('./render'); -var DraftUtils = require('./draft'); var JSONUtils = require('./json'); +var ProseMirrorUtils = require('./prosemirror'); var genKey = require('./utils/genKey'); +var transform = require('./utils/transform'); module.exports = Markup; // Method -module.exports.parse = parse; -module.exports.parseInline = parse.inline; - +module.exports.parse = parse; module.exports.render = render; // Models -module.exports.Content = Content; -module.exports.Token = Token; -module.exports.Syntax = Syntax; -module.exports.Rule = Rule; +module.exports.Content = Content; +module.exports.Token = Token; +module.exports.Syntax = Syntax; +module.exports.Rule = Rule; module.exports.RulesSet = RulesSet; // Utils -module.exports.DraftUtils = DraftUtils; -module.exports.JSONUtils = JSONUtils; -module.exports.genKey = genKey; +module.exports.JSONUtils = JSONUtils; +module.exports.ProseMirrorUtils = ProseMirrorUtils; +module.exports.genKey = genKey; +module.exports.transform = transform; // Constants -module.exports.STYLES = STYLES; +module.exports.STYLES = STYLES; module.exports.ENTITIES = ENTITIES; -module.exports.BLOCKS = BLOCKS; +module.exports.BLOCKS = BLOCKS; diff --git a/lib/json/__tests__/decode.js b/lib/json/__tests__/decode.js index 0b6ef609..1afe1d50 100644 --- a/lib/json/__tests__/decode.js +++ b/lib/json/__tests__/decode.js @@ -7,13 +7,16 @@ describe('decode', function() { before(function() { content = decode({ syntax: 'mysyntax', - tokens: [ - { - type: BLOCKS.PARAGRAPH, - text: 'Hello World', - raw: 'Hello World' - } - ] + token: { + type: BLOCKS.DOCUMENT, + tokens: [ + { + type: BLOCKS.PARAGRAPH, + text: 'Hello World', + raw: 'Hello World' + } + ] + } }); }); @@ -22,7 +25,8 @@ describe('decode', function() { }); it('should decode tokens tree', function() { - var tokens = content.getTokens(); + var doc = content.getToken(); + var tokens = doc.getTokens(); tokens.size.should.equal(1); var p = tokens.get(0); diff --git a/lib/json/__tests__/encode.js b/lib/json/__tests__/encode.js index 0436fa10..11ada671 100644 --- a/lib/json/__tests__/encode.js +++ b/lib/json/__tests__/encode.js @@ -1,7 +1,7 @@ var encode = require('../encode'); var BLOCKS = require('../../constants/blocks'); -describe('decode', function() { +describe('encode', function() { var json; before(function() { @@ -13,9 +13,13 @@ describe('decode', function() { }); it('should encode tokens', function() { - json.tokens.should.have.lengthOf(1); + json.should.have.property('token'); - var p = json.tokens[0]; + var doc = json.token; + + doc.tokens.should.have.lengthOf(1); + + var p = doc.tokens[0]; p.type.should.equal(BLOCKS.PARAGRAPH); p.text.should.equal('Hello World'); p.tokens.should.be.an.Array().with.lengthOf(2); diff --git a/lib/json/decode.js b/lib/json/decode.js index b761343a..33dd33b4 100644 --- a/lib/json/decode.js +++ b/lib/json/decode.js @@ -36,9 +36,9 @@ function decodeTokensFromJSON(json) { * @return {Content} */ function decodeContentFromJSON(json) { - return Content.createFromTokens( + return Content.createFromToken( json.syntax, - decodeTokensFromJSON(json.tokens) + decodeTokenFromJSON(json.token) ); } diff --git a/lib/json/encode.js b/lib/json/encode.js index 900b827c..ba5cff05 100644 --- a/lib/json/encode.js +++ b/lib/json/encode.js @@ -32,7 +32,7 @@ function encodeTokenToJSON(token) { * Encode a list of tokens to JSON * * @paran {List} tokens - * @return {Object} + * @return {Array} */ function encodeTokensToJSON(tokens) { return tokens.map(encodeTokenToJSON).toJS(); @@ -47,7 +47,7 @@ function encodeTokensToJSON(tokens) { function encodeContentToJSON(content) { return { syntax: content.getSyntax(), - tokens: encodeTokensToJSON(content.getTokens()) + token: encodeTokenToJSON(content.getToken()) }; } diff --git a/lib/models/__tests__/token.js b/lib/models/__tests__/token.js index 493ad40f..4f381a89 100644 --- a/lib/models/__tests__/token.js +++ b/lib/models/__tests__/token.js @@ -4,8 +4,8 @@ var STYLES = require('../../constants/styles'); describe('Token', function() { describe('.mergeWith', function() { it('should merge text and raw', function() { - var base = Token.createInlineText('Hello '); - var other = Token.createInlineText('world'); + var base = Token.createText('Hello '); + var other = Token.createText('world'); var token = base.mergeWith(other); token.getType().should.equal(STYLES.TEXT); diff --git a/lib/models/content.js b/lib/models/content.js index 0e2b4081..a3a9d3bd 100644 --- a/lib/models/content.js +++ b/lib/models/content.js @@ -1,11 +1,13 @@ var Immutable = require('immutable'); +var Token = require('./token'); +var BLOCKS = require('../constants/BLOCKS'); var Content = Immutable.Record({ // Name of the syntax used to parse syntax: String(), - // List of block tokens - tokens: new Immutable.List() + // Entry token + token: Token.create(BLOCKS.DOCUMENT) }); // ---- GETTERS ---- @@ -13,8 +15,8 @@ Content.prototype.getSyntax = function() { return this.get('syntax'); }; -Content.prototype.getTokens = function() { - return this.get('tokens'); +Content.prototype.getToken = function() { + return this.get('token'); }; // ---- STATICS ---- @@ -23,13 +25,13 @@ Content.prototype.getTokens = function() { * Create a content from a syntax and a list of tokens * * @param {Syntax} syntax - * @param {Array|List} + * @param {Token} * @return {Content} */ -Content.createFromTokens = function(syntax, tokens) { +Content.createFromToken = function(syntax, token) { return new Content({ syntax: syntax, - tokens: new Immutable.List(tokens) + token: token }); }; diff --git a/lib/models/rule.js b/lib/models/rule.js index 3eea70b7..8d1ee295 100644 --- a/lib/models/rule.js +++ b/lib/models/rule.js @@ -2,20 +2,12 @@ var Immutable = require('immutable'); var is = require('is'); var inherits = require('util').inherits; +var Token = require('./token'); + var RuleRecord = Immutable.Record({ // Type of the rule type: new String(), - // Options for this rule - options: new Immutable.Map({ - // Mode for parsing inner content of a token - // Values can be "inline", "block" or false - parse: 'inline', - - // Render inner content - renderInner: true - }), - // Listener / Handlers {Map()} listeners: new Immutable.Map() }); @@ -35,10 +27,6 @@ Rule.prototype.getType = function() { return this.get('type'); }; -Rule.prototype.getOptions = function() { - return this.get('options'); -}; - Rule.prototype.getListeners = function() { return this.get('listeners'); }; @@ -63,30 +51,6 @@ Rule.prototype.on = function(key, fn) { return this.set('listeners', listeners); }; -/** - * Set an option - * @param {String} key - * @param {String|Number|Boolean} value - * @return {Rule} - */ -Rule.prototype.setOption = function(key, value) { - var options = this.getOptions(); - - options = options.set(key, value); - - return this.set('options', options); -}; - -/** - * Get an option - * @param {String} key - * @return {String|Number|Boolean} - */ -Rule.prototype.getOption = function(key, defaultValue) { - var options = this.getOptions(); - return options.get(key, defaultValue); -}; - /** * Add a template or function to render a token * @param {String|Function} fn @@ -95,9 +59,9 @@ Rule.prototype.getOption = function(key, defaultValue) { Rule.prototype.toText = function(fn) { if (is.string(fn)) { var tpl = fn; - fn = function (text) { + fn = function (state, token) { return tpl.replace('%s', function() { - return text; + return state.render(token); }); }; } @@ -105,15 +69,6 @@ Rule.prototype.toText = function(fn) { return this.on('text', fn); }; -/** - * Add a finish callback - * @param {Function} fn - * @return {Rule} - */ -Rule.prototype.finish = function(fn) { - return this.on('finish', fn); -}; - /** * Add a match callback * @param {Function} fn @@ -130,21 +85,28 @@ Rule.prototype.match = function(fn) { * @return {Rule} */ Rule.prototype.regExp = function(re, propsFn) { - var ruleType = this.get('type'); - - return this.match(function(text, parent) { + return this.match(function(state, text) { var block = {}; var match = re.exec(text); - if (!match) return null; - - if (propsFn) block = propsFn.call(this, match, parent); - if (!block) return null; - if (is.array(block)) return block; + if (!match) { + return null; + } + if (propsFn) { + block = propsFn.call(null, state, match); + } + + if (!block) { + return null; + } + else if (block instanceof Token) { + return block; + } + else if (is.array(block)) { + return block; + } block.raw = is.undefined(block.raw)? match[0] : block.raw; - block.text = is.undefined(block.text)? match[0] : block.text; - block.type = block.type || ruleType; return block; }); @@ -155,8 +117,8 @@ Rule.prototype.regExp = function(re, propsFn) { * @param {String} key * @return {Mixed} */ -Rule.prototype.emit = function(key, ctx) { - var args = Array.prototype.slice.call(arguments, 2); +Rule.prototype.emit = function(key) { + var args = Array.prototype.slice.call(arguments, 1); var listeners = this.getListeners(); // Add the function to the list @@ -165,23 +127,28 @@ Rule.prototype.emit = function(key, ctx) { return fns.reduce(function(result, fn) { if (result) return result; - return fn.apply(ctx, args); + return fn.apply(null, args); }, null); }; -// Parse a text in a specific context -Rule.prototype.onText = function(ctx, text, parent) { - return this.emit('match', ctx, text, parent); -}; - -// Parsing is finished -Rule.prototype.onFinish = function(ctx, token) { - return this.emit('finish', ctx, token) || token; +/** + * Parse a text using matching rules and return a list of tokens + * @param {ParsingState} state + * @param {String} text + * @return {List} + */ +Rule.prototype.onText = function(state, text) { + return this.emit('match', state, text); }; -// Output inner of block as a string -Rule.prototype.onToken = function(ctx, text, entity, pos) { - return this.emit('text', ctx, text, entity, pos); +/** + * Render a token as a string + * @param {RenderingState} state + * @param {Token} token + * @return {String} + */ +Rule.prototype.onToken = function(state, text) { + return this.emit('text', state, text); }; module.exports = Rule; diff --git a/lib/models/syntax.js b/lib/models/syntax.js index e5cf87b3..712ea9fa 100644 --- a/lib/models/syntax.js +++ b/lib/models/syntax.js @@ -1,13 +1,15 @@ var Immutable = require('immutable'); var inherits = require('util').inherits; +var Rule = require('./rule'); var RulesSet = require('./rules'); var defaultRules = require('../constants/defaultRules'); var SyntaxSetRecord = Immutable.Record({ - name: String(), - inline: new RulesSet([]), - blocks: new RulesSet([]) + name: String(), + entryRule: new Rule(), + inline: new RulesSet([]), + blocks: new RulesSet([]) }); function SyntaxSet(name, def) { @@ -16,14 +18,19 @@ function SyntaxSet(name, def) { } SyntaxSetRecord.call(this, { - name: name, - inline: new RulesSet(def.inline), - blocks: new RulesSet(def.blocks) + name: name, + entryRule: def.entryRule, + inline: new RulesSet(def.inline), + blocks: new RulesSet(def.blocks) }); } inherits(SyntaxSet, SyntaxSetRecord); // ---- GETTERS ---- +SyntaxSet.prototype.getEntryRule = function() { + return this.get('entryRule') || defaultRules.documentRule; +}; + SyntaxSet.prototype.getName = function() { return this.get('name'); }; diff --git a/lib/models/token.js b/lib/models/token.js index d85e9279..36a5a488 100644 --- a/lib/models/token.js +++ b/lib/models/token.js @@ -16,9 +16,10 @@ var TokenRecord = Immutable.Record({ data: new Immutable.Map(), // Inner text of this token (for inline tokens) - text: String(), + text: null, // Original raw content of this token + // Can be use for annotating raw: String(), // List of children tokens (for block tokens) @@ -32,10 +33,10 @@ function Token(def) { } TokenRecord.call(this, { - type: def.type, - data: new Immutable.Map(def.data), - text: def.text, - raw: def.raw, + type: def.type, + data: new Immutable.Map(def.data), + text: def.text, + raw: def.raw, tokens: new Immutable.List(def.tokens) }); } @@ -94,14 +95,6 @@ Token.prototype.isEntity = function() { return isEntity(this); }; -/** - * Return true if is a list item token - * @return {Boolean} - */ -Token.prototype.isListItem = function() { - return this.getType() === BLOCKS.UL_ITEM || this.getType() === BLOCKS.OL_ITEM; -}; - /** * Merge this token with another one * @param {Token} token @@ -119,6 +112,31 @@ Token.prototype.mergeWith = function(token) { }); }; +/** + * Update data of the token + * @param {Object|Map} + * @return {Token} + */ +Token.prototype.setData = function(data) { + return this.set('data', Immutable.Map(data)); +}; + +/** + * Return plain text of a token merged with its children. + * @return {String} + */ +Token.prototype.getAsPlainText = function() { + var tokens = this.getTokens(); + + if (tokens.size === 0) { + return (this.getText() || ''); + } + + return tokens.reduce(function(text, tok) { + return text + tok.getAsPlainText(); + }, ''); +}; + // ---- STATICS ---- /** @@ -127,14 +145,22 @@ Token.prototype.mergeWith = function(token) { * @return {Token} */ Token.create = function(type, tok) { - var text = tok.text || ''; + tok = tok || {}; + + var text = tok.text || ''; + var tokens = Immutable.List(tok.tokens || []); + var data = Immutable.Map(tok.data || {}); + + if (tokens.size > 0) { + text = undefined; + } return new Token({ type: type, text: text, raw: tok.raw || '', - tokens: Immutable.List(tok.tokens || []), - data: Immutable.Map(tok.data || {}) + tokens: tokens, + data: data }); }; @@ -143,24 +169,10 @@ Token.create = function(type, tok) { * @param {String} text * @return {Token} */ -Token.createInlineText = function(text) { +Token.createText = function(text) { return Token.create(STYLES.TEXT, { text: text, - raw: text - }); -}; - -/** - * Create a token for a block text - * @param {String} raw - * @return {Token} - */ -Token.createBlockText = function(raw) { - var text = raw.trim(); - - return Token.create(BLOCKS.TEXT, { - text: text, - raw: raw + raw: text }); }; diff --git a/lib/parse/__tests__/mergeTokens.js b/lib/parse/__tests__/mergeTokens.js index 3de416aa..cf91d09d 100644 --- a/lib/parse/__tests__/mergeTokens.js +++ b/lib/parse/__tests__/mergeTokens.js @@ -7,8 +7,8 @@ var mergeTokens = require('../mergeTokens'); describe('mergeTokens', function() { it('should merge two tokens', function() { var tokens = Immutable.List([ - Token.createInlineText('Hello '), - Token.createInlineText('world') + Token.createText('Hello '), + Token.createText('world') ]); var merged = mergeTokens(tokens, [STYLES.TEXT]); @@ -21,9 +21,9 @@ describe('mergeTokens', function() { it('should merge three tokens', function() { var tokens = Immutable.List([ - Token.createInlineText('Hello '), - Token.createInlineText('world'), - Token.createInlineText('!') + Token.createText('Hello '), + Token.createText('world'), + Token.createText('!') ]); var merged = mergeTokens(tokens, [STYLES.TEXT]); @@ -36,14 +36,14 @@ describe('mergeTokens', function() { it('should merge 2x2 tokens', function() { var tokens = Immutable.List([ - Token.createInlineText('Hello '), - Token.createInlineText('world'), + Token.createText('Hello '), + Token.createText('world'), new Token({ type: STYLES.BOLD, text: ', right?' }), - Token.createInlineText('!'), - Token.createInlineText('!') + Token.createText('!'), + Token.createText('!') ]); var merged = mergeTokens(tokens, [STYLES.TEXT]); diff --git a/lib/parse/cleanup.js b/lib/parse/cleanup.js deleted file mode 100644 index 061079ab..00000000 --- a/lib/parse/cleanup.js +++ /dev/null @@ -1,17 +0,0 @@ - -/** - * Cleanup a text before parsing: normalize newlines and tabs - * - * @param {String} src - * @return {String} - */ -function cleanupText(src) { - return src - .replace(/\r\n|\r/g, '\n') - .replace(/\t/g, ' ') - .replace(/\u00a0/g, ' ') - .replace(/\u2424/g, '\n') - .replace(/^ +$/gm, ''); -} - -module.exports = cleanupText; diff --git a/lib/parse/finish.js b/lib/parse/finish.js deleted file mode 100644 index 212b6ff3..00000000 --- a/lib/parse/finish.js +++ /dev/null @@ -1,36 +0,0 @@ -var transform = require('../utils/transform'); - -/** - * Post processing for parsing. - * Call `onFinish` of rules. - * - * @param {Syntax} syntax - * @param {List} tokens - * @return {List} - */ -function finish(syntax, content, ctx) { - return transform(content, function(token, depth) { - var tokenType = token.getType(); - var rule; - - if (token.isInline()) { - rule = syntax.getInlineRule(tokenType); - } else { - rule = syntax.getBlockRule(tokenType); - } - - var def = { - type: token.getType(), - data: token.getData().toJS(), - text: token.getText(), - raw: token.getRaw() - }; - - return token.merge( - rule.onFinish(ctx, def) - ); - }); -} - - -module.exports = finish; diff --git a/lib/parse/index.js b/lib/parse/index.js index 38f1d1bf..df570d88 100644 --- a/lib/parse/index.js +++ b/lib/parse/index.js @@ -1,149 +1,19 @@ -var Immutable = require('immutable'); - -var Token = require('../models/token'); var Content = require('../models/content'); -var getText = require('../utils/getText'); -var lex = require('./lex'); -var finish = require('./finish'); -var cleanup = require('./cleanup'); - -function createContext(ctx) { - return (ctx || {}); -} - -/** - * Parse an inline text into a list of tokens - * - * @param {Syntax} syntax - * @param {String} text - * @param {List} parents - * @param {Object} ctx - * @return {List} - */ -function parseAsInlineTokens(syntax, text, parents, ctx) { - var inlineRulesSet = syntax.getInlineRulesSet(); - var inlineRules = inlineRulesSet.getRules(); - - // Parse block tokens - var tokens = lex.inline(inlineRules, text, parents, ctx); - - // Parse inline content for each token - tokens = tokens.map(function(token) { - return parseInnerToken(syntax, token, parents, ctx); - }); - - return tokens; -} - - -/** - * Parse a text using a syntax into a list of block tokens - * - * @param {Syntax} syntax - * @param {String} text - * @param {List} parents - * @param {Object} ctx - * @return {List} - */ -function parseAsBlockTokens(syntax, text, parents, ctx) { - text = cleanup(text); - - var blockRulesSet = syntax.getBlockRulesSet(); - var blockRules = blockRulesSet.getRules(); - - // Parse block tokens - var tokens = lex.block(blockRules, text, parents, ctx); - - // Parse inline content for each token - tokens = tokens.map(function(token) { - return parseInnerToken(syntax, token, parents, ctx); - }); - - return tokens; -} +var ParsingState = require('./state'); +var matchRule = require('./matchRule'); /** - * Parse inner of a token according to the options. - * It returns the modified token. - * - * @param {Syntax} syntax - * @param {Token} token - * @param {List} parents - * @param {Object} ctx - * @return {Token} - */ -function parseInnerToken(syntax, token, parents, ctx) { - var tokenType = token.getType(); - var rule = (token.isInline()? - syntax.getInlineRule(tokenType) : - syntax.getBlockRule(tokenType) - ); - var parseMode = rule.getOption('parse'); - - if (!parseMode) { - return token; - } - - // Add it to the new parents list - parents = parents.push(token); - - // Parse inner tokens if none - var tokens = token.getTokens(); - if (tokens.size === 0) { - if (parseMode === 'block') { - tokens = parseAsBlockTokens(syntax, token.getText(), parents, ctx); - } else { - tokens = parseAsInlineTokens(syntax, token.getText(), parents, ctx); - } - - token = token.set('tokens', tokens); - } - - token = token.set('text', getText(token)); - - return token; -} - -/** - * Parse a text using a syntax into a Content - * - * @param {Syntax} syntax - * @param {String} text - * @param {Object} ctx + * Parse a text using a syntax + * @param {Syntax} syntax + * @param {String} text * @return {Content} */ -function parseAsContent(syntax, text, ctx) { - // Create a new context - ctx = createContext(ctx); - - // Parse inline content for each token - var tokens = parseAsBlockTokens(syntax, text, Immutable.List(), ctx); - - // We always return at least one block token - if (tokens.size === 0) { - tokens = tokens.push(Token.createBlockText('')); - } - - var content = Content.createFromTokens(syntax.getName(), tokens); - return finish(syntax, content, ctx); -} - -/** - * Parse an inline string to a Content - * - * @param {Syntax} syntax - * @param {String} text - * @param {Object} ctx - * @return {Content} - */ -function parseInline(syntax, text, ctx) { - text = cleanup(text); - - var tokens = parseAsInlineTokens(syntax, text, Immutable.List(), ctx); +function parse(syntax, text) { + var entryRule = syntax.getEntryRule(); + var state = new ParsingState(syntax); + var tokens = matchRule(state, entryRule, text); - var content = Content.createFromTokens(syntax.getName(), tokens); - return finish(syntax, content, ctx); + return Content.createFromToken(syntax.getName(), tokens.first()); } -module.exports = parseAsContent; -module.exports.inline = parseInline; +module.exports = parse; diff --git a/lib/parse/lex.js b/lib/parse/lex.js index f68f997e..0e4087c5 100644 --- a/lib/parse/lex.js +++ b/lib/parse/lex.js @@ -1,120 +1,60 @@ -var is = require('is'); var Immutable = require('immutable'); -var Token = require('../models/token'); -var BLOCKS = require('../constants/blocks'); -var STYLES = require('../constants/styles'); -var mergeTokens = require('./mergeTokens'); - -/** - * Convert a normal text into a list of unstyled tokens (block or inline) - * - * @param {String} text - * @param {Boolean} inlineMode - * @return {List} - */ -function textToUnstyledTokens(text, inlineMode) { - var accu = '', c, wasNewLine = false; - var result = []; - - function pushAccu() { - var token = inlineMode? - Token.createInlineText(accu) : - Token.createBlockText(accu); - - accu = ''; - if (token.getText().length !== 0) { - result.push(token); - } - } - - for (var i = 0; i < text.length; i++) { - c = text[i]; - - if (c !== '\n' && wasNewLine) { - pushAccu(); - } - - accu += c; - wasNewLine = (c === '\n'); - } - - pushAccu(); - - return new Immutable.List(result); -} +var textToUnstyledTokens = require('./textToUnstyledTokens'); +var matchRule = require('./matchRule'); /** * Process a text using a set of rules * to return a flat list of tokens * - * @param {Boolean} inlineMode + * @param {ParsingState} state * @param {List} rules + * @param {Boolean} isInline * @param {String} text - * @param {List} parents - * @param {Object} ctx * @return {List} */ -function lex(inlineMode, rules, text, parents, ctx, nonParsed) { - var parsedTokens; - var tokens = new Immutable.List(); +function lex(state, rules, isInline, text, nonParsed) { + var tokens = Immutable.List(); + var matchedTokens; + nonParsed = nonParsed || ''; if (!text) { - return tokens.concat(textToUnstyledTokens(nonParsed, inlineMode)); + return tokens.concat(textToUnstyledTokens(state, isInline, nonParsed)); } rules.forEach(function(rule) { - var matches = rule.onText(ctx, text, parents); - - if (!matches || matches.length === 0) return; - if (!is.array(matches)) { - matches = [matches]; + matchedTokens = matchRule(state, rule, text); + if (!matchedTokens) { + return; } - parsedTokens = Immutable.List(matches) - .map(function(match) { - return Token.create(match.type, match); - }); - return false; }); - // Nothing match this text, we move to the next character and try again - // When found, we add a new token "unstyled" - if (!parsedTokens) { - nonParsed += text[0]; - text = text.substring(1); - - return lex(inlineMode, rules, text, parents, ctx, nonParsed); - } else { - tokens = tokens.concat(textToUnstyledTokens(nonParsed, inlineMode)); - parsedTokens.forEach(function(token) { - // Push new token - if (token.getType() != BLOCKS.IGNORE) tokens = tokens.push(token); - - // Update source text - text = text.substring(token.getRaw().length); - }); + if (!matchedTokens) { + nonParsed += text[0]; + text = text.substring(1); - // Keep parsing - tokens = tokens.concat( - lex(inlineMode, rules, text, parents, ctx) - ); + return lex(state, rules, isInline, text, nonParsed); } - if (inlineMode) { - tokens = mergeTokens(tokens, [ - STYLES.TEXT - ]); - } + var newText = matchedTokens.reduce(function(result, token) { + return result.substring(token.getRaw().length); + }, text); + + // Keep parsing + tokens = textToUnstyledTokens(state, isInline, nonParsed) + .concat( + matchedTokens + ) + .concat( + lex(state, rules, isInline, newText) + ); return tokens; } -module.exports = { - inline: lex.bind(null, true), - block: lex.bind(null, false) -}; +module.exports = lex; diff --git a/lib/parse/matchRule.js b/lib/parse/matchRule.js new file mode 100644 index 00000000..f2036654 --- /dev/null +++ b/lib/parse/matchRule.js @@ -0,0 +1,30 @@ +var is = require('is'); +var Immutable = require('immutable'); + +var Token = require('../models/token'); + +/** + * Match a text using a rule + * @param {ParsingState} state + * @param {Rule} rule + * @param {String} text + * @return {List|null} + */ +function matchRule(state, rule, text) { + var matches = rule.onText(state, text); + var ruleType = rule.getType(); + + if (!matches) { + return; + } + if (!is.array(matches) && !Immutable.List.isList(matches)) { + matches = [matches]; + } + + return Immutable.List(matches) + .map(function(match) { + return Token.create(match.type || ruleType, match); + }); +} + +module.exports = matchRule; diff --git a/lib/parse/state.js b/lib/parse/state.js new file mode 100644 index 00000000..ffa40a1e --- /dev/null +++ b/lib/parse/state.js @@ -0,0 +1,123 @@ + +var STYLES = require('../constants/styles'); +var lex = require('./lex'); +var mergeTokens = require('./mergeTokens'); + +function ParsingState(syntax) { + if (!(this instanceof ParsingState)) { + return new ParsingState(syntax); + } + + this._ = {}; + this.depth = 0; + this.syntax = syntax; +} + +/** + * Get depth of parsing + * @return {Number} + */ +ParsingState.prototype.getDepth = function() { + return this.depth; +}; + +/** + * Get depth of parent token + * @return {Number} + */ +ParsingState.prototype.getParentDepth = function() { + return this.getDepth() - 1; +}; + +/** + * Get a state + * @param {String} key + * @return {Mixed} + */ +ParsingState.prototype.get = function(key) { + return this._[key]; +}; + +/** + * Get a state + * @param {String} key + * @param {Mixed} value + * @return {Mixed} + */ +ParsingState.prototype.set = function(key, value) { + this._[key] = value; + return this; +}; + +/** + * Toggle a state and execute the function + * @param {String} key + * @param {[type]} [varname] [description] + * @return {Mixed} + */ +ParsingState.prototype.toggle = function(key, value, fn) { + if (!fn) { + fn = value; + value = this.depth; + } + + var prevValue = this.get(key); + + this._[key] = value; + var result = fn(); + this._[key] = prevValue; + + return result; +}; + +/** + * Parse a text using a set of rules + * @param {RulesSet} rules + * @param {Boolean} isInline + * @param {String} text + * @return {List} + */ +ParsingState.prototype.parse = function(rulesSet, isInline, text) { + this.depth++; + + var rules = rulesSet.getRules(); + var tokens = lex(this, rules, isInline, text); + + if (isInline) { + tokens = mergeTokens(tokens, [ + STYLES.TEXT + ]); + } + + this.depth--; + + return tokens; +}; + +/** + * Parse a text using inline rules + * @param {String} text + * @return {List} + */ +ParsingState.prototype.parseAsInline = function(text) { + return this.parse( + this.syntax.getInlineRulesSet(), + true, + text + ); +}; + +/** + * Parse a text using inline rules + * @param {String} text + * @return {List} + */ +ParsingState.prototype.parseAsBlock = function(text) { + return this.parse( + this.syntax.getBlockRulesSet(), + false, + text + ); +}; + +module.exports = ParsingState; diff --git a/lib/parse/textToUnstyledTokens.js b/lib/parse/textToUnstyledTokens.js new file mode 100644 index 00000000..97b9c6e8 --- /dev/null +++ b/lib/parse/textToUnstyledTokens.js @@ -0,0 +1,61 @@ +var Immutable = require('immutable'); + +var defaultRules = require('../constants/defaultRules'); +var matchRule = require('./matchRule'); + +/** + * Create a text token inline or block + * + * @param {ParsingState} state + * @param {Boolean} isInline + * @param {String} text + * @return {Token} + */ +function createTextToken(state, isInline, text) { + var rule = isInline? defaultRules.inlineRule : defaultRules.blockRule; + return matchRule(state, rule, text).get(0); +} + +/** + * Convert a normal text into a list of unstyled tokens (block or inline) + * + * @param {ParsingState} state + * @param {Boolean} isInline + * @param {String} text + * @return {List} + */ +function textToUnstyledTokens(state, isInline, text) { + if (!text) { + return Immutable.List(); + } + + var accu = '', c, wasNewLine = false; + var result = []; + + function pushAccu() { + var isEmpty = !(accu.trim()) + var token = createTextToken(state, isInline, accu); + accu = ''; + + if (!isEmpty) { + result.push(token); + } + } + + for (var i = 0; i < text.length; i++) { + c = text[i]; + + if (c !== '\n' && wasNewLine) { + pushAccu(); + } + + accu += c; + wasNewLine = (c === '\n'); + } + + pushAccu(); + + return new Immutable.List(result); +} + +module.exports = textToUnstyledTokens; diff --git a/lib/prosemirror/__tests__/specs.js b/lib/prosemirror/__tests__/specs.js new file mode 100644 index 00000000..5f7a224d --- /dev/null +++ b/lib/prosemirror/__tests__/specs.js @@ -0,0 +1,32 @@ +var fs = require('fs'); +var path = require('path'); + +var encode = require('../encode'); +var JSONUtils = require('../../json'); + +var FIXTURES = path.resolve(__dirname, 'specs'); + +var files = fs.readdirSync(FIXTURES); + +describe('encode', function() { + files.forEach(function(file) { + if (path.extname(file) !== '.js') return; + + it(file, function () { + var content = require(path.join(FIXTURES, file)); + var contentState = JSONUtils.decode(content.json); + + encode(contentState).should.deepEqual(content.prosemirror); + }); + }); +}); + +describe('decode', function() { + files.forEach(function(file) { + if (path.extname(file) !== '.md') return; + + it(file, function () { + + }); + }); +}); diff --git a/lib/prosemirror/__tests__/specs/codeBlocks.js b/lib/prosemirror/__tests__/specs/codeBlocks.js new file mode 100644 index 00000000..30b38aa3 --- /dev/null +++ b/lib/prosemirror/__tests__/specs/codeBlocks.js @@ -0,0 +1,69 @@ +module.exports = { + json: { + 'syntax': 'markdown', + 'token': { + 'type': 'doc', + 'data': {}, + 'tokens': [ + { + 'type': 'paragraph', + 'data': {}, + 'tokens': [ + { + 'type': 'text', + 'text': 'Hello World', + 'data': {}, + 'tokens': [] + } + ] + }, + { + 'type': 'code_block', + 'data': { + 'syntax': 'js' + }, + 'tokens': [ + { + 'type': 'text', + 'text': 'var a = \'test\'\n', + 'data': {}, + 'tokens': [] + } + ] + } + ] + } + }, + prosemirror: { + 'type': 'doc', + 'attrs': {}, + 'content': [ + { + 'type': 'paragraph', + 'attrs': {}, + 'content': [ + { + 'type': 'text', + 'text': 'Hello World', + 'attrs': {}, + 'marks': [] + } + ] + }, + { + 'type': 'code_block', + 'attrs': { + 'syntax': 'js' + }, + 'content': [ + { + 'type': 'text', + 'text': 'var a = \'test\'\n', + 'attrs': {}, + 'marks': [] + } + ] + } + ] + } +}; \ No newline at end of file diff --git a/lib/prosemirror/__tests__/specs/imageAndStyles.js b/lib/prosemirror/__tests__/specs/imageAndStyles.js new file mode 100644 index 00000000..d1382cef --- /dev/null +++ b/lib/prosemirror/__tests__/specs/imageAndStyles.js @@ -0,0 +1,216 @@ +module.exports = { + prosemirror: { + 'type': 'doc', + 'attrs': {}, + 'content': [ + { + 'type': 'header_one', + 'attrs': { + 'id': null + }, + 'content': [ + { + 'type': 'text', + 'text': 'Hello', + 'attrs': {}, + 'marks': [] + } + ] + }, + { + 'type': 'paragraph', + 'attrs': {}, + 'content': [ + { + 'type': 'text', + 'text': 'Hello ', + 'attrs': {}, + 'marks': [] + }, + { + 'type': 'text', + 'text': 'Wo', + 'attrs': {}, + 'marks': [ + { + 'href': 'a.md', + '_': 'link' + }, + { + '_': 'BOLD' + } + ] + }, + { + 'type': 'text', + 'text': 'rld', + 'attrs': {}, + 'marks': [ + { + 'href': 'a.md', + '_': 'link' + }, + { + '_': 'BOLD' + }, + { + '_': 'ITALIC' + } + ] + }, + { + 'type': 'text', + 'text': ' ', + 'attrs': {}, + 'marks': [] + }, + { + 'type': 'text', + 'text': ' ', + 'attrs': {}, + 'marks': [ + { + '_': 'BOLD' + } + ] + }, + { + 'type': 'image', + 'text': '', + 'attrs': { + 'alt': '', + 'src': 'test.png' + }, + 'marks': [ + { + '_': 'BOLD' + } + ] + }, + { + 'type': 'text', + 'text': ' ', + 'attrs': {}, + 'marks': [ + { + '_': 'BOLD' + } + ] + }, + { + 'type': 'text', + 'text': '.', + 'attrs': {}, + 'marks': [] + } + ] + } + ] + }, + json: { + 'syntax': 'prosemirror', + 'token': { + 'type': 'doc', + 'data': {}, + 'tokens': [ + { + 'type': 'header_one', + 'data': { + 'id': null + }, + 'tokens': [ + { + 'type': 'text', + 'text': 'Hello', + 'data': {}, + 'tokens': [] + } + ] + }, + { + 'type': 'paragraph', + 'data': {}, + 'tokens': [ + { + 'type': 'text', + 'text': 'Hello ', + 'data': {}, + 'tokens': [] + }, + { + 'type': 'link', + 'data': { + 'href': 'a.md' + }, + 'tokens': [ + { + 'type': 'BOLD', + 'data': {}, + 'tokens': [ + { + 'type': 'text', + 'text': 'Wo', + 'data': {}, + 'tokens': [] + }, + { + 'type': 'ITALIC', + 'data': {}, + 'tokens': [ + { + 'type': 'text', + 'text': 'rld', + 'data': {}, + 'tokens': [] + } + ] + } + ] + } + ] + }, + { + 'type': 'text', + 'text': ' ', + 'data': {}, + 'tokens': [] + }, + { + 'type': 'BOLD', + 'data': {}, + 'tokens': [ + { + 'type': 'text', + 'text': ' ', + 'data': {}, + 'tokens': [] + }, + { + 'type': 'image', + 'text': '', + 'data': { + 'alt': '', + 'src': 'test.png' + }, + 'tokens': [] + }, + { + 'type': 'text', + 'text': ' ', + 'data': {}, + 'tokens': [] + } + ] + }, + { + 'type': 'text', + 'text': '.', + 'data': {}, + 'tokens': [] + } + ] + } + ] + } + } +}; diff --git a/lib/prosemirror/decode.js b/lib/prosemirror/decode.js new file mode 100644 index 00000000..e8f0f83c --- /dev/null +++ b/lib/prosemirror/decode.js @@ -0,0 +1,67 @@ +var Immutable = require('immutable'); + +var Content = require('../models/content'); +var Token = require('../models/token'); + +/** + * Decode marks as tokens + * + * @param {Array} marks + * @param {String} text + * @return {Token} + */ +function decodeMarksAsToken(marks, text) { + return marks.reduce(function(child, mark) { + return new Token({ + type: mark._, + data: Immutable.Map(mark).delete('_'), + tokens: [child] + }); + + }, Token.createText(text)); +} + +/** + * Decode a token + * + * @paran {Object} json + * @return {Token} + */ +function decodeTokenFromJSON(json) { + if (json.marks) { + return decodeMarksAsToken(json.marks, json.text); + } + + return new Token({ + type: json.type, + text: json.text, + raw: json.raw, + data: json.attrs, + tokens: decodeTokensFromJSON(json.content || []) + }); +} + +/** + * Decode a list of tokens + * + * @paran {Object} json + * @return {List} + */ +function decodeTokensFromJSON(json) { + return new Immutable.List(json.map(decodeTokenFromJSON)); +} + +/** + * Decode a JSON into a Content + * + * @paran {Object} json + * @return {Content} + */ +function decodeContentFromProseMirror(json) { + return Content.createFromToken( + 'prosemirror', + decodeTokenFromJSON(json) + ); +} + +module.exports = decodeContentFromProseMirror; diff --git a/lib/prosemirror/encode.js b/lib/prosemirror/encode.js new file mode 100644 index 00000000..75a0bada --- /dev/null +++ b/lib/prosemirror/encode.js @@ -0,0 +1,122 @@ +var Immutable = require('immutable'); + +var BLOCKS = require('../constants/blocks'); +var Token = require('../models/token'); +var MARK_TYPES = require('./markTypes'); + +/** + * Encode data of a token, it ignores undefined value + * + * @paran {Map} data + * @return {Object} + */ +function encodeDataToAttrs(data) { + return data + .filter(function(value, key) { + return (value !== undefined); + }) + .toJS(); +} + +/** + * Encode token as a mark + * @param {Token} token + * @return {Object} + */ +function tokenToMark(token) { + var data = token.getData(); + + return data + .merge({ + '_': token.getType() + }) + .toJS(); +} + +/** + * Encode an inline token + * @paran {Token} tokens + * @param {List} + * @return {Array} + */ +function encodeInlineTokenToJSON(token, marks) { + marks = marks || Immutable.List(); + + if (!MARK_TYPES.includes(token.getType())) { + return [ + { + type: token.getType(), + text: token.getText(), + attrs: encodeDataToAttrs(token.getData()), + marks: marks.toJS() + } + ]; + } + + var mark = tokenToMark(token); + var innerMarks = marks.push(mark); + var tokens = token.getTokens(); + + return tokens + .reduce(function(accu, token) { + return accu.concat(encodeInlineTokenToJSON(token, innerMarks)); + }, []); +} + +/** + * Encode a block token + * @paran {Token} tokens + * @return {Object} + */ +function encodeBlockTokenToJSON(token) { + return { + type: token.getType(), + attrs: encodeDataToAttrs(token.getData()), + content: encodeTokensToJSON(token.getTokens()) + }; +} + +/** + * Encode a token to JSON + * + * @paran {Token} tokens + * @return {Array} + */ +function encodeTokenToJSON(token) { + if (token.isInline()) { + return encodeInlineTokenToJSON(token); + } else { + return [encodeBlockTokenToJSON(token)]; + } +} + +/** + * Encode a list of tokens to JSON + * + * @paran {List} tokens + * @return {Array} + */ +function encodeTokensToJSON(tokens) { + return tokens.reduce(function(accu, token) { + return accu.concat(encodeTokenToJSON(token)); + }, []); +} + +/** + * Encode a Content into a ProseMirror JSON object + * + * @paran {Content} content + * @return {Object} + */ +function encodeContentToProseMirror(content) { + var doc = encodeTokenToJSON(content.getToken())[0]; + + // ProseMirror crash with empty doc node + if (doc.content.length === 0) { + doc.content = encodeTokenToJSON(Token.create(BLOCKS.PARAGRAPH)); + } + + return doc; +} + +module.exports = encodeContentToProseMirror; diff --git a/lib/prosemirror/index.js b/lib/prosemirror/index.js new file mode 100644 index 00000000..22bb0fda --- /dev/null +++ b/lib/prosemirror/index.js @@ -0,0 +1,5 @@ + +module.exports = { + encode: require('./encode'), + decode: require('./decode') +}; diff --git a/lib/prosemirror/markTypes.js b/lib/prosemirror/markTypes.js new file mode 100644 index 00000000..b7f8a255 --- /dev/null +++ b/lib/prosemirror/markTypes.js @@ -0,0 +1,13 @@ +var Immutable = require('immutable'); + +var STYLES = require('../constants/styles'); +var ENTITIES = require('../constants/entities'); + +module.exports = Immutable.List([ + STYLES.BOLD, + STYLES.ITALIC, + STYLES.CODE, + STYLES.STRIKETHROUGH, + ENTITIES.LINK +]); + diff --git a/lib/render/index.js b/lib/render/index.js index 9fcb3f9f..73f1a5f3 100644 --- a/lib/render/index.js +++ b/lib/render/index.js @@ -1,14 +1,4 @@ -/** - * Return context to describe a token - */ -function getTokenCtx(token) { - return { - type: token.getType(), - text: token.getText(), - raw: token.getRaw(), - data: token.getData().toJS() - }; -} +var RenderingState = require('./state'); /** * Render a Content instance using a syntax @@ -16,45 +6,11 @@ function getTokenCtx(token) { * @return {String} */ function render(syntax, content) { - var ctx = {}; - - function _renderTokens(tokens, depth) { - depth = depth || 0; - - return tokens.map(function(token, i) { - var tokenType = token.getType(); - var innerTokens = token.getTokens(); - var rule, renderInner, innerText; - - if (token.isInline()) { - rule = syntax.getInlineRule(tokenType); - } else { - rule = syntax.getBlockRule(tokenType); - } - - if (rule.getOption('renderInner') && innerTokens.size > 0) { - renderInner = true; - } - - if (renderInner) { - innerText = _renderTokens(innerTokens, depth + 1); - } else { - innerText = token.getText(); - } - - // Create context to describe a token - var prevToken = i > 0? tokens.get(i - 1) : null; - var nextToken = i < (tokens.size - 1)? tokens.get(i + 1) : null; - - var tokenCtx = getTokenCtx(token); - tokenCtx.next = nextToken? getTokenCtx(nextToken) : null; - tokenCtx.prev = prevToken? getTokenCtx(prevToken) : null; - - return rule.onToken(ctx, innerText, tokenCtx); - }).join(''); - } + var state = new RenderingState(syntax); + var entryRule = syntax.getEntryRule(); + var token = content.getToken(); - return _renderTokens(content.getTokens()); + return entryRule.onToken(state, token); } module.exports = render; diff --git a/lib/render/state.js b/lib/render/state.js new file mode 100644 index 00000000..3208c763 --- /dev/null +++ b/lib/render/state.js @@ -0,0 +1,52 @@ +var Token = require('../models/token'); + +function RenderingState(syntax) { + if (!(this instanceof RenderingState)) { + return new RenderingState(syntax); + } + + this.syntax = syntax; +} + +/** + * Render a token using a set of rules + * @param {RulesSet} rules + * @param {Boolean} isInline + * @param {Token|List} tokens + * @return {List} + */ +RenderingState.prototype.render = function(tokens) { + var state = this; + var syntax = this.syntax; + + if (tokens instanceof Token) { + var token = tokens; + tokens = token.getTokens(); + + if (tokens.size === 0) { + return token.getAsPlainText(); + } + } + + return tokens.reduce(function(text, token) { + var tokenType = token.getType(); + var rule = (token.isInline()? syntax.getInlineRule(tokenType) + : syntax.getBlockRule(tokenType)); + + if (!rule) { + throw new Error('Unexpected error: no rule to render "' + tokenType + '"'); + } + + return text + rule.onToken(state, token); + }, ''); +}; + +RenderingState.prototype.renderAsInline = function(tokens) { + return this.render(tokens); +}; + +RenderingState.prototype.renderAsBlock = function(tokens) { + return this.render(tokens); +}; + +module.exports = RenderingState; diff --git a/lib/utils/getText.js b/lib/utils/getText.js deleted file mode 100644 index b71c0dbd..00000000 --- a/lib/utils/getText.js +++ /dev/null @@ -1,19 +0,0 @@ - -/** - * Extract plain text from a token. - * - * @param {Token} - * @return {String} - */ -function getText(token) { - var tokens = token.getTokens(); - if (tokens.size === 0) { - return token.getText(); - } - - return tokens.reduce(function(result, tok) { - return result + tok.getText(); - }, ''); -} - -module.exports = getText; diff --git a/lib/utils/transform.js b/lib/utils/transform.js index a1f86cbb..63113ad3 100644 --- a/lib/utils/transform.js +++ b/lib/utils/transform.js @@ -1,25 +1,62 @@ +var Immutable = require('immutable'); var Content = require('../models/content'); /** * Walk throught the children tokens tree, and * map each token using a transformation * - * @param {Token} base + * The transformation iterator can return a list, a new token or undefined. + * + * @param {Token|Content} base * @param {Function(token, depth)} iter * @param {Number} depth * @return {Token} */ -function transform(base, iter, depth) { +function transformToken(base, iter, depth) { depth = depth || 0; var tokens = base.getTokens(); + var newTokens = transformTokens(tokens, iter, depth); + base = base.set('tokens', newTokens); - tokens = tokens.map(function(token) { - return transform(token, iter, depth + 1); - }); + return (base instanceof Content)? base : iter(base, depth); +} - base = base.set('tokens', tokens); +/** + * Transform a list of tokens + * @param {List} tokens + * @param {Function} iter + * @param {Number} depth + * @return {List} + */ +function transformTokens(tokens, iter, depth) { + return tokens + .reduce(function(list, token) { + var result = transform(token, iter, depth + 1); - return (base instanceof Content)? base : iter(base, depth); + if (Immutable.List.isList(result)) { + return list.concat(result); + } + else if (result) { + return list.push(result); + } + + return list; + }, Immutable.List()); +} + + +/** + * Transform that works on token or list of tokens + * @param {Token|List|Content} base + * @param {Function} iter + * @return {Token|List|Content} + */ +function transform(base, iter) { + if (Immutable.List.isList(base)) { + return transformTokens(base, iter); + } else { + return transformToken(base, iter); + } } module.exports = transform; diff --git a/lib/utils/walk.js b/lib/utils/walk.js index e6e0acd8..709766e2 100644 --- a/lib/utils/walk.js +++ b/lib/utils/walk.js @@ -15,6 +15,8 @@ function walk(base, iter) { var tokens = base.getTokens(); if (tokens.size === 0) { + iter(base, Range(0, base.getText().length)); + return base.getText(); } diff --git a/package.json b/package.json index a4006c85..c8342a51 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "scripts": { "lint": "eslint .", - "test": "./node_modules/.bin/mocha ./testing/setup.js \"./lib/**/*/__tests__/*.js\" \"./lib/__tests__/*.js\" \"./syntaxes/**/*/__tests__/*.js\" --reporter=list --timeout=10000" + "test": "./node_modules/.bin/mocha ./testing/setup.js \"./lib/**/*/__tests__/*.js\" \"./lib/__tests__/*.js\" \"./syntaxes/**/*/__tests__/*.js\" --bail --reporter=list --timeout=10000" }, "repository": { "type": "git", diff --git a/syntaxes/gitbook/__tests__/index.js b/syntaxes/gitbook/__tests__/index.js index d0668169..70600067 100644 --- a/syntaxes/gitbook/__tests__/index.js +++ b/syntaxes/gitbook/__tests__/index.js @@ -7,29 +7,32 @@ describe('GitBook Markdown', function() { describe('Math', function() { it('should parse a block', function() { var content = markup.toContent('$$\na = b\n$$'); - var blocks = content.getTokens(); + var math = content.getToken().getTokens().get(0); - blocks.size.should.equal(1); - blocks.get(0).getText().should.equal('a = b'); - blocks.get(0).getType().should.equal('math'); + math.getData().get('tex').should.equal('a = b'); + math.getType().should.equal(MarkupIt.BLOCKS.MATH); }); it('should parse inline math', function() { var content = markup.toContent('$$a = b$$'); - var blocks = content.getTokens(); + var p = content.getToken().getTokens().get(0); - blocks.size.should.equal(1); - blocks.get(0).getText().should.equal('a = b'); - blocks.get(0).getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH); + p.getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH); + + var math = p.getTokens().get(0); + math.getData().get('tex').should.equal('a = b'); + math.getType().should.equal(MarkupIt.ENTITIES.MATH); }); it('should parse inline math with text', function() { var content = markup.toContent('Here are some math $$a = b$$, awesome!'); - var blocks = content.getTokens(); + var p = content.getToken().getTokens().get(0); + + p.getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH); - blocks.size.should.equal(1); - blocks.get(0).getText().should.equal('Here are some math a = b, awesome!'); - blocks.get(0).getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH); + var math = p.getTokens().get(1); + math.getData().get('tex').should.equal('a = b'); + math.getType().should.equal(MarkupIt.ENTITIES.MATH); }); }); }); diff --git a/syntaxes/gitbook/index.js b/syntaxes/gitbook/index.js index 2bcaa4d1..117036c5 100644 --- a/syntaxes/gitbook/index.js +++ b/syntaxes/gitbook/index.js @@ -6,39 +6,41 @@ var reMathBlock = /^\$\$\n([^$]+)\n\$\$/; var reTpl = /^{([#%{])\s*(.*?)\s*(?=[#%}]})}}/; var inlineMathRule = markup.Rule(markup.ENTITIES.MATH) - .setOption('parse', false) - .setOption('renderInner', false) - .regExp(reMathInline, function(match) { + .regExp(reMathInline, function(state, match) { var text = match[1]; - if (text.trim().length == 0) return; + if (text.trim().length == 0) { + return; + } return { - text: text, - data: {} + data: { + tex: text + } }; }) - .toText(function(text, block) { - return '$$' + text + '$$'; + .toText(function(state, token) { + return '$$' + token.getData().get('tex') + '$$'; }); var blockMathRule = markup.Rule(markup.BLOCKS.MATH) - .setOption('parse', false) - .setOption('renderInner', false) - .regExp(reMathBlock, function(match) { + .regExp(reMathBlock, function(state, match) { var text = match[1]; - if (text.trim().length == 0) return; + if (text.trim().length == 0) { + return; + } return { - text: text + data: { + tex: text + } }; }) - .toText(function(text, block) { - return '$$\n' + text + '\n$$\n\n'; + .toText(function(state, token) { + return '$$\n' + token.getData().get('tex') + '\n$$\n\n'; }); var tplExpr = markup.Rule(markup.STYLES.TEMPLATE) - .setOption('parse', false) - .regExp(reTpl, function(match) { + .regExp(reTpl, function(state, match) { var type = match[0]; var text = match[2]; @@ -47,14 +49,18 @@ var tplExpr = markup.Rule(markup.STYLES.TEMPLATE) else if (type == '{') type = 'var'; return { - text: text, data: { type: type - } + }, + tokens: [ + MarkupIt.Token.createText(text) + ] }; }) - .toText(function(text, block) { - var type = block.data.type; + .toText(function(sttae, token) { + var data = token.getData(); + var text = token.getAsPlainText(); + var type = data.get('type'); if (type == 'expr') text = '{% ' + text + ' %}'; else if (type == 'comment') text = '{# ' + text + ' #}'; diff --git a/syntaxes/html/blocks.js b/syntaxes/html/blocks.js index 25dcd17e..583d460f 100644 --- a/syntaxes/html/blocks.js +++ b/syntaxes/html/blocks.js @@ -7,17 +7,15 @@ var listRules = require('./list'); var utils = require('./utils'); /* - Generate an heading rule for a specific level -*/ + * Generate an heading rule for a specific level + */ function headingRule(n) { var type = MarkupIt.BLOCKS['HEADING_' + n]; - return HTMLRule(type, 'h'+n); + return HTMLRule(type, 'h' + n); } module.exports = [ tableRules.block, - tableRules.header, - tableRules.body, tableRules.row, tableRules.cell, @@ -36,8 +34,10 @@ module.exports = [ HTMLRule(MarkupIt.BLOCKS.BLOCKQUOTE, 'blockquote'), MarkupIt.Rule(MarkupIt.BLOCKS.FOOTNOTE) - .toText(function(text, token) { - var refname = token.data.id; + .toText(function(state, token) { + var data = token.getData(); + var text = state.renderAsInline(token); + var refname = data.get('id'); return '
\n' + '' + refname + '. ' @@ -50,11 +50,14 @@ module.exports = [ .toText('%s\n\n'), MarkupIt.Rule(MarkupIt.BLOCKS.CODE) - .toText(function(text, token) { - var attr = ''; + .toText(function(state, token) { + var attr = ''; + var data = token.getData(); + var text = token.getAsPlainText(); + var syntax = data.get('syntax'); - if (token.data.syntax) { - attr = ' class="lang-' + token.data.syntax +'"'; + if (syntax) { + attr = ' class="lang-' + syntax +'"'; } return '
' + utils.escape(text) + '
\n'; diff --git a/syntaxes/html/index.js b/syntaxes/html/index.js index 122041b0..f20d4577 100644 --- a/syntaxes/html/index.js +++ b/syntaxes/html/index.js @@ -1,6 +1,20 @@ -var markup = require('../../'); +var MarkupIt = require('../../'); +var htmlToTokens = require('./parse'); + +var documentRule = MarkupIt.Rule(MarkupIt.BLOCKS.DOCUMENT) + .match(function(state, text) { + return { + tokens: htmlToTokens(text) + }; + }) + .toText(function(state, token) { + return state.renderAsBlock(token); + }); + + +module.exports = MarkupIt.Syntax('html', { + entryRule: documentRule, -module.exports = markup.Syntax('html', { // List of rules for parsing blocks inline: require('./inline'), diff --git a/syntaxes/html/inline.js b/syntaxes/html/inline.js index 67ff100a..77ae0e6e 100644 --- a/syntaxes/html/inline.js +++ b/syntaxes/html/inline.js @@ -6,13 +6,14 @@ var HTMLRule = require('./rule'); module.exports = [ // ---- TEXT ---- markup.Rule(markup.STYLES.TEXT) - .setOption('parse', false) - .toText(utils.escape), + .toText(function(state, token) { + return utils.escape(token.getAsPlainText()); + }), // ---- CODE ---- markup.Rule(markup.STYLES.CODE) - .toText(function(text, token) { - return '' + utils.escape(text) + ''; + .toText(function(state, token) { + return '' + utils.escape(token.getAsPlainText()) + ''; }), // ---- BOLD ---- @@ -31,19 +32,20 @@ module.exports = [ HTMLRule(markup.ENTITIES.LINK, 'a', function(data) { return { title: data.title? utils.escape(data.title) : undefined, - href: utils.escape(data.href || '') + href: utils.escape(data.href || '') }; }), // ---- FOOTNOTE ---- markup.Rule(markup.ENTITIES.FOOTNOTE_REF) - .toText(function(refname, token) { + .toText(function(state, token) { + var refname = token.getAsPlainText(); return '' + refname + ''; }), // ---- HTML ---- markup.Rule(markup.STYLES.HTML) - .toText(function(text, token) { - return text; + .toText(function(state, token) { + return token.getAsPlainText(); }) ]; diff --git a/syntaxes/html/list.js b/syntaxes/html/list.js index 2caea678..81e217da 100644 --- a/syntaxes/html/list.js +++ b/syntaxes/html/list.js @@ -1,43 +1,28 @@ var MarkupIt = require('../../'); -/* - Render a list item - - @param {String} text - @param {Token} token - @return {String} -*/ -function renderListItem(text, token) { - var isOrdered = token.type == MarkupIt.BLOCKS.OL_ITEM; - var listTag = isOrdered? 'ol' : 'ul'; - var depth = token.data.depth; - - var prevToken = token.prev? token.prev.type : null; - var nextToken = token.next? token.next.type : null; - - var prevTokenDepth = token.prev? token.prev.data.depth : 0; - var nextTokenDepth = token.next? token.next.data.depth : 0; - - var output = ''; - - if (prevToken != token.type || prevTokenDepth < depth) { - output += '<' + listTag + '>\n'; - } - - output += '
  • ' + text + '
  • \n'; - - if (nextToken != token.type || nextTokenDepth < depth) { - output += '\n'; - } - - return output; +/** + * Render a list item + * + * @param {String} text + * @param {Token} token + * @return {String} + */ +function renderListItem(state, token) { + var isOrdered = (token.type == MarkupIt.BLOCKS.OL_LIST); + var listTag = isOrdered? 'ol' : 'ul'; + var items = token.getTokens(); + + return '<' + listTag + '>' + + items.map(function(item) { + return '
  • ' + state.render(item) + '
  • '; + }).join('\n') + + '\n'; } - -var ruleOL = MarkupIt.Rule(MarkupIt.BLOCKS.OL_ITEM) +var ruleOL = MarkupIt.Rule(MarkupIt.BLOCKS.OL_LIST) .toText(renderListItem); -var ruleUL = MarkupIt.Rule(MarkupIt.BLOCKS.UL_ITEM) +var ruleUL = MarkupIt.Rule(MarkupIt.BLOCKS.UL_LIST) .toText(renderListItem); module.exports = { diff --git a/syntaxes/html/parse.js b/syntaxes/html/parse.js index f7fbf81c..1442646d 100644 --- a/syntaxes/html/parse.js +++ b/syntaxes/html/parse.js @@ -1,38 +1,37 @@ var Immutable = require('immutable'); var htmlparser = require('htmlparser2'); -var markup = require('../../'); +var MarkupIt = require('../../'); var TAGS_TO_TYPE = { - a: markup.ENTITIES.LINK, - img: markup.ENTITIES.IMAGE, + a: MarkupIt.ENTITIES.LINK, + img: MarkupIt.ENTITIES.IMAGE, - h1: markup.BLOCKS.HEADING_1, - h2: markup.BLOCKS.HEADING_2, - h3: markup.BLOCKS.HEADING_3, - h4: markup.BLOCKS.HEADING_4, - h5: markup.BLOCKS.HEADING_5, - h6: markup.BLOCKS.HEADING_6, - pre: markup.BLOCKS.CODE, - blockquote: markup.BLOCKS.BLOCKQUOTE, - p: markup.BLOCKS.PARAGRAPH, - hr: markup.BLOCKS.HR, + h1: MarkupIt.BLOCKS.HEADING_1, + h2: MarkupIt.BLOCKS.HEADING_2, + h3: MarkupIt.BLOCKS.HEADING_3, + h4: MarkupIt.BLOCKS.HEADING_4, + h5: MarkupIt.BLOCKS.HEADING_5, + h6: MarkupIt.BLOCKS.HEADING_6, + pre: MarkupIt.BLOCKS.CODE, + blockquote: MarkupIt.BLOCKS.BLOCKQUOTE, + p: MarkupIt.BLOCKS.PARAGRAPH, + hr: MarkupIt.BLOCKS.HR, - table: markup.BLOCKS.TABLE, - tr: markup.BLOCKS.TR_ROW, - td: markup.BLOCKS.TABLE_CELL, + table: MarkupIt.BLOCKS.TABLE, + tr: MarkupIt.BLOCKS.TR_ROW, + td: MarkupIt.BLOCKS.TABLE_CELL, - b: markup.INLINES.BOLD, - strike: markup.INLINES.STRIKETHROUGH, - em: markup.INLINES.ITALIC, - code: markup.INLINES.CODE + b: MarkupIt.STYLES.BOLD, + strike: MarkupIt.STYLES.STRIKETHROUGH, + em: MarkupIt.STYLES.ITALIC, + code: MarkupIt.STYLES.CODE }; /** - Parse an HTML string into a token tree - - @param {String} str - @return {List} -*/ + * Parse an HTML string into a token tree + * @param {String} str + * @return {List} + */ function htmlToTokens(str) { var accuText = ''; var result = []; @@ -50,8 +49,7 @@ function htmlToTokens(str) { return; } - var token = new markup.Token({ - type: type, + var token = MarkupIt.Token.create(type, { text: accuText }); diff --git a/syntaxes/html/rule.js b/syntaxes/html/rule.js index 4506cbbf..fd248edc 100644 --- a/syntaxes/html/rule.js +++ b/syntaxes/html/rule.js @@ -1,15 +1,14 @@ var is = require('is'); -var markup = require('../../'); +var MarkupIt = require('../../'); var identity = require('../../lib/utils/identity'); var SINGLE_TAG = ['img', 'hr']; /** - Convert a map of attributes into a string - - @param {Object} attrs - @return {String} + * Convert a map of attributes into a string + * @param {Object} attrs + * @return {String} */ function attrsToString(attrs) { var output = '', value; @@ -25,20 +24,22 @@ function attrsToString(attrs) { } else { output += ' ' + key + '=' + JSON.stringify(value); } - - } return output; } + function HTMLRule(type, tag, getAttrs) { getAttrs = getAttrs || identity; var isSingleTag = SINGLE_TAG.indexOf(tag) >= 0; - return markup.Rule(type) - .toText(function(text, token) { - var attrs = getAttrs(token.data, token); + return MarkupIt.Rule(type) + .toText(function(state, token) { + var text = state.render(token); + var data = token.getData().toJS(); + var attrs = getAttrs(data, token); + var output = '<' + tag + attrsToString(attrs) + (isSingleTag? '/>' : '>'); if (!isSingleTag) { diff --git a/syntaxes/html/table.js b/syntaxes/html/table.js index 2e124a30..a2aec9bc 100644 --- a/syntaxes/html/table.js +++ b/syntaxes/html/table.js @@ -1,46 +1,54 @@ var MarkupIt = require('../../'); var blockRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE) - .toText(function(innerHTML) { - this._rowIndex = 0; + .toText(function(state, token) { + state._rowIndex = 0; - return '\n' + innerHTML + '
    \n\n'; - }); + var data = token.getData(); + var rows = token.getTokens(); + var align = data.get('align'); -var headerRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_HEADER) - .toText(function(innerHTML) { - return '\n' + innerHTML + ''; - }); + var headerRows = rows.slice(0, 1); + var bodyRows = rows.slice(1); + + state._tableAlign = align; -var bodyRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_BODY) - .toText(function(innerHTML) { - return '\n' + innerHTML + ''; + var headerText = state.render(headerRows); + var bodyText = state.render(bodyRows); + + return '\n' + + '\n' + headerText + '' + + '\n' + bodyText + '' + + '
    \n\n'; }); var rowRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_ROW) - .toText(function(innerHTML) { - this._rowIndex = (this._rowIndex || 0) + 1; + .toText(function(state, token) { + var innerContent = state.render(token); + state._rowIndex = state._rowIndex + 1; + state._cellIndex = 0; - return '' + innerHTML + ''; + return '' + innerContent + ''; }); var cellRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_CELL) - .toText(function(innerHTML, token) { - var isHeader = (this._rowIndex || 0) === 0; - var align = token.data.align; + .toText(function(state, token) { + var align = state._tableAlign[state._cellIndex]; + var isHeader = (state._rowIndex || 0) === 0; + var innerHTML = state.render(token); var type = isHeader ? 'th' : 'td'; var tag = align ? '<' + type + ' style="text-align:' + align + '">' : '<' + type + '>'; + state._cellIndex = state._cellIndex + 1; + return tag + innerHTML + '\n'; }); module.exports = { block: blockRule, - header: headerRule, - body: bodyRule, row: rowRule, cell: cellRule }; \ No newline at end of file diff --git a/syntaxes/html/utils.js b/syntaxes/html/utils.js index fb13927a..d1884c4e 100644 --- a/syntaxes/html/utils.js +++ b/syntaxes/html/utils.js @@ -1,14 +1,20 @@ var entities = require('html-entities'); var htmlEntities = new entities.AllHtmlEntities(); -// Escape markdown syntax -// We escape only basic XML entities +/** + * Escape all entities (HTML + XML) + * @param {String} str + * @return {String} + */ function escape(str) { return htmlEntities.encode(str); } -// Unescape markdown syntax -// We unescape all entities (HTML + XML) +/** + * Unescape all entities (HTML + XML) + * @param {String} str + * @return {String} + */ function unescape(str) { return htmlEntities.decode(str); } diff --git a/syntaxes/markdown/__tests__/specs.js b/syntaxes/markdown/__tests__/specs.js index a442c388..82d68e19 100644 --- a/syntaxes/markdown/__tests__/specs.js +++ b/syntaxes/markdown/__tests__/specs.js @@ -53,11 +53,10 @@ function testMdIdempotence(fixture) { backToMd = markdown.toText(content2); var content3 = markdown.toContent(backToMd); - var jsonContent1 = MarkupIt.JSONUtils.encode(content1); - var jsonContent2 = MarkupIt.JSONUtils.encode(content2); - var jsonContent3 = MarkupIt.JSONUtils.encode(content3); + var resultHtml2 = html.toText(content2); + var resultHtml3 = html.toText(content3); - jsonContent3.should.eql(jsonContent2); + (resultHtml2).should.be.html(resultHtml3); } function readFixture(filename) { diff --git a/syntaxes/markdown/__tests__/specs/blockquote_list_item.html b/syntaxes/markdown/__tests__/specs/blockquote_list_item.html index d149cbc0..f263f53c 100755 --- a/syntaxes/markdown/__tests__/specs/blockquote_list_item.html +++ b/syntaxes/markdown/__tests__/specs/blockquote_list_item.html @@ -1,4 +1,3 @@ -

    This fails in markdown.pl and upskirt:

      -
    • hello -
      world
    • -
    \ No newline at end of file +

    This fails in markdown.pl and upskirt:

    + +
    • hello

      world

    \ No newline at end of file diff --git a/syntaxes/markdown/__tests__/specs/def_blocks.html b/syntaxes/markdown/__tests__/specs/def_blocks.html index 49a7a750..4a5dc40e 100755 --- a/syntaxes/markdown/__tests__/specs/def_blocks.html +++ b/syntaxes/markdown/__tests__/specs/def_blocks.html @@ -13,6 +13,10 @@
    • hello
    • [3]: hello
    • +
    + + +
    • hello
    diff --git a/syntaxes/markdown/__tests__/specs/loose_lists.html b/syntaxes/markdown/__tests__/specs/loose_lists.html index 6ead82cb..1155c905 100755 --- a/syntaxes/markdown/__tests__/specs/loose_lists.html +++ b/syntaxes/markdown/__tests__/specs/loose_lists.html @@ -13,26 +13,50 @@
    • hello

      • world how

        are -you

      • today

    • hi
    • +you

    • today

  • hi
  • + + + +
    • hello

    • world

    • hi
    • +
    + + + +
    • hello
    • world

    • hi

    • +
    + + + +
    • hello
    • world

      +

      how

    • hi
    • +
    + + +
    • hello
    • world
    • how

      are

    • +
    + + + +
    • hello
    • world

    • how

      are

    • -
    + \ No newline at end of file diff --git a/syntaxes/markdown/blocks.js b/syntaxes/markdown/blocks.js index 2b913373..af5ff8bb 100644 --- a/syntaxes/markdown/blocks.js +++ b/syntaxes/markdown/blocks.js @@ -1,5 +1,5 @@ var reBlock = require('./re/block'); -var markup = require('../../'); +var MarkupIt = require('../../'); var heading = require('./heading'); var list = require('./list'); @@ -7,90 +7,71 @@ var code = require('./code'); var table = require('./table'); var utils = require('./utils'); -/** - * Is top block check that a paragraph can be parsed - * Paragraphs can exists only in loose list or blockquote. - * - * @param {List} parents - * @return {Boolean} - */ -function isTop(parents) { - return parents.find(function(token) { - var isBlockquote = (token.getType() === markup.BLOCKS.BLOCKQUOTE); - var isLooseList = (token.isListItem() && token.getData().get('loose')); - - return (!isBlockquote && !isLooseList); - }) === undefined; -} - -module.exports = markup.RulesSet([ +module.exports = MarkupIt.RulesSet([ // ---- CODE BLOCKS ---- code.block, // ---- FOOTNOTES ---- - markup.Rule(markup.BLOCKS.FOOTNOTE) - .regExp(reBlock.footnote, function(match) { + MarkupIt.Rule(MarkupIt.BLOCKS.FOOTNOTE) + .regExp(reBlock.footnote, function(state, match) { var text = match[2]; return { - text: text, + tokens: state.parseAsInline(text), data: { id: match[1] } }; }) - .toText(function(text, block) { - return '[^' + block.data.id + ']: ' + text + '\n\n'; + .toText(function(state, token) { + var data = token.getData(); + var id = data.get('id'); + var innerContent = state.renderAsInline(token); + + return '[^' + id + ']: ' + innerContent + '\n\n'; }), // ---- HEADING ---- - heading.rule(6), - heading.rule(5), - heading.rule(4), - heading.rule(3), - heading.rule(2), - heading.rule(1), - - heading.lrule(2), - heading.lrule(1), + heading(6), + heading(5), + heading(4), + heading(3), + heading(2), + heading(1), // ---- TABLE ---- table.block, - table.header, - table.body, table.row, table.cell, // ---- HR ---- - markup.Rule(markup.BLOCKS.HR) - .setOption('parse', false) - .setOption('renderInner', false) + MarkupIt.Rule(MarkupIt.BLOCKS.HR) .regExp(reBlock.hr, function() { - return { - text: '' - }; + return {}; }) .toText('---\n\n'), // ---- BLOCKQUOTE ---- - markup.Rule(markup.BLOCKS.BLOCKQUOTE) - .setOption('parse', 'block') - .regExp(reBlock.blockquote, function(match) { + MarkupIt.Rule(MarkupIt.BLOCKS.BLOCKQUOTE) + .regExp(reBlock.blockquote, function(state, match) { var inner = match[0].replace(/^ *> ?/gm, '').trim(); - return { - text: inner - }; + return state.toggle('blockquote', function() { + return { + tokens: state.parseAsBlock(inner) + }; + }); }) - .toText(function(text) { - var lines = utils.splitLines(text.trim()); + .toText(function(state, token) { + var innerContent = state.renderAsBlock(token); + var lines = utils.splitLines(innerContent.trim()); return lines - .map(function(line) { - return '> ' + line; - }) - .join('\n') + '\n\n'; + .map(function(line) { + return '> ' + line; + }) + .join('\n') + '\n\n'; }), // ---- LISTS ---- @@ -98,9 +79,8 @@ module.exports = markup.RulesSet([ list.ol, // ---- HTML ---- - markup.Rule(markup.BLOCKS.HTML) - .setOption('parse', false) - .regExp(reBlock.html, function(match) { + MarkupIt.Rule(MarkupIt.BLOCKS.HTML) + .regExp(reBlock.html, function(state, match) { return { text: match[0] }; @@ -108,50 +88,53 @@ module.exports = markup.RulesSet([ .toText('%s'), // ---- DEFINITION ---- - markup.Rule(markup.BLOCKS.DEFINITION) - .regExp(reBlock.def, function(match, parents) { - if (parents.size > 0) { - return null; + MarkupIt.Rule() + .regExp(reBlock.def, function(state, match) { + if (state.getDepth() > 1) { + return; } - var id = match[1].toLowerCase(); - var href = match[2]; + var id = match[1].toLowerCase(); + var href = match[2]; var title = match[3]; - this.refs = this.refs || {}; - this.refs[id] = { + state.refs = state.refs || {}; + state.refs[id] = { href: href, title: title }; return { - type: markup.BLOCKS.IGNORE + type: 'definition' }; }), // ---- PARAGRAPH ---- - markup.Rule(markup.BLOCKS.PARAGRAPH) - .regExp(reBlock.paragraph, function(match, parents) { - if (!isTop(parents)) { + MarkupIt.Rule(MarkupIt.BLOCKS.PARAGRAPH) + .regExp(reBlock.paragraph, function(state, match) { + var isInBlocquote = (state.get('blockquote') === state.getParentDepth()); + var isInLooseList = (state.get('looseList') === state.getParentDepth()); + var isTop = (state.getDepth() === 1); + + if (!isTop && !isInBlocquote && !isInLooseList) { return; } - var text = match[1].trim(); return { - text: text + tokens: state.parseAsInline(text) }; }) .toText('%s\n\n'), - // ---- PARAGRAPH ---- - markup.Rule(markup.BLOCKS.TEXT) - .regExp(reBlock.text, function(match, parents) { - // Top-level should never reach here. + // ---- TEXT ---- + // Top-level should never reach here. + MarkupIt.Rule(MarkupIt.BLOCKS.TEXT) + .regExp(reBlock.text, function(state, match) { var text = match[0]; return { - text: text + tokens: state.parseAsInline(text) }; }) .toText('%s\n') diff --git a/syntaxes/markdown/code.js b/syntaxes/markdown/code.js index 23d40b97..92f5280a 100644 --- a/syntaxes/markdown/code.js +++ b/syntaxes/markdown/code.js @@ -1,16 +1,17 @@ var reBlock = require('./re/block'); -var markup = require('../../'); +var MarkupIt = require('../../'); var utils = require('./utils'); // Rule for parsing code blocks -var blockRule = markup.Rule(markup.BLOCKS.CODE) - .setOption('parse', false) - .setOption('renderInner', false) - +var blockRule = MarkupIt.Rule(MarkupIt.BLOCKS.CODE) // Fences - .regExp(reBlock.fences, function(match) { + .regExp(reBlock.fences, function(state, match) { + var inner = match[3]; + return { - text: match[3], + tokens: [ + MarkupIt.Token.createText(inner) + ], data: { syntax: match[2] } @@ -18,7 +19,7 @@ var blockRule = markup.Rule(markup.BLOCKS.CODE) }) // 4 spaces / Tab - .regExp(reBlock.code, function(match) { + .regExp(reBlock.code, function(state, match) { var inner = match[0]; // Remove indentation @@ -28,7 +29,9 @@ var blockRule = markup.Rule(markup.BLOCKS.CODE) inner = inner.replace(/\n+$/, ''); return { - text: inner, + tokens: [ + MarkupIt.Token.createText(inner) + ], data: { syntax: undefined } @@ -36,13 +39,16 @@ var blockRule = markup.Rule(markup.BLOCKS.CODE) }) // Output code blocks - .toText(function(text, block) { + .toText(function(state, token) { + var text = token.getAsPlainText(); + var data = token.getData(); + var syntax = data.get('syntax') || ''; var hasFences = text.indexOf('`') >= 0; // Use fences if syntax is set - if (!hasFences) { + if (!hasFences || syntax) { return ( - '```' + (block.data.syntax || '') + '\n' + '```' + syntax + '\n' + text + '```\n\n' ); diff --git a/syntaxes/markdown/document.js b/syntaxes/markdown/document.js new file mode 100644 index 00000000..4051d5ab --- /dev/null +++ b/syntaxes/markdown/document.js @@ -0,0 +1,83 @@ +var Immutable = require('immutable'); +var MarkupIt = require('../../'); + +/** + * Cleanup a text before parsing: normalize newlines and tabs + * + * @param {String} src + * @return {String} + */ +function cleanupText(src) { + return src + .replace(/\r\n|\r/g, '\n') + .replace(/\t/g, ' ') + .replace(/\u00a0/g, ' ') + .replace(/\u2424/g, '\n') + .replace(/^ +$/gm, ''); +} + +/** + * Resolve definition links + * + * @param {ParsingState} state + * @param {Token} token + * @return {Token} + */ +function resolveLink(state, token) { + var tokenType = token.getType(); + var data = token.getData(); + + if (tokenType === 'definition') { + return false; + } + if (tokenType !== MarkupIt.ENTITIES.LINK) { + return token; + } + + // Normal link + if (!data.has('ref')) { + return token; + } + + // Resolve reference + var refs = (state.refs || {}); + var refId = data.get('ref') + .replace(/\s+/g, ' ') + .toLowerCase(); + var ref = refs[refId]; + + // Parse reference as text + if (!ref) { + var rawText = token.getRaw(); + + var tokens = Immutable.List([ + MarkupIt.Token.createText(rawText[0]) + ]) + .concat( + state.parseAsInline(rawText.slice(1)) + ); + + return MarkupIt.transform(tokens, resolveLink.bind(null, state)); + } + + // Update link attributes + return token.setData( + data.merge(ref) + ); +} + +var documentRule = MarkupIt.Rule(MarkupIt.BLOCKS.DOCUMENT) + .match(function(state, text) { + text = cleanupText(text); + + var token = MarkupIt.Token.create(MarkupIt.BLOCKS.DOCUMENT, { + tokens: state.parseAsBlock(text) + }); + + return MarkupIt.transform(token, resolveLink.bind(null, state)); + }) + .toText(function(state, token) { + return state.renderAsBlock(token); + }); + +module.exports = documentRule; diff --git a/syntaxes/markdown/heading.js b/syntaxes/markdown/heading.js index f6d5c78a..5b6119fb 100644 --- a/syntaxes/markdown/heading.js +++ b/syntaxes/markdown/heading.js @@ -1,8 +1,13 @@ var reHeading = require('./re/heading'); var markup = require('../../'); -// Parse inner text of header to extract ID entity -function parseHeadingText(text) { +/** + * Parse inner text of header to extract ID entity + * @param {ParsingState} state + * @param {String} text + * @return {TokenLike} + */ +function parseHeadingText(state, text) { var id, match; reHeading.id.lastIndex = 0; @@ -17,44 +22,54 @@ function parseHeadingText(text) { } return { - text: text, + tokens: state.parseAsInline(text), data: { id: id } }; } -// Generator for HEADING_X rules +/** + * Generator for HEADING_X rules + * @param {Number} level + * @return {Rule} + */ function headingRule(level) { var prefix = Array(level + 1).join('#'); return markup.Rule(markup.BLOCKS['HEADING_' + level]) - .regExp(reHeading.normal, function(match) { - if (match[1].length != level) return null; - return parseHeadingText(match[2]); - }) - .toText(function (text, block) { - if (block.data.id) { - text += ' {#' + block.data.id + '}'; + + // Normal heading like + .regExp(reHeading.normal, function(state, match) { + if (match[1].length != level) { + return; } - return prefix + ' ' + text + '\n\n'; - }); -} + return parseHeadingText(state, match[2]); + }) -// Generator for HEADING_X rules for line heading -// Since normal heading are listed first, onText is not required here -function lheadingRule(level) { - return markup.Rule(markup.BLOCKS['HEADING_' + level]) - .regExp(reHeading.line, function(match) { + // Line heading + .regExp(reHeading.line, function(state, match) { var matchLevel = (match[2] === '=')? 1 : 2; - if (matchLevel != level) return null; + if (matchLevel != level) { + return; + } - return parseHeadingText(match[1]); + return parseHeadingText(state, match[1]); + }) + + .toText(function (state, token) { + var data = token.getData(); + var innerContent = state.renderAsInline(token); + var id = data.get('id'); + + if (id) { + innerContent += ' {#' + id + '}'; + } + + return prefix + ' ' + innerContent + '\n\n'; }); } -module.exports = { - rule: headingRule, - lrule: lheadingRule -}; + +module.exports = headingRule; diff --git a/syntaxes/markdown/index.js b/syntaxes/markdown/index.js index 5aa40662..44dcf257 100644 --- a/syntaxes/markdown/index.js +++ b/syntaxes/markdown/index.js @@ -1,6 +1,8 @@ -var markup = require('../../'); +var MarkupIt = require('../../'); + +module.exports = MarkupIt.Syntax('markdown', { + entryRule: require('./document'), -module.exports = markup.Syntax('markdown', { // List of rules for parsing blocks inline: require('./inline'), diff --git a/syntaxes/markdown/inline.js b/syntaxes/markdown/inline.js index ad7ccc6a..1b1452f3 100644 --- a/syntaxes/markdown/inline.js +++ b/syntaxes/markdown/inline.js @@ -1,173 +1,152 @@ +var Immutable = require('immutable'); + var reInline = require('./re/inline'); -var markup = require('../../'); +var MarkupIt = require('../../'); var utils = require('./utils'); var isHTMLBlock = require('./isHTMLBlock'); -/** - * Test if we are parsing inside a link - * @param {List} parents - * @return {Boolean} - */ -function isInLink(parents, ctx) { - if (ctx.isLink) { - return true; - } - - return parents.find(function(tok) { - if (tok.getType() === markup.ENTITIES.LINK) { - return true; - } - - return false; - }) !== undefined; -} - -/** - * Resolve a reflink - * @param {Object} ctx - * @param {String} text - * @return {Object|null} - */ -function resolveRefLink(ctx, text) { - var refs = (ctx.refs || {}); - - // Normalize the refId - var refId = (text) - .replace(/\s+/g, ' ') - .toLowerCase(); - var ref = refs[refId]; - - return (ref && ref.href)? ref : null; -} - -var inlineRules = markup.RulesSet([ +var inlineRules = MarkupIt.RulesSet([ // ---- FOOTNOTE REFS ---- - markup.Rule(markup.ENTITIES.FOOTNOTE_REF) - .regExp(reInline.reffn, function(match) { + MarkupIt.Rule(MarkupIt.ENTITIES.FOOTNOTE_REF) + .regExp(reInline.reffn, function(state, match) { return { - text: match[1], - data: {} + tokens: [ + MarkupIt.Token.createText(match[1]) + ] }; }) - .toText(function(text) { - return '[^' + text + ']'; + .toText(function(state, token) { + return '[^' + token.getAsPlainText() + ']'; }), // ---- IMAGES ---- - markup.Rule(markup.ENTITIES.IMAGE) - .regExp(reInline.link, function(match) { + MarkupIt.Rule(MarkupIt.ENTITIES.IMAGE) + .regExp(reInline.link, function(state, match) { var isImage = match[0].charAt(0) === '!'; - if (!isImage) return null; + if (!isImage) { + return; + } return { - text: ' ', data: { alt: match[1], src: match[2] } }; }) - .toText(function(text, entity) { - return '![' + entity.data.alt + '](' + entity.data.src + ')'; + .toText(function(state, token) { + var data = token.getData(); + var alt = data.get('alt', ''); + var src = data.get('src', ''); + + return '![' + alt + '](' + src + ')'; }), // ---- LINK ---- - markup.Rule(markup.ENTITIES.LINK) - .regExp(reInline.link, function(match) { - return { - text: match[1], - data: { - href: match[2], - title: match[3] - } - }; + MarkupIt.Rule(MarkupIt.ENTITIES.LINK) + .regExp(reInline.link, function(state, match) { + return state.toggle('link', function() { + return { + tokens: state.parseAsInline(match[1]), + data: { + href: match[2], + title: match[3] + } + }; + }); }) - .regExp(reInline.autolink, function(match) { - return { - text: match[1], - data: { - href: match[1] - } - }; + .regExp(reInline.autolink, function(state, match) { + return state.toggle('link', function() { + return { + tokens: state.parseAsInline(match[1]), + data: { + href: match[1] + } + }; + }); }) - .regExp(reInline.url, function(match, parents) { - if (isInLink(parents, this)) { + .regExp(reInline.url, function(state, match, parents) { + if (state.get('link')) { return; } - var uri = match[1]; return { - text: uri, data: { href: uri - } + }, + tokens: [ + MarkupIt.Token.createText(uri) + ] }; }) - .toText(function(text, entity) { - var title = entity.data.title? ' "' + entity.data.title + '"' : ''; - return '[' + text + '](' + entity.data.href + title + ')'; - }), - - // ---- REF LINKS ---- - // Doesn't render, but match and resolve reference - markup.Rule(markup.ENTITIES.LINK_REF) - .regExp(reInline.reflink, function(match) { - var ref = resolveRefLink(this, (match[2] || match[1])); - - if (!ref) { - return null; - } - - return { - type: markup.ENTITIES.LINK, - text: match[1], - data: ref - }; + .regExp(reInline.reflink, function(state, match) { + var refId = (match[2] || match[1]); + var innerText = match[1]; + + return state.toggle('link', function() { + return { + type: MarkupIt.ENTITIES.LINK, + data: { + ref: refId + }, + tokens: [ + MarkupIt.Token.createText(innerText) + ] + }; + }); }) - .regExp(reInline.nolink, function(match) { - var ref = resolveRefLink(this, (match[2] || match[1])); - - if (!ref) { - return null; - } - - return { - type: markup.ENTITIES.LINK, - text: match[1], - data: ref - }; + .regExp(reInline.nolink, function(state, match) { + var refId = (match[2] || match[1]); + + return state.toggle('link', function() { + return { + type: MarkupIt.ENTITIES.LINK, + tokens: state.parseAsInline(match[1]), + data: { + ref: refId + } + }; + }); }) - .regExp(reInline.reffn, function(match) { - var ref = resolveRefLink(this, match[1]); - - if (!ref) { - return null; - } - - return { - text: match[1], - data: ref - }; + .regExp(reInline.reffn, function(state, match) { + var refId = match[1]; + + return state.toggle('link', function() { + return { + tokens: state.parseAsInline(match[1]), + data: { + ref: refId + } + }; + }); }) - .toText(function(text, entity) { - var title = entity.data.title? ' "' + entity.data.title + '"' : ''; - return '[' + text + '](' + entity.data.href + title + ')'; + .toText(function(state, token) { + var data = token.getData(); + var title = data.get('title'); + var href = data.get('href'); + var innerContent = state.renderAsInline(token); + title = title? ' "' + title + '"' : ''; + + return '[' + innerContent + '](' + href + title + ')'; }), // ---- CODE ---- - markup.Rule(markup.STYLES.CODE) - .setOption('parse', false) - .regExp(reInline.code, function(match) { + MarkupIt.Rule(MarkupIt.STYLES.CODE) + .regExp(reInline.code, function(state, match) { return { - text: match[2] + tokens: [ + MarkupIt.Token.createText(match[2]) + ] }; }) - .toText(function(text) { - // We need to find the right separator not present in the content + .toText(function(state, token) { var separator = '`'; - while(text.indexOf(separator) >= 0) { + var text = token.getAsPlainText(); + + // We need to find the right separator not present in the content + while (text.indexOf(separator) >= 0) { separator += '`'; } @@ -175,117 +154,102 @@ var inlineRules = markup.RulesSet([ }), // ---- BOLD ---- - markup.Rule(markup.STYLES.BOLD) - .regExp(reInline.strong, function(match) { + MarkupIt.Rule(MarkupIt.STYLES.BOLD) + .regExp(reInline.strong, function(state, match) { return { - text: match[2] || match[1] + tokens: state.parseAsInline(match[2] || match[1]) }; }) .toText('**%s**'), // ---- ITALIC ---- - markup.Rule(markup.STYLES.ITALIC) - .regExp(reInline.em, function(match) { + MarkupIt.Rule(MarkupIt.STYLES.ITALIC) + .regExp(reInline.em, function(state, match) { return { - text: match[2] || match[1] + tokens: state.parseAsInline(match[2] || match[1]) }; }) .toText('_%s_'), // ---- STRIKETHROUGH ---- - markup.Rule(markup.STYLES.STRIKETHROUGH) - .regExp(reInline.del, function(match) { + MarkupIt.Rule(MarkupIt.STYLES.STRIKETHROUGH) + .regExp(reInline.del, function(state, match) { return { - text: match[1] + tokens: state.parseAsInline(match[1]) }; }) .toText('~~%s~~'), // ---- HTML ---- - markup.Rule(markup.STYLES.HTML) - .setOption('parse', false) - .setOption('renderInline', false) - .regExp(reInline.html, function(match, parents) { - var tag = match[0]; - var tagName = match[1]; + MarkupIt.Rule(MarkupIt.STYLES.HTML) + .regExp(reInline.html, function(state, match) { + var tag = match[0]; + var tagName = match[1]; var innerText = match[2] || ''; - var startTag, endTag; + var innerTokens = []; if (innerText) { startTag = tag.substring(0, tag.indexOf(innerText)); - endTag = tag.substring(tag.indexOf(innerText) + innerText.length); + endTag = tag.substring(tag.indexOf(innerText) + innerText.length); } else { startTag = match[0]; - endTag = ''; + endTag = ''; } - - // todo: handle link tags - /*if (tagName === 'a' && innerText) { - - }*/ - - var innerTokens = []; - if (tagName && !isHTMLBlock(tagName) && innerText) { - var inlineSyntax = markup.Syntax('markdown+html', { - inline: inlineRules + var isLink = (tagName.toLowerCase() === 'a'); + + innerTokens = state.toggle(isLink? 'link' : 'html', function() { + return state.parseAsInline(innerText); }); - var oldIsLink = this.isLink; - this.isLink = this.isLink || (tagName.toLowerCase() === 'a'); - innerTokens = markup.parseInline(inlineSyntax, innerText, this) - .getTokens() - .toArray(); - this.isLink = oldIsLink; } else { innerTokens = [ { - type: markup.STYLES.HTML, + type: MarkupIt.STYLES.HTML, text: innerText, - raw: innerText + raw: innerText } ]; } - var result = []; - - result.push({ - type: markup.STYLES.HTML, - text: startTag, - raw: startTag - }); + var result = Immutable.List() + .push({ + type: MarkupIt.STYLES.HTML, + text: startTag, + raw: startTag + }); result = result.concat(innerTokens); if (endTag) { - result.push({ - type: markup.STYLES.HTML, + result = result.push({ + type: MarkupIt.STYLES.HTML, text: endTag, - raw: endTag + raw: endTag }); } return result; }) - .toText(function(text, token) { - return text; + .toText(function(state, token) { + return token.getAsPlainText(); }), // ---- ESCAPED ---- - markup.Rule(markup.STYLES.TEXT) - .setOption('parse', false) - .regExp(reInline.escape, function(match) { + MarkupIt.Rule(MarkupIt.STYLES.TEXT) + .regExp(reInline.escape, function(state, match) { return { text: match[1] }; }) - .regExp(reInline.text, function(match) { + .regExp(reInline.text, function(state, match) { return { text: utils.unescape(match[0]) }; }) - .toText(function(text) { + .toText(function(state, token) { + var text = token.getAsPlainText(); return utils.escape(text, false); }) ]); diff --git a/syntaxes/markdown/isHTMLBlock.js b/syntaxes/markdown/isHTMLBlock.js index a53c31e3..9c8df072 100644 --- a/syntaxes/markdown/isHTMLBlock.js +++ b/syntaxes/markdown/isHTMLBlock.js @@ -68,11 +68,10 @@ var htmlBlocks = Immutable.List([ ]); /** - Test if a tag name is a valid HTML block - - @param {String} tag - @return {Boolean} -*/ + * Test if a tag name is a valid HTML block + * @param {String} tag + * @return {Boolean} + */ function isHTMLBlock(tag) { tag = tag.toLowerCase(); return htmlBlocks.includes(tag); diff --git a/syntaxes/markdown/list.js b/syntaxes/markdown/list.js index 266caeb1..e6764fa1 100644 --- a/syntaxes/markdown/list.js +++ b/syntaxes/markdown/list.js @@ -1,25 +1,19 @@ var reBlock = require('./re/block'); -var markup = require('../../'); +var MarkupIt = require('../../'); var utils = require('./utils'); var reList = reBlock.list; -// Return true if block is a list -function isListItem(type) { - return (type == markup.BLOCKS.UL_ITEM || type == markup.BLOCKS.OL_ITEM); -} - // Rule for lists, rBlock.list match the whole (multilines) list, we stop at the first item function listRule(type) { - return markup.Rule(type) - .setOption('parse', 'block') - .regExp(reList.block, function(match) { + return MarkupIt.Rule(type) + .regExp(reList.block, function(state, match) { var rawList = match[0]; var bull = match[2]; var ordered = bull.length > 1; - if (ordered && type === markup.BLOCKS.UL_ITEM) return; - if (!ordered && type === markup.BLOCKS.OL_ITEM) return; + if (ordered && type === MarkupIt.BLOCKS.UL_LIST) return; + if (!ordered && type === MarkupIt.BLOCKS.OL_LIST) return; var item, loose, next = false; @@ -33,7 +27,6 @@ function listRule(type) { rawItem = rawList.slice(lastIndex, reList.item.lastIndex); lastIndex = reList.item.lastIndex; - items.push([item, rawItem]); } @@ -63,45 +56,56 @@ function listRule(type) { if (!loose) loose = next; } - result.push({ - type: type, - raw: rawItem, - text: textItem, - data:{ - loose: loose - } - }); - } + var parse = function() { + return MarkupIt.Token.create(MarkupIt.BLOCKS.LIST_ITEM, { + raw: rawItem, + tokens: state.parseAsBlock(textItem), + data: { + loose: loose + } + }); + }; - return result; - }) - .toText(function(text, block) { - // Determine which bullet to use - var bullet = '*'; - if (type == markup.BLOCKS.OL_ITEM) { - bullet = '1.'; + result.push( + loose? state.toggle('looseList', parse) : parse() + ); } - var nextBlock = block.next? block.next.type : null; - - // Prepend text with spacing - var rows = utils.splitLines(text); - var head = rows[0]; - var rest = utils.indent(rows.slice(1).join('\n'), ' '); - - var eol = rest? '' : '\n'; - if (nextBlock && !isListItem(nextBlock)) { - eol += '\n'; - } - - var result = bullet + ' ' + head + (rest ? '\n' + rest : '') + eol; + return { + tokens: result + }; + }) + .toText(function(state, token) { + var listType = token.getType(); + var items = token.getTokens(); + + return items.reduce(function(text, item, i) { + // Determine which bullet to use + var bullet = '*'; + if (listType == MarkupIt.BLOCKS.OL_LIST) { + bullet = (i + 1) + '.'; + } - return result; + // Prepend text with spacing + var innerText = state.renderAsBlock(item); + var rows = utils.splitLines(innerText); + var head = rows[0]; + var rest = utils.indent(rows.slice(1).join('\n'), ' '); + var eol = rest? '' : '\n'; + var isLoose = item.getTokens() + .find(function(p) { + return p.getType() === MarkupIt.BLOCKS.PARAGRAPH; + }) !== undefined; + //if (isLoose) eol += '\n'; + + var itemText = bullet + ' ' + head + (rest ? '\n' + rest : '') + eol; + return text + itemText; + }, '') + '\n'; }); } module.exports = { - ul: listRule(markup.BLOCKS.UL_ITEM), - ol: listRule(markup.BLOCKS.OL_ITEM) + ul: listRule(MarkupIt.BLOCKS.UL_LIST), + ol: listRule(MarkupIt.BLOCKS.OL_LIST) }; diff --git a/syntaxes/markdown/table.js b/syntaxes/markdown/table.js index 7f4ab646..10c2a781 100644 --- a/syntaxes/markdown/table.js +++ b/syntaxes/markdown/table.js @@ -1,48 +1,35 @@ -var reTable = require('./re/table'); -var markup = require('../../'); +var Immutable = require('immutable'); +var MarkupIt = require('../../'); +var reTable = require('./re/table'); var tableRow = require('./tableRow'); -// Create a table entity +var ALIGN = { + LEFT: 'left', + RIGHT: 'right', + CENTER: 'center' +}; /** * Create a table entity from parsed header/rows * + * @param {ParsingState} state * @param {Array} header * @param {Array} align * @param {Array} rows * @rteturn {Object} tokenMatch */ -function Table(header, align, rows) { - var ctx = this; - - var headerRow = tableRow.parse(header, ctx, align); +function Table(state, header, align, rows) { + var headerRow = tableRow.parse(state, header); var rowTokens = rows.map(function(row) { - return tableRow.parse(row, ctx, align); - }); - - var headerToken = markup.Token.create(markup.BLOCKS.TABLE_HEADER, { - tokens: [headerRow], - data: { - align: align - } - }); - - var bodyToken = markup.Token.create(markup.BLOCKS.TABLE_BODY, { - tokens: rowTokens, - data: { - align: align - } + return tableRow.parse(state, row); }); return { data: { align: align }, - tokens: [ - headerToken, - bodyToken - ] + tokens: Immutable.List([headerRow]).concat(rowTokens) }; } @@ -55,11 +42,11 @@ function Table(header, align, rows) { function mapAlign(align) { return align.map(function(s) { if (reTable.alignRight.test(s)) { - return 'right'; + return ALIGN.RIGHT; } else if (reTable.alignCenter.test(s)) { - return 'center'; + return ALIGN.CENTER; } else if (reTable.alignLeft.test(s)) { - return 'left'; + return ALIGN.LEFT; } else { return null; } @@ -86,10 +73,10 @@ function alignToText(row) { }).join(''); } -var blockRule = markup.Rule(markup.BLOCKS.TABLE) +var blockRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE) // Table no leading pipe (gfm) - .regExp(reTable.nptable, function(match) { + .regExp(reTable.nptable, function(state, match) { var header = match[1]; var align = match[2] .replace(reTable.trailingPipeAlign, '') @@ -101,11 +88,11 @@ var blockRule = markup.Rule(markup.BLOCKS.TABLE) // Align for columns align = mapAlign(align); - return Table.call(this, header, align, rows); + return Table(state, header, align, rows); }) // Normal table - .regExp(reTable.normal, function(match) { + .regExp(reTable.normal, function(state, match) { var header = match[1]; var align = match[2] .replace(reTable.trailingPipeAlign, '') @@ -118,37 +105,49 @@ var blockRule = markup.Rule(markup.BLOCKS.TABLE) // Align for columns align = mapAlign(align); - return Table.call(this, header, align, rows); + return Table(state, header, align, rows); }) - .toText(function(text) { - return text + '\n'; - }); + .toText(function(state, token) { + var data = token.getData(); + var rows = token.getTokens(); + var align = data.get('align'); -var headerRule = markup.Rule(markup.BLOCKS.TABLE_HEADER) - .toText(function(text, tok) { - return text + alignToText(tok.data.align) + '\n'; - }); + var headerRows = rows.slice(0, 1); + var bodyRows = rows.slice(1); + var headerRow = headerRows.get(0); + var countCells = headerRow.getTokens().size; + + align = align || []; + align = Array + .apply(null, Array(countCells)) + .map(function(v, i){ + return align[i] || ALIGN.LEFT; + }); + + var headerText = state.render(headerRows); + var bodyText = state.render(bodyRows); + + return (headerText + + alignToText(align) + '\n' + + bodyText + '\n'); -var bodyRule = markup.Rule(markup.BLOCKS.TABLE_BODY) - .toText(function(text) { - return text; }); -var rowRule = markup.Rule(markup.BLOCKS.TABLE_ROW) - .toText(function(text) { - return '|' + text + '\n'; +var rowRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_ROW) + .toText(function(state, token) { + var innerContent = state.render(token); + return '|' + innerContent + '\n'; }); -var cellRule = markup.Rule(markup.BLOCKS.TABLE_CELL) - .toText(function(text) { - return ' ' + text.trim() + ' |'; +var cellRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_CELL) + .toText(function(state, token) { + var innerContent = state.render(token); + return ' ' + innerContent.trim() + ' |'; }); module.exports = { block: blockRule, - header: headerRule, - body: bodyRule, cell: cellRule, row: rowRule }; diff --git a/syntaxes/markdown/tableRow.js b/syntaxes/markdown/tableRow.js index 338506ca..1314e77b 100644 --- a/syntaxes/markdown/tableRow.js +++ b/syntaxes/markdown/tableRow.js @@ -7,61 +7,49 @@ var utils = require('./utils'); var CELL_SEPARATOR = 'cell'; /* - Custom inline syntax to parse each row with custom cell separator tokens -*/ + * Custom inline syntax to parse each row with custom cell separator tokens + */ var rowRules = inlineRules .unshift( MarkupIt.Rule(CELL_SEPARATOR) - .setOption('parse', false) .regExp(reTable.cellSeparation, function(match) { return { - text: match[0] + raw: match[0] }; }) ) .replace( MarkupIt.Rule(MarkupIt.STYLES.TEXT) - .setOption('parse', false) - .regExp(reTable.cellInlineEscape, function(match) { - return { - text: utils.unescape(match[0]) - }; + .regExp(reTable.cellInlineEscape, function(state, match) { + var text = utils.unescape(match[0]); + return { text: text }; }) - .regExp(reTable.cellInlineText, function(match) { - return { - text: utils.unescape(match[0]) - }; + .regExp(reTable.cellInlineText, function(state, match) { + var text = utils.unescape(match[0]); + return { text: text }; }) .toText(utils.escape) ); -var rowSyntax = MarkupIt.Syntax('markdown+row', { - inline: rowRules -}); - -/* - Parse a row from a table - - @param {String} text - @param {Object} ctx - @param {Array} align - - @return {Token} -*/ -function parseRow(text, ctx, align) { +/** + * Parse a row from a table + * + * @param {ParsingState} state + * @param {String} text + * @return {Token} + */ +function parseRow(state, text) { var cells = []; var accu = []; - var content = MarkupIt.parseInline(rowSyntax, text, ctx); - var tokens = content.getTokens(); + var tokens = state.parse(rowRules, true, text); function pushCell() { - if (accu.length == 0) return; + if (accu.length == 0) { + return; + } var cell = MarkupIt.Token.create(MarkupIt.BLOCKS.TABLE_CELL, { - tokens: accu, - data: { - align: align[cells.length] - } + tokens: accu }); cells.push(cell); @@ -75,7 +63,6 @@ function parseRow(text, ctx, align) { accu.push(token); } }); - pushCell(); return MarkupIt.Token.create(MarkupIt.BLOCKS.TABLE_ROW, { diff --git a/test.md b/test.md new file mode 100644 index 00000000..b0e81e78 --- /dev/null +++ b/test.md @@ -0,0 +1,3 @@ +Some long sentence. [^footnote] + +[^footnote]: Test, [Link](https://google.com). \ No newline at end of file diff --git a/testing/mock.js b/testing/mock.js index 9ee2a980..b66fc276 100644 --- a/testing/mock.js +++ b/testing/mock.js @@ -32,6 +32,16 @@ var helloWorld = MarkupIt.Token({ }); module.exports = { - paragraph: MarkupIt.Content.createFromTokens('mysyntax', [helloWorld]), - titleParagraph: MarkupIt.Content.createFromTokens('mysyntax', [helloTitle, helloWorld]) + paragraph: MarkupIt.Content.createFromToken( + 'mysyntax', + MarkupIt.Token.create(MarkupIt.BLOCKS.DOCUMENT, { + tokens: [helloWorld] + }) + ), + titleParagraph: MarkupIt.Content.createFromToken( + 'mysyntax', + MarkupIt.Token.create(MarkupIt.BLOCKS.DOCUMENT, { + tokens: [helloTitle, helloWorld] + }) + ) };