diff --git a/README.md b/README.md
index 132d9ae6..c922477a 100644
--- a/README.md
+++ b/README.md
@@ -60,22 +60,26 @@ var content = MarkupIt.DraftUtils.decode(rawContent);
var text = markdown.toText(content);
```
-#### Custom Syntax
+### Extend Syntax
This module contains the [markdown syntax](./syntaxes/markdown), but you can write your custom syntax or extend the existing ones.
+#### Create rules
+
```js
var myRule = MarkupIt.Rule(DraftMarkup.BLOCKS.HEADING_1)
- .regExp(/^
(\S+)<\/h1>/, function(match) {
+ .regExp(/^(\S+)<\/h1>/, function(state, match) {
return {
- text: match[1]
+ tokens: state.parseAsInline(match[1])
};
})
- .toText(function(innerText) {
- return '' + innerText+ ' ';
+ .toText(function(state, token) {
+ return '' + state.renderAsInline(token) + ' ';
});
```
+#### Custom Syntax
+
Create a new syntax inherited from the markdown one:
```js
diff --git a/bin/toProseMirror.js b/bin/toProseMirror.js
new file mode 100755
index 00000000..69fd0eb1
--- /dev/null
+++ b/bin/toProseMirror.js
@@ -0,0 +1,14 @@
+#! /usr/bin/env node
+/* eslint-disable no-console */
+
+var MarkupIt = require('../');
+var utils = require('./utils');
+
+utils.command(function(content) {
+ console.log(
+ JSON.stringify(
+ MarkupIt.ProseMirrorUtils.encode(content),
+ null, 4
+ )
+ );
+});
diff --git a/lib/__tests__/syntax.js b/lib/__tests__/syntax.js
index 56cdc09a..ace913fe 100644
--- a/lib/__tests__/syntax.js
+++ b/lib/__tests__/syntax.js
@@ -4,7 +4,7 @@ describe('Custom Syntax', function() {
var syntax = MarkupIt.Syntax('mysyntax', {
inline: [
MarkupIt.Rule(MarkupIt.STYLES.BOLD)
- .regExp(/^\*\*([\s\S]+?)\*\*/, function(match) {
+ .regExp(/^\*\*([\s\S]+?)\*\*/, function(state, match) {
return {
text: match[1]
};
@@ -22,24 +22,27 @@ describe('Custom Syntax', function() {
it('should parse as unstyled', function() {
var content = markup.toContent('Hello World');
- var tokens = content.getTokens();
- tokens.size.should.equal(1);
- var p = tokens.get(0);
+ var doc = content.getToken();
+ var blocks = doc.getTokens();
+
+ blocks.size.should.equal(1);
+ var p = blocks.get(0);
p.getType().should.equal(MarkupIt.BLOCKS.TEXT);
- p.getText().should.equal('Hello World');
+ p.getAsPlainText().should.equal('Hello World');
});
it('should parse inline', function() {
var content = markup.toContent('Hello **World**');
- var tokens = content.getTokens();
+ var doc = content.getToken();
+ var blocks = doc.getTokens();
- tokens.size.should.equal(1);
- var p = tokens.get(0);
+ blocks.size.should.equal(1);
+ var p = blocks.get(0);
p.getType().should.equal(MarkupIt.BLOCKS.TEXT);
- p.getText().should.equal('Hello World');
+ p.getAsPlainText().should.equal('Hello World');
});
});
@@ -47,22 +50,24 @@ describe('Custom Syntax', function() {
it('should output correct string', function() {
var content = MarkupIt.JSONUtils.decode({
syntax: 'mysyntax',
- tokens: [
- {
- type: MarkupIt.BLOCKS.PARAGRAPH,
- text: 'Hello World',
- tokens: [
- {
- type: MarkupIt.STYLES.TEXT,
- text: 'Hello '
- },
- {
- type: MarkupIt.STYLES.BOLD,
- text: 'World'
- }
- ]
- }
- ]
+ token: {
+ type: MarkupIt.BLOCKS.DOCUMENT,
+ tokens: [
+ {
+ type: MarkupIt.BLOCKS.PARAGRAPH,
+ tokens: [
+ {
+ type: MarkupIt.STYLES.TEXT,
+ text: 'Hello '
+ },
+ {
+ type: MarkupIt.STYLES.BOLD,
+ text: 'World'
+ }
+ ]
+ }
+ ]
+ }
});
var text = markup.toText(content);
text.should.equal('Hello **World**\n');
diff --git a/lib/constants/blocks.js b/lib/constants/blocks.js
index 4fa29a8a..c6d6484f 100644
--- a/lib/constants/blocks.js
+++ b/lib/constants/blocks.js
@@ -1,32 +1,31 @@
module.exports = {
- IGNORE: 'ignore',
- TEXT: 'text',
+ DOCUMENT: 'doc',
+ TEXT: 'unstyled',
- CODE: 'code-block',
+ CODE: 'code_block',
BLOCKQUOTE: 'blockquote',
- PARAGRAPH: 'unstyled',
+ PARAGRAPH: 'paragraph',
FOOTNOTE: 'footnote',
- DEFINITION: 'definition',
- HTML: 'html-block',
- HR: 'atomic',
+ HTML: 'html_block',
+ HR: 'hr',
- HEADING_1: 'header-one',
- HEADING_2: 'header-two',
- HEADING_3: 'header-three',
- HEADING_4: 'header-four',
- HEADING_5: 'header-five',
- HEADING_6: 'header-six',
+ HEADING_1: 'header_one',
+ HEADING_2: 'header_two',
+ HEADING_3: 'header_three',
+ HEADING_4: 'header_four',
+ HEADING_5: 'header_five',
+ HEADING_6: 'header_six',
TABLE: 'table',
- TABLE_HEADER: 'table-header',
- TABLE_BODY: 'table-body',
- TABLE_ROW: 'table-row',
- TABLE_CELL: 'table-cell',
+ TABLE_ROW: 'table_row',
+ TABLE_CELL: 'table_cell',
- OL_ITEM: 'ordered-list-item',
- UL_ITEM: 'unordered-list-item',
+ OL_LIST: 'ordered_list',
+ UL_LIST: 'unordered_list',
+
+ LIST_ITEM: 'list_item',
// GitBook specifics
TEMPLATE: 'template',
- MATH: 'math'
+ MATH: 'math_block'
};
diff --git a/lib/constants/defaultRules.js b/lib/constants/defaultRules.js
index efe0687d..2b62ad1e 100644
--- a/lib/constants/defaultRules.js
+++ b/lib/constants/defaultRules.js
@@ -3,14 +3,34 @@ var Rule = require('../models/rule');
var BLOCKS = require('./blocks');
var STYLES = require('./styles');
+var defaultDocumentRule = Rule(BLOCKS.DOCUMENT)
+ .match(function(state, text) {
+ return {
+ tokens: state.parseAsBlock(text)
+ };
+ })
+ .toText(function(state, token) {
+ return state.renderAsBlock(token);
+ });
+
var defaultBlockRule = Rule(BLOCKS.TEXT)
+ .match(function(state, text) {
+ return {
+ tokens: state.parseAsInline(text)
+ };
+ })
.toText('%s\n');
var defaultInlineRule = Rule(STYLES.TEXT)
- .setOption('parse', false)
+ .match(function(state, text) {
+ return {
+ text: text
+ };
+ })
.toText('%s');
module.exports = {
- blockRule: defaultBlockRule,
- inlineRule: defaultInlineRule
+ documentRule: defaultDocumentRule,
+ blockRule: defaultBlockRule,
+ inlineRule: defaultInlineRule
};
diff --git a/lib/constants/entities.js b/lib/constants/entities.js
index 15a05ec2..b7044f98 100644
--- a/lib/constants/entities.js
+++ b/lib/constants/entities.js
@@ -1,8 +1,6 @@
module.exports = {
LINK: 'link',
- LINK_REF: 'link-ref',
IMAGE: 'image',
- LINK_IMAGE: 'link-image',
FOOTNOTE_REF: 'footnote-ref',
// GitBook specifics
diff --git a/lib/constants/styles.js b/lib/constants/styles.js
index 1472d172..e8ac9bf7 100644
--- a/lib/constants/styles.js
+++ b/lib/constants/styles.js
@@ -1,9 +1,10 @@
module.exports = {
+ TEXT: 'text',
+
BOLD: 'BOLD',
ITALIC: 'ITALIC',
CODE: 'CODE',
STRIKETHROUGH: 'STRIKETHROUGH',
- TEXT: 'UNSTYLED',
HTML: 'HTML',
// GitBook specifics
diff --git a/lib/draft/__tests__/decode.js b/lib/draft/__tests__/decode.js
deleted file mode 100644
index 46f651f0..00000000
--- a/lib/draft/__tests__/decode.js
+++ /dev/null
@@ -1,78 +0,0 @@
-var decode = require('../decode');
-var ENTITIES = require('../../constants/entities');
-var BLOCKS = require('../../constants/blocks');
-var STYLES = require('../../constants/styles');
-
-describe('decode', function() {
- var content;
-
- before(function() {
- var rawContent = {
- entityMap: {
- '1': {
- type: ENTITIES.LINK,
- mutability: 'MUTABLE',
- data: {
- href: 'http://google.fr'
- }
- }
- },
- blocks: [
- {
- type: BLOCKS.HEADING_1,
- text: 'Hello World',
- inlineStyleRanges: [],
- entityRanges: []
- },
- {
- type: BLOCKS.PARAGRAPH,
- text: 'This is a link',
- inlineStyleRanges: [
- {
- offset: 0,
- length: 4,
- style: STYLES.BOLD
- }
- ],
- entityRanges: [
- {
- offset: 10,
- length: 4,
- key: '1'
- }
- ]
- }
- ]
- };
-
- content = decode(rawContent);
- });
-
- it('should correctly extract block tokens', function() {
- var tokens = content.getTokens();
-
- tokens.size.should.equal(2);
- tokens.get(0).getType().should.equal(BLOCKS.HEADING_1);
- tokens.get(1).getType().should.equal(BLOCKS.PARAGRAPH);
- });
-
- it('should correctly extract inline styles', function() {
- var tokens = content.getTokens();
- var p = tokens.get(1);
- var inline = p.getTokens();
-
- inline.size.should.equal(3);
-
- var bold = inline.get(0);
- bold.getType().should.equal(STYLES.BOLD);
- bold.getText().should.equal('This');
-
- var text = inline.get(1);
- text.getType().should.equal(STYLES.TEXT);
- text.getText().should.equal(' is a ');
-
- var link = inline.get(2);
- link.getType().should.equal(ENTITIES.LINK);
- link.getText().should.equal('link');
- });
-});
diff --git a/lib/draft/__tests__/encode.js b/lib/draft/__tests__/encode.js
deleted file mode 100644
index bedf96df..00000000
--- a/lib/draft/__tests__/encode.js
+++ /dev/null
@@ -1,24 +0,0 @@
-var encode = require('../encode');
-var BLOCKS = require('../../constants/blocks');
-
-describe('encode', function() {
- describe('paragraph + heading', function() {
- var rawContent = encode(mock.titleParagraph);
-
- it('should return empty entityMap', function() {
- rawContent.should.have.property('entityMap');
- rawContent.entityMap.should.deepEqual({});
- });
-
- it('should return blocks', function() {
- rawContent.should.have.property('blocks');
- rawContent.blocks.should.have.lengthOf(2);
-
- rawContent.blocks[0].should.have.property('type');
- rawContent.blocks[0].should.have.property('text');
- rawContent.blocks[0].type.should.equal(BLOCKS.HEADING_1);
-
- rawContent.blocks[1].type.should.equal(BLOCKS.PARAGRAPH);
- });
- });
-});
diff --git a/lib/draft/decode.js b/lib/draft/decode.js
deleted file mode 100644
index b9a8bf6a..00000000
--- a/lib/draft/decode.js
+++ /dev/null
@@ -1,137 +0,0 @@
-var Immutable = require('immutable');
-var Range = require('range-utils');
-
-var Content = require('../models/content');
-var Token = require('../models/token');
-
-var STYLES = require('../constants/styles');
-
-var decodeEntities = require('./decodeEntities');
-
-/**
- * Return true if range is an entity
- *
- * @param {Range}
- * @return {Boolean}
- */
-function isRangeEntity(range) {
- return Boolean(range.entity);
-}
-
-/**
- * Convert a RangeTreeNode to a token
- *
- * @param {String}
- * @param {RangeTreeNode}
- * @return {Token}
- */
-function encodeRangeNodeToToken(baseText, range) {
- var innerText = baseText.slice(range.offset, range.offset + range.length);
- var isRange = isRangeEntity(range);
-
- return new Token({
- type: isRange? range.entity.type : range.style,
- text: innerText,
- data: isRange? range.entity.data : {},
- tokens: range.style == STYLES.TEXT? [] : encodeRangeTreeToTokens(innerText, range.children || [])
- });
-}
-
-/**
- * Convert a RangeTree to a list of tokens
- *
- * @param {String}
- * @param {RangeTree}
- * @return {List}
- */
-function encodeRangeTreeToTokens(baseText, ranges) {
- ranges = Range.fill(baseText, ranges, {
- style: STYLES.TEXT
- });
-
- return new Immutable.List(
- ranges.map(function(range) {
- return encodeRangeNodeToToken(baseText, range);
- })
- );
-}
-
-/**
- * Convert a ContentBlock into a token
- *
- * @param {ContentBlock} block
- * @param {Object} entityMap
- * @return {Token}
- */
-function encodeBlockToToken(block, entityMap) {
- var text = block.text;
-
- var styleRanges = block.inlineStyleRanges;
- var entityRanges = block.entityRanges;
- var blocks = block.blocks;
- var tokens;
-
- if (blocks && blocks.length > 0) {
- tokens = decodeDraftBlocksToTokens(blocks, entityMap);
- } else {
- // Unmerge link-image
- entityRanges = decodeEntities(entityRanges, entityMap);
-
- var allEntities = []
- .concat(entityRanges)
- .concat(styleRanges);
-
- var tree = Range.toTree(allEntities, isRangeEntity);
- tokens = encodeRangeTreeToTokens(text, tree);
- }
-
- return new Token({
- type: block.type,
- text: text,
- data: block.data,
- tokens: tokens
- });
-}
-
-/**
- * Transform a an array of blocks to a list of tokens
- *
- * @param {Array} rawBlocks
- * @param {Object} entityMap
- * @return {List}
- */
-function decodeDraftBlocksToTokens(rawBlocks, entityMap) {
- return Immutable.List(rawBlocks)
- .map(function(block) {
- return encodeBlockToToken(block, entityMap);
- });
-}
-
-/**
- * Transform a RawContentState from draft into a list of tokens
- *
- * @param {RawContentState} rawContentState
- * @return {List}
- */
-function decodeDraftToTokens(rawContentState) {
- var blocks = rawContentState.blocks;
- var entityMap = rawContentState.entityMap;
-
- return decodeDraftBlocksToTokens(blocks, entityMap);
-}
-
-/**
- * Transform a RawContentState from draft into a Content instance
- *
- * @param {RawContentState} rawContentState
- * @param {String} syntaxName
- * @return {Content}
- */
-function decodeDraftToContent(rawContentState, syntaxName) {
- syntaxName = syntaxName || 'draft-js';
- var tokens = decodeDraftToTokens(rawContentState);
-
- return Content.createFromTokens(syntaxName, tokens);
-}
-
-module.exports = decodeDraftToContent;
diff --git a/lib/draft/decodeEntities.js b/lib/draft/decodeEntities.js
deleted file mode 100644
index b6451ca9..00000000
--- a/lib/draft/decodeEntities.js
+++ /dev/null
@@ -1,47 +0,0 @@
-var is = require('is');
-var Range = require('range-utils');
-var ENTITIES = require('../constants/entities');
-
-/**
- * Normalize and decode a list of entity ranges
- *
- * @param {Array} entityRanges
- * @param {Object} entityMap
- * @return {Array}
- */
-function decodeEntities(entityRanges, entityMap) {
- return entityRanges.reduce(function(result, range) {
- if (is.defined(range.key)) {
- range.entity = entityMap[String(range.key)];
- delete range.key;
- }
-
- if (!range.entity || range.entity.type != ENTITIES.LINK_IMAGE) {
- result.push(range);
- return result;
- }
-
- result.push(Range(range.offset, range.length, {
- entity: {
- type: ENTITIES.IMAGE,
- data: {
- src: range.entity.data.src,
- title: range.entity.data.imageTitle
- }
- }
- }));
- result.push(Range(range.offset, range.length, {
- entity: {
- type: ENTITIES.LINK,
- data: {
- href: range.entity.data.href,
- title: range.entity.data.linkTitle
- }
- }
- }));
-
- return result;
- }, []);
-}
-
-module.exports = decodeEntities;
diff --git a/lib/draft/encode.js b/lib/draft/encode.js
deleted file mode 100644
index 9ee8278b..00000000
--- a/lib/draft/encode.js
+++ /dev/null
@@ -1,209 +0,0 @@
-var Immutable = require('immutable');
-var Range = require('range-utils');
-
-var walk = require('../utils/walk');
-var genKey = require('../utils/genKey');
-var getMutability = require('./getMutability');
-
-var ENTITIES = require('../constants/entities');
-var STYLES = require('../constants/styles');
-
-/**
- * Default option for encoding to draft
- */
-var EncodingOption = Immutable.Record({
- // Blacklist some type of tokens
- blacklist: []
-});
-
-
-/**
- * Encode an entity from a token
- *
- * @param {Token} token
- * @return {Object}
- */
-function encodeTokenToEntity(token) {
- return {
- type: token.getType(),
- mutability: getMutability(token),
- data: token.getData().toJS()
- };
-}
-
-/**
- * Encode a token into a ContentBlock
- *
- * @param {Token} token
- * @param {Function} registerEntity
- * @param {Function} filter
- * @return {Object}
- */
-function encodeTokenToBlock(token, registerEntity, filter) {
- var tokenType = token.getType();
- var inlineStyleRanges = [];
- var entityRanges = [];
- var blockTokens = [];
-
- // Add child block tokens as data.innerContent
- token.getTokens()
- .forEach(function(tok) {
- if (!tok.isBlock()) {
- return;
- }
-
- blockTokens.push(tok);
- });
-
- var innerText = walk(token, function(tok, range) {
- if (tok.isEntity()) {
- if (!filter(tok)) {
- return;
- }
-
- var entity = encodeTokenToEntity(tok);
-
- entityRanges.push(
- Range(
- range.offset,
- range.length,
- {
- entity: entity
- }
- )
- );
-
- } else if (tok.isStyle() && tok.getType() !== STYLES.TEXT) {
- if (!filter(tok)) {
- return;
- }
-
- inlineStyleRanges.push(
- Range(
- range.offset,
- range.length,
- {
- style: tok.getType()
- }
- )
- );
- }
- });
-
- // Linearize/Merge ranges (draft-js doesn't support multiple entities on the same range)
- entityRanges = Range.merge(entityRanges, function(a, b) {
- if (
- (a.entity.type == ENTITIES.IMAGE || a.entity.type == ENTITIES.LINK) &&
- (b.entity.type == ENTITIES.IMAGE || b.entity.type == ENTITIES.LINK) &&
- (a.entity.type !== b.entity.type)
- ) {
- var img = ((a.entity.type == ENTITIES.IMAGE)? a : b).entity.data;
- var link = ((a.entity.type == ENTITIES.LINK)? a : b).entity.data;
-
- return Range(a.offset, a.length, {
- entity: {
- type: ENTITIES.LINK_IMAGE,
- mutability: getMutability(ENTITIES.LINK_IMAGE),
- data: {
- src: img.src,
- href: link.href,
- imageTitle: img.title,
- linkTitle: link.title
- }
- }
- });
- }
-
- return a;
- });
-
- // Register all entities
- entityRanges = entityRanges.map(function(range) {
- var entityKey = registerEntity(range.entity);
-
- return Range(range.offset, range.length, {
- key: entityKey
- });
- });
-
- // Metadata for this blocks
- var data = token.getData().toJS();
-
- // Encode inner tokens
- blockTokens = encodeTokensToBlocks(Immutable.List(blockTokens), registerEntity, filter);
-
- return {
- key: genKey(),
- type: tokenType,
- text: innerText,
- data: data,
- inlineStyleRanges: inlineStyleRanges,
- entityRanges: entityRanges,
- blocks: blockTokens
- };
-}
-
-
-/**
- * Encode a list of Token into a RawContentState for draft
- *
- * @paran {List} tokens
- * @param {Function} registerEntity
- * @param {Function} filter
- * @return {Object}
- */
-function encodeTokensToBlocks(blockTokens, registerEntity, filter) {
- return blockTokens
- .map(function(token) {
- if (!filter(token)) {
- return;
- }
-
- return encodeTokenToBlock(token, registerEntity, filter);
- })
- .filter(function(blk) {
- return Boolean(blk);
- })
- .toJS();
-}
-
-
-/**
- * Encode a Content instance into a RawContentState for draft
- *
- * @paran {Content} content
- * @return {Object}
- */
-function encodeContentToDraft(content, options) {
- options = EncodingOption(options || {});
- var blacklist = Immutable.List(options.get('blacklist'));
- var blockTokens = content.getTokens();
- var entityKey = 0;
- var entityMap = {};
-
- // Register an entity and returns its key/ID
- function registerEntity(entity) {
- entityKey++;
- entityMap[entityKey] = entity;
-
- return entityKey;
- }
-
- // Filter tokens
- function filter(token) {
- var type = token.getType();
-
- if (blacklist.includes(type)) {
- throw new Error('Content of type "' + type + '" is not allowed');
- }
-
- return true;
- }
-
- return {
- blocks: encodeTokensToBlocks(blockTokens, registerEntity, filter),
- entityMap: entityMap
- };
-}
-
-module.exports = encodeContentToDraft;
diff --git a/lib/draft/getMutability.js b/lib/draft/getMutability.js
deleted file mode 100644
index 479f3de1..00000000
--- a/lib/draft/getMutability.js
+++ /dev/null
@@ -1,23 +0,0 @@
-var is = require('is');
-var ENTITIES = require('../constants/entities');
-
-var MUTABLE = 'MUTABLE';
-var IMMUTABLE = 'IMMUTABLE';
-
-var MUTABLE_TYPES = [
- ENTITIES.LINK, ENTITIES.LINK_REF, ENTITIES.FOOTNOTE_REF
-];
-
-/**
- * Get mutability of a token
- *
- * @param {Token|String} token
- * @return @String
- */
-function getMutability(token) {
- var tokenType = is.string(token)? token : token.getType();
- return (MUTABLE_TYPES.indexOf(tokenType) >= 0)? MUTABLE : IMMUTABLE;
-}
-
-
-module.exports = getMutability;
diff --git a/lib/draft/index.js b/lib/draft/index.js
deleted file mode 100644
index 53f9a69c..00000000
--- a/lib/draft/index.js
+++ /dev/null
@@ -1,7 +0,0 @@
-var encode = require('./encode');
-var decode = require('./decode');
-
-module.exports = {
- encode: encode,
- decode: decode
-};
diff --git a/lib/index.js b/lib/index.js
index dabbd22f..4feb67aa 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -13,32 +13,32 @@ var Token = require('./models/token');
var parse = require('./parse');
var render = require('./render');
-var DraftUtils = require('./draft');
var JSONUtils = require('./json');
+var ProseMirrorUtils = require('./prosemirror');
var genKey = require('./utils/genKey');
+var transform = require('./utils/transform');
module.exports = Markup;
// Method
-module.exports.parse = parse;
-module.exports.parseInline = parse.inline;
-
+module.exports.parse = parse;
module.exports.render = render;
// Models
-module.exports.Content = Content;
-module.exports.Token = Token;
-module.exports.Syntax = Syntax;
-module.exports.Rule = Rule;
+module.exports.Content = Content;
+module.exports.Token = Token;
+module.exports.Syntax = Syntax;
+module.exports.Rule = Rule;
module.exports.RulesSet = RulesSet;
// Utils
-module.exports.DraftUtils = DraftUtils;
-module.exports.JSONUtils = JSONUtils;
-module.exports.genKey = genKey;
+module.exports.JSONUtils = JSONUtils;
+module.exports.ProseMirrorUtils = ProseMirrorUtils;
+module.exports.genKey = genKey;
+module.exports.transform = transform;
// Constants
-module.exports.STYLES = STYLES;
+module.exports.STYLES = STYLES;
module.exports.ENTITIES = ENTITIES;
-module.exports.BLOCKS = BLOCKS;
+module.exports.BLOCKS = BLOCKS;
diff --git a/lib/json/__tests__/decode.js b/lib/json/__tests__/decode.js
index 0b6ef609..1afe1d50 100644
--- a/lib/json/__tests__/decode.js
+++ b/lib/json/__tests__/decode.js
@@ -7,13 +7,16 @@ describe('decode', function() {
before(function() {
content = decode({
syntax: 'mysyntax',
- tokens: [
- {
- type: BLOCKS.PARAGRAPH,
- text: 'Hello World',
- raw: 'Hello World'
- }
- ]
+ token: {
+ type: BLOCKS.DOCUMENT,
+ tokens: [
+ {
+ type: BLOCKS.PARAGRAPH,
+ text: 'Hello World',
+ raw: 'Hello World'
+ }
+ ]
+ }
});
});
@@ -22,7 +25,8 @@ describe('decode', function() {
});
it('should decode tokens tree', function() {
- var tokens = content.getTokens();
+ var doc = content.getToken();
+ var tokens = doc.getTokens();
tokens.size.should.equal(1);
var p = tokens.get(0);
diff --git a/lib/json/__tests__/encode.js b/lib/json/__tests__/encode.js
index 0436fa10..11ada671 100644
--- a/lib/json/__tests__/encode.js
+++ b/lib/json/__tests__/encode.js
@@ -1,7 +1,7 @@
var encode = require('../encode');
var BLOCKS = require('../../constants/blocks');
-describe('decode', function() {
+describe('encode', function() {
var json;
before(function() {
@@ -13,9 +13,13 @@ describe('decode', function() {
});
it('should encode tokens', function() {
- json.tokens.should.have.lengthOf(1);
+ json.should.have.property('token');
- var p = json.tokens[0];
+ var doc = json.token;
+
+ doc.tokens.should.have.lengthOf(1);
+
+ var p = doc.tokens[0];
p.type.should.equal(BLOCKS.PARAGRAPH);
p.text.should.equal('Hello World');
p.tokens.should.be.an.Array().with.lengthOf(2);
diff --git a/lib/json/decode.js b/lib/json/decode.js
index b761343a..33dd33b4 100644
--- a/lib/json/decode.js
+++ b/lib/json/decode.js
@@ -36,9 +36,9 @@ function decodeTokensFromJSON(json) {
* @return {Content}
*/
function decodeContentFromJSON(json) {
- return Content.createFromTokens(
+ return Content.createFromToken(
json.syntax,
- decodeTokensFromJSON(json.tokens)
+ decodeTokenFromJSON(json.token)
);
}
diff --git a/lib/json/encode.js b/lib/json/encode.js
index 900b827c..ba5cff05 100644
--- a/lib/json/encode.js
+++ b/lib/json/encode.js
@@ -32,7 +32,7 @@ function encodeTokenToJSON(token) {
* Encode a list of tokens to JSON
*
* @paran {List} tokens
- * @return {Object}
+ * @return {Array}
*/
function encodeTokensToJSON(tokens) {
return tokens.map(encodeTokenToJSON).toJS();
@@ -47,7 +47,7 @@ function encodeTokensToJSON(tokens) {
function encodeContentToJSON(content) {
return {
syntax: content.getSyntax(),
- tokens: encodeTokensToJSON(content.getTokens())
+ token: encodeTokenToJSON(content.getToken())
};
}
diff --git a/lib/models/__tests__/token.js b/lib/models/__tests__/token.js
index 493ad40f..4f381a89 100644
--- a/lib/models/__tests__/token.js
+++ b/lib/models/__tests__/token.js
@@ -4,8 +4,8 @@ var STYLES = require('../../constants/styles');
describe('Token', function() {
describe('.mergeWith', function() {
it('should merge text and raw', function() {
- var base = Token.createInlineText('Hello ');
- var other = Token.createInlineText('world');
+ var base = Token.createText('Hello ');
+ var other = Token.createText('world');
var token = base.mergeWith(other);
token.getType().should.equal(STYLES.TEXT);
diff --git a/lib/models/content.js b/lib/models/content.js
index 0e2b4081..a3a9d3bd 100644
--- a/lib/models/content.js
+++ b/lib/models/content.js
@@ -1,11 +1,13 @@
var Immutable = require('immutable');
+var Token = require('./token');
+var BLOCKS = require('../constants/BLOCKS');
var Content = Immutable.Record({
// Name of the syntax used to parse
syntax: String(),
- // List of block tokens
- tokens: new Immutable.List()
+ // Entry token
+ token: Token.create(BLOCKS.DOCUMENT)
});
// ---- GETTERS ----
@@ -13,8 +15,8 @@ Content.prototype.getSyntax = function() {
return this.get('syntax');
};
-Content.prototype.getTokens = function() {
- return this.get('tokens');
+Content.prototype.getToken = function() {
+ return this.get('token');
};
// ---- STATICS ----
@@ -23,13 +25,13 @@ Content.prototype.getTokens = function() {
* Create a content from a syntax and a list of tokens
*
* @param {Syntax} syntax
- * @param {Array|List}
+ * @param {Token}
* @return {Content}
*/
-Content.createFromTokens = function(syntax, tokens) {
+Content.createFromToken = function(syntax, token) {
return new Content({
syntax: syntax,
- tokens: new Immutable.List(tokens)
+ token: token
});
};
diff --git a/lib/models/rule.js b/lib/models/rule.js
index 3eea70b7..8d1ee295 100644
--- a/lib/models/rule.js
+++ b/lib/models/rule.js
@@ -2,20 +2,12 @@ var Immutable = require('immutable');
var is = require('is');
var inherits = require('util').inherits;
+var Token = require('./token');
+
var RuleRecord = Immutable.Record({
// Type of the rule
type: new String(),
- // Options for this rule
- options: new Immutable.Map({
- // Mode for parsing inner content of a token
- // Values can be "inline", "block" or false
- parse: 'inline',
-
- // Render inner content
- renderInner: true
- }),
-
// Listener / Handlers {Map()}
listeners: new Immutable.Map()
});
@@ -35,10 +27,6 @@ Rule.prototype.getType = function() {
return this.get('type');
};
-Rule.prototype.getOptions = function() {
- return this.get('options');
-};
-
Rule.prototype.getListeners = function() {
return this.get('listeners');
};
@@ -63,30 +51,6 @@ Rule.prototype.on = function(key, fn) {
return this.set('listeners', listeners);
};
-/**
- * Set an option
- * @param {String} key
- * @param {String|Number|Boolean} value
- * @return {Rule}
- */
-Rule.prototype.setOption = function(key, value) {
- var options = this.getOptions();
-
- options = options.set(key, value);
-
- return this.set('options', options);
-};
-
-/**
- * Get an option
- * @param {String} key
- * @return {String|Number|Boolean}
- */
-Rule.prototype.getOption = function(key, defaultValue) {
- var options = this.getOptions();
- return options.get(key, defaultValue);
-};
-
/**
* Add a template or function to render a token
* @param {String|Function} fn
@@ -95,9 +59,9 @@ Rule.prototype.getOption = function(key, defaultValue) {
Rule.prototype.toText = function(fn) {
if (is.string(fn)) {
var tpl = fn;
- fn = function (text) {
+ fn = function (state, token) {
return tpl.replace('%s', function() {
- return text;
+ return state.render(token);
});
};
}
@@ -105,15 +69,6 @@ Rule.prototype.toText = function(fn) {
return this.on('text', fn);
};
-/**
- * Add a finish callback
- * @param {Function} fn
- * @return {Rule}
- */
-Rule.prototype.finish = function(fn) {
- return this.on('finish', fn);
-};
-
/**
* Add a match callback
* @param {Function} fn
@@ -130,21 +85,28 @@ Rule.prototype.match = function(fn) {
* @return {Rule}
*/
Rule.prototype.regExp = function(re, propsFn) {
- var ruleType = this.get('type');
-
- return this.match(function(text, parent) {
+ return this.match(function(state, text) {
var block = {};
var match = re.exec(text);
- if (!match) return null;
-
- if (propsFn) block = propsFn.call(this, match, parent);
- if (!block) return null;
- if (is.array(block)) return block;
+ if (!match) {
+ return null;
+ }
+ if (propsFn) {
+ block = propsFn.call(null, state, match);
+ }
+
+ if (!block) {
+ return null;
+ }
+ else if (block instanceof Token) {
+ return block;
+ }
+ else if (is.array(block)) {
+ return block;
+ }
block.raw = is.undefined(block.raw)? match[0] : block.raw;
- block.text = is.undefined(block.text)? match[0] : block.text;
- block.type = block.type || ruleType;
return block;
});
@@ -155,8 +117,8 @@ Rule.prototype.regExp = function(re, propsFn) {
* @param {String} key
* @return {Mixed}
*/
-Rule.prototype.emit = function(key, ctx) {
- var args = Array.prototype.slice.call(arguments, 2);
+Rule.prototype.emit = function(key) {
+ var args = Array.prototype.slice.call(arguments, 1);
var listeners = this.getListeners();
// Add the function to the list
@@ -165,23 +127,28 @@ Rule.prototype.emit = function(key, ctx) {
return fns.reduce(function(result, fn) {
if (result) return result;
- return fn.apply(ctx, args);
+ return fn.apply(null, args);
}, null);
};
-// Parse a text in a specific context
-Rule.prototype.onText = function(ctx, text, parent) {
- return this.emit('match', ctx, text, parent);
-};
-
-// Parsing is finished
-Rule.prototype.onFinish = function(ctx, token) {
- return this.emit('finish', ctx, token) || token;
+/**
+ * Parse a text using matching rules and return a list of tokens
+ * @param {ParsingState} state
+ * @param {String} text
+ * @return {List}
+ */
+Rule.prototype.onText = function(state, text) {
+ return this.emit('match', state, text);
};
-// Output inner of block as a string
-Rule.prototype.onToken = function(ctx, text, entity, pos) {
- return this.emit('text', ctx, text, entity, pos);
+/**
+ * Render a token as a string
+ * @param {RenderingState} state
+ * @param {Token} token
+ * @return {String}
+ */
+Rule.prototype.onToken = function(state, text) {
+ return this.emit('text', state, text);
};
module.exports = Rule;
diff --git a/lib/models/syntax.js b/lib/models/syntax.js
index e5cf87b3..712ea9fa 100644
--- a/lib/models/syntax.js
+++ b/lib/models/syntax.js
@@ -1,13 +1,15 @@
var Immutable = require('immutable');
var inherits = require('util').inherits;
+var Rule = require('./rule');
var RulesSet = require('./rules');
var defaultRules = require('../constants/defaultRules');
var SyntaxSetRecord = Immutable.Record({
- name: String(),
- inline: new RulesSet([]),
- blocks: new RulesSet([])
+ name: String(),
+ entryRule: new Rule(),
+ inline: new RulesSet([]),
+ blocks: new RulesSet([])
});
function SyntaxSet(name, def) {
@@ -16,14 +18,19 @@ function SyntaxSet(name, def) {
}
SyntaxSetRecord.call(this, {
- name: name,
- inline: new RulesSet(def.inline),
- blocks: new RulesSet(def.blocks)
+ name: name,
+ entryRule: def.entryRule,
+ inline: new RulesSet(def.inline),
+ blocks: new RulesSet(def.blocks)
});
}
inherits(SyntaxSet, SyntaxSetRecord);
// ---- GETTERS ----
+SyntaxSet.prototype.getEntryRule = function() {
+ return this.get('entryRule') || defaultRules.documentRule;
+};
+
SyntaxSet.prototype.getName = function() {
return this.get('name');
};
diff --git a/lib/models/token.js b/lib/models/token.js
index d85e9279..36a5a488 100644
--- a/lib/models/token.js
+++ b/lib/models/token.js
@@ -16,9 +16,10 @@ var TokenRecord = Immutable.Record({
data: new Immutable.Map(),
// Inner text of this token (for inline tokens)
- text: String(),
+ text: null,
// Original raw content of this token
+ // Can be use for annotating
raw: String(),
// List of children tokens (for block tokens)
@@ -32,10 +33,10 @@ function Token(def) {
}
TokenRecord.call(this, {
- type: def.type,
- data: new Immutable.Map(def.data),
- text: def.text,
- raw: def.raw,
+ type: def.type,
+ data: new Immutable.Map(def.data),
+ text: def.text,
+ raw: def.raw,
tokens: new Immutable.List(def.tokens)
});
}
@@ -94,14 +95,6 @@ Token.prototype.isEntity = function() {
return isEntity(this);
};
-/**
- * Return true if is a list item token
- * @return {Boolean}
- */
-Token.prototype.isListItem = function() {
- return this.getType() === BLOCKS.UL_ITEM || this.getType() === BLOCKS.OL_ITEM;
-};
-
/**
* Merge this token with another one
* @param {Token} token
@@ -119,6 +112,31 @@ Token.prototype.mergeWith = function(token) {
});
};
+/**
+ * Update data of the token
+ * @param {Object|Map}
+ * @return {Token}
+ */
+Token.prototype.setData = function(data) {
+ return this.set('data', Immutable.Map(data));
+};
+
+/**
+ * Return plain text of a token merged with its children.
+ * @return {String}
+ */
+Token.prototype.getAsPlainText = function() {
+ var tokens = this.getTokens();
+
+ if (tokens.size === 0) {
+ return (this.getText() || '');
+ }
+
+ return tokens.reduce(function(text, tok) {
+ return text + tok.getAsPlainText();
+ }, '');
+};
+
// ---- STATICS ----
/**
@@ -127,14 +145,22 @@ Token.prototype.mergeWith = function(token) {
* @return {Token}
*/
Token.create = function(type, tok) {
- var text = tok.text || '';
+ tok = tok || {};
+
+ var text = tok.text || '';
+ var tokens = Immutable.List(tok.tokens || []);
+ var data = Immutable.Map(tok.data || {});
+
+ if (tokens.size > 0) {
+ text = undefined;
+ }
return new Token({
type: type,
text: text,
raw: tok.raw || '',
- tokens: Immutable.List(tok.tokens || []),
- data: Immutable.Map(tok.data || {})
+ tokens: tokens,
+ data: data
});
};
@@ -143,24 +169,10 @@ Token.create = function(type, tok) {
* @param {String} text
* @return {Token}
*/
-Token.createInlineText = function(text) {
+Token.createText = function(text) {
return Token.create(STYLES.TEXT, {
text: text,
- raw: text
- });
-};
-
-/**
- * Create a token for a block text
- * @param {String} raw
- * @return {Token}
- */
-Token.createBlockText = function(raw) {
- var text = raw.trim();
-
- return Token.create(BLOCKS.TEXT, {
- text: text,
- raw: raw
+ raw: text
});
};
diff --git a/lib/parse/__tests__/mergeTokens.js b/lib/parse/__tests__/mergeTokens.js
index 3de416aa..cf91d09d 100644
--- a/lib/parse/__tests__/mergeTokens.js
+++ b/lib/parse/__tests__/mergeTokens.js
@@ -7,8 +7,8 @@ var mergeTokens = require('../mergeTokens');
describe('mergeTokens', function() {
it('should merge two tokens', function() {
var tokens = Immutable.List([
- Token.createInlineText('Hello '),
- Token.createInlineText('world')
+ Token.createText('Hello '),
+ Token.createText('world')
]);
var merged = mergeTokens(tokens, [STYLES.TEXT]);
@@ -21,9 +21,9 @@ describe('mergeTokens', function() {
it('should merge three tokens', function() {
var tokens = Immutable.List([
- Token.createInlineText('Hello '),
- Token.createInlineText('world'),
- Token.createInlineText('!')
+ Token.createText('Hello '),
+ Token.createText('world'),
+ Token.createText('!')
]);
var merged = mergeTokens(tokens, [STYLES.TEXT]);
@@ -36,14 +36,14 @@ describe('mergeTokens', function() {
it('should merge 2x2 tokens', function() {
var tokens = Immutable.List([
- Token.createInlineText('Hello '),
- Token.createInlineText('world'),
+ Token.createText('Hello '),
+ Token.createText('world'),
new Token({
type: STYLES.BOLD,
text: ', right?'
}),
- Token.createInlineText('!'),
- Token.createInlineText('!')
+ Token.createText('!'),
+ Token.createText('!')
]);
var merged = mergeTokens(tokens, [STYLES.TEXT]);
diff --git a/lib/parse/cleanup.js b/lib/parse/cleanup.js
deleted file mode 100644
index 061079ab..00000000
--- a/lib/parse/cleanup.js
+++ /dev/null
@@ -1,17 +0,0 @@
-
-/**
- * Cleanup a text before parsing: normalize newlines and tabs
- *
- * @param {String} src
- * @return {String}
- */
-function cleanupText(src) {
- return src
- .replace(/\r\n|\r/g, '\n')
- .replace(/\t/g, ' ')
- .replace(/\u00a0/g, ' ')
- .replace(/\u2424/g, '\n')
- .replace(/^ +$/gm, '');
-}
-
-module.exports = cleanupText;
diff --git a/lib/parse/finish.js b/lib/parse/finish.js
deleted file mode 100644
index 212b6ff3..00000000
--- a/lib/parse/finish.js
+++ /dev/null
@@ -1,36 +0,0 @@
-var transform = require('../utils/transform');
-
-/**
- * Post processing for parsing.
- * Call `onFinish` of rules.
- *
- * @param {Syntax} syntax
- * @param {List} tokens
- * @return {List}
- */
-function finish(syntax, content, ctx) {
- return transform(content, function(token, depth) {
- var tokenType = token.getType();
- var rule;
-
- if (token.isInline()) {
- rule = syntax.getInlineRule(tokenType);
- } else {
- rule = syntax.getBlockRule(tokenType);
- }
-
- var def = {
- type: token.getType(),
- data: token.getData().toJS(),
- text: token.getText(),
- raw: token.getRaw()
- };
-
- return token.merge(
- rule.onFinish(ctx, def)
- );
- });
-}
-
-
-module.exports = finish;
diff --git a/lib/parse/index.js b/lib/parse/index.js
index 38f1d1bf..df570d88 100644
--- a/lib/parse/index.js
+++ b/lib/parse/index.js
@@ -1,149 +1,19 @@
-var Immutable = require('immutable');
-
-var Token = require('../models/token');
var Content = require('../models/content');
-var getText = require('../utils/getText');
-var lex = require('./lex');
-var finish = require('./finish');
-var cleanup = require('./cleanup');
-
-function createContext(ctx) {
- return (ctx || {});
-}
-
-/**
- * Parse an inline text into a list of tokens
- *
- * @param {Syntax} syntax
- * @param {String} text
- * @param {List} parents
- * @param {Object} ctx
- * @return {List}
- */
-function parseAsInlineTokens(syntax, text, parents, ctx) {
- var inlineRulesSet = syntax.getInlineRulesSet();
- var inlineRules = inlineRulesSet.getRules();
-
- // Parse block tokens
- var tokens = lex.inline(inlineRules, text, parents, ctx);
-
- // Parse inline content for each token
- tokens = tokens.map(function(token) {
- return parseInnerToken(syntax, token, parents, ctx);
- });
-
- return tokens;
-}
-
-
-/**
- * Parse a text using a syntax into a list of block tokens
- *
- * @param {Syntax} syntax
- * @param {String} text
- * @param {List} parents
- * @param {Object} ctx
- * @return {List}
- */
-function parseAsBlockTokens(syntax, text, parents, ctx) {
- text = cleanup(text);
-
- var blockRulesSet = syntax.getBlockRulesSet();
- var blockRules = blockRulesSet.getRules();
-
- // Parse block tokens
- var tokens = lex.block(blockRules, text, parents, ctx);
-
- // Parse inline content for each token
- tokens = tokens.map(function(token) {
- return parseInnerToken(syntax, token, parents, ctx);
- });
-
- return tokens;
-}
+var ParsingState = require('./state');
+var matchRule = require('./matchRule');
/**
- * Parse inner of a token according to the options.
- * It returns the modified token.
- *
- * @param {Syntax} syntax
- * @param {Token} token
- * @param {List} parents
- * @param {Object} ctx
- * @return {Token}
- */
-function parseInnerToken(syntax, token, parents, ctx) {
- var tokenType = token.getType();
- var rule = (token.isInline()?
- syntax.getInlineRule(tokenType) :
- syntax.getBlockRule(tokenType)
- );
- var parseMode = rule.getOption('parse');
-
- if (!parseMode) {
- return token;
- }
-
- // Add it to the new parents list
- parents = parents.push(token);
-
- // Parse inner tokens if none
- var tokens = token.getTokens();
- if (tokens.size === 0) {
- if (parseMode === 'block') {
- tokens = parseAsBlockTokens(syntax, token.getText(), parents, ctx);
- } else {
- tokens = parseAsInlineTokens(syntax, token.getText(), parents, ctx);
- }
-
- token = token.set('tokens', tokens);
- }
-
- token = token.set('text', getText(token));
-
- return token;
-}
-
-/**
- * Parse a text using a syntax into a Content
- *
- * @param {Syntax} syntax
- * @param {String} text
- * @param {Object} ctx
+ * Parse a text using a syntax
+ * @param {Syntax} syntax
+ * @param {String} text
* @return {Content}
*/
-function parseAsContent(syntax, text, ctx) {
- // Create a new context
- ctx = createContext(ctx);
-
- // Parse inline content for each token
- var tokens = parseAsBlockTokens(syntax, text, Immutable.List(), ctx);
-
- // We always return at least one block token
- if (tokens.size === 0) {
- tokens = tokens.push(Token.createBlockText(''));
- }
-
- var content = Content.createFromTokens(syntax.getName(), tokens);
- return finish(syntax, content, ctx);
-}
-
-/**
- * Parse an inline string to a Content
- *
- * @param {Syntax} syntax
- * @param {String} text
- * @param {Object} ctx
- * @return {Content}
- */
-function parseInline(syntax, text, ctx) {
- text = cleanup(text);
-
- var tokens = parseAsInlineTokens(syntax, text, Immutable.List(), ctx);
+function parse(syntax, text) {
+ var entryRule = syntax.getEntryRule();
+ var state = new ParsingState(syntax);
+ var tokens = matchRule(state, entryRule, text);
- var content = Content.createFromTokens(syntax.getName(), tokens);
- return finish(syntax, content, ctx);
+ return Content.createFromToken(syntax.getName(), tokens.first());
}
-module.exports = parseAsContent;
-module.exports.inline = parseInline;
+module.exports = parse;
diff --git a/lib/parse/lex.js b/lib/parse/lex.js
index f68f997e..0e4087c5 100644
--- a/lib/parse/lex.js
+++ b/lib/parse/lex.js
@@ -1,120 +1,60 @@
-var is = require('is');
var Immutable = require('immutable');
-var Token = require('../models/token');
-var BLOCKS = require('../constants/blocks');
-var STYLES = require('../constants/styles');
-var mergeTokens = require('./mergeTokens');
-
-/**
- * Convert a normal text into a list of unstyled tokens (block or inline)
- *
- * @param {String} text
- * @param {Boolean} inlineMode
- * @return {List}
- */
-function textToUnstyledTokens(text, inlineMode) {
- var accu = '', c, wasNewLine = false;
- var result = [];
-
- function pushAccu() {
- var token = inlineMode?
- Token.createInlineText(accu) :
- Token.createBlockText(accu);
-
- accu = '';
- if (token.getText().length !== 0) {
- result.push(token);
- }
- }
-
- for (var i = 0; i < text.length; i++) {
- c = text[i];
-
- if (c !== '\n' && wasNewLine) {
- pushAccu();
- }
-
- accu += c;
- wasNewLine = (c === '\n');
- }
-
- pushAccu();
-
- return new Immutable.List(result);
-}
+var textToUnstyledTokens = require('./textToUnstyledTokens');
+var matchRule = require('./matchRule');
/**
* Process a text using a set of rules
* to return a flat list of tokens
*
- * @param {Boolean} inlineMode
+ * @param {ParsingState} state
* @param {List} rules
+ * @param {Boolean} isInline
* @param {String} text
- * @param {List} parents
- * @param {Object} ctx
* @return {List}
*/
-function lex(inlineMode, rules, text, parents, ctx, nonParsed) {
- var parsedTokens;
- var tokens = new Immutable.List();
+function lex(state, rules, isInline, text, nonParsed) {
+ var tokens = Immutable.List();
+ var matchedTokens;
+
nonParsed = nonParsed || '';
if (!text) {
- return tokens.concat(textToUnstyledTokens(nonParsed, inlineMode));
+ return tokens.concat(textToUnstyledTokens(state, isInline, nonParsed));
}
rules.forEach(function(rule) {
- var matches = rule.onText(ctx, text, parents);
-
- if (!matches || matches.length === 0) return;
- if (!is.array(matches)) {
- matches = [matches];
+ matchedTokens = matchRule(state, rule, text);
+ if (!matchedTokens) {
+ return;
}
- parsedTokens = Immutable.List(matches)
- .map(function(match) {
- return Token.create(match.type, match);
- });
-
return false;
});
- // Nothing match this text, we move to the next character and try again
- // When found, we add a new token "unstyled"
- if (!parsedTokens) {
- nonParsed += text[0];
- text = text.substring(1);
-
- return lex(inlineMode, rules, text, parents, ctx, nonParsed);
- } else {
- tokens = tokens.concat(textToUnstyledTokens(nonParsed, inlineMode));
- parsedTokens.forEach(function(token) {
- // Push new token
- if (token.getType() != BLOCKS.IGNORE) tokens = tokens.push(token);
-
- // Update source text
- text = text.substring(token.getRaw().length);
- });
+ if (!matchedTokens) {
+ nonParsed += text[0];
+ text = text.substring(1);
- // Keep parsing
- tokens = tokens.concat(
- lex(inlineMode, rules, text, parents, ctx)
- );
+ return lex(state, rules, isInline, text, nonParsed);
}
- if (inlineMode) {
- tokens = mergeTokens(tokens, [
- STYLES.TEXT
- ]);
- }
+ var newText = matchedTokens.reduce(function(result, token) {
+ return result.substring(token.getRaw().length);
+ }, text);
+
+ // Keep parsing
+ tokens = textToUnstyledTokens(state, isInline, nonParsed)
+ .concat(
+ matchedTokens
+ )
+ .concat(
+ lex(state, rules, isInline, newText)
+ );
return tokens;
}
-module.exports = {
- inline: lex.bind(null, true),
- block: lex.bind(null, false)
-};
+module.exports = lex;
diff --git a/lib/parse/matchRule.js b/lib/parse/matchRule.js
new file mode 100644
index 00000000..f2036654
--- /dev/null
+++ b/lib/parse/matchRule.js
@@ -0,0 +1,30 @@
+var is = require('is');
+var Immutable = require('immutable');
+
+var Token = require('../models/token');
+
+/**
+ * Match a text using a rule
+ * @param {ParsingState} state
+ * @param {Rule} rule
+ * @param {String} text
+ * @return {List|null}
+ */
+function matchRule(state, rule, text) {
+ var matches = rule.onText(state, text);
+ var ruleType = rule.getType();
+
+ if (!matches) {
+ return;
+ }
+ if (!is.array(matches) && !Immutable.List.isList(matches)) {
+ matches = [matches];
+ }
+
+ return Immutable.List(matches)
+ .map(function(match) {
+ return Token.create(match.type || ruleType, match);
+ });
+}
+
+module.exports = matchRule;
diff --git a/lib/parse/state.js b/lib/parse/state.js
new file mode 100644
index 00000000..ffa40a1e
--- /dev/null
+++ b/lib/parse/state.js
@@ -0,0 +1,123 @@
+
+var STYLES = require('../constants/styles');
+var lex = require('./lex');
+var mergeTokens = require('./mergeTokens');
+
+function ParsingState(syntax) {
+ if (!(this instanceof ParsingState)) {
+ return new ParsingState(syntax);
+ }
+
+ this._ = {};
+ this.depth = 0;
+ this.syntax = syntax;
+}
+
+/**
+ * Get depth of parsing
+ * @return {Number}
+ */
+ParsingState.prototype.getDepth = function() {
+ return this.depth;
+};
+
+/**
+ * Get depth of parent token
+ * @return {Number}
+ */
+ParsingState.prototype.getParentDepth = function() {
+ return this.getDepth() - 1;
+};
+
+/**
+ * Get a state
+ * @param {String} key
+ * @return {Mixed}
+ */
+ParsingState.prototype.get = function(key) {
+ return this._[key];
+};
+
+/**
+ * Get a state
+ * @param {String} key
+ * @param {Mixed} value
+ * @return {Mixed}
+ */
+ParsingState.prototype.set = function(key, value) {
+ this._[key] = value;
+ return this;
+};
+
+/**
+ * Toggle a state and execute the function
+ * @param {String} key
+ * @param {[type]} [varname] [description]
+ * @return {Mixed}
+ */
+ParsingState.prototype.toggle = function(key, value, fn) {
+ if (!fn) {
+ fn = value;
+ value = this.depth;
+ }
+
+ var prevValue = this.get(key);
+
+ this._[key] = value;
+ var result = fn();
+ this._[key] = prevValue;
+
+ return result;
+};
+
+/**
+ * Parse a text using a set of rules
+ * @param {RulesSet} rules
+ * @param {Boolean} isInline
+ * @param {String} text
+ * @return {List}
+ */
+ParsingState.prototype.parse = function(rulesSet, isInline, text) {
+ this.depth++;
+
+ var rules = rulesSet.getRules();
+ var tokens = lex(this, rules, isInline, text);
+
+ if (isInline) {
+ tokens = mergeTokens(tokens, [
+ STYLES.TEXT
+ ]);
+ }
+
+ this.depth--;
+
+ return tokens;
+};
+
+/**
+ * Parse a text using inline rules
+ * @param {String} text
+ * @return {List}
+ */
+ParsingState.prototype.parseAsInline = function(text) {
+ return this.parse(
+ this.syntax.getInlineRulesSet(),
+ true,
+ text
+ );
+};
+
+/**
+ * Parse a text using inline rules
+ * @param {String} text
+ * @return {List}
+ */
+ParsingState.prototype.parseAsBlock = function(text) {
+ return this.parse(
+ this.syntax.getBlockRulesSet(),
+ false,
+ text
+ );
+};
+
+module.exports = ParsingState;
diff --git a/lib/parse/textToUnstyledTokens.js b/lib/parse/textToUnstyledTokens.js
new file mode 100644
index 00000000..97b9c6e8
--- /dev/null
+++ b/lib/parse/textToUnstyledTokens.js
@@ -0,0 +1,61 @@
+var Immutable = require('immutable');
+
+var defaultRules = require('../constants/defaultRules');
+var matchRule = require('./matchRule');
+
+/**
+ * Create a text token inline or block
+ *
+ * @param {ParsingState} state
+ * @param {Boolean} isInline
+ * @param {String} text
+ * @return {Token}
+ */
+function createTextToken(state, isInline, text) {
+ var rule = isInline? defaultRules.inlineRule : defaultRules.blockRule;
+ return matchRule(state, rule, text).get(0);
+}
+
+/**
+ * Convert a normal text into a list of unstyled tokens (block or inline)
+ *
+ * @param {ParsingState} state
+ * @param {Boolean} isInline
+ * @param {String} text
+ * @return {List}
+ */
+function textToUnstyledTokens(state, isInline, text) {
+ if (!text) {
+ return Immutable.List();
+ }
+
+ var accu = '', c, wasNewLine = false;
+ var result = [];
+
+ function pushAccu() {
+ var isEmpty = !(accu.trim())
+ var token = createTextToken(state, isInline, accu);
+ accu = '';
+
+ if (!isEmpty) {
+ result.push(token);
+ }
+ }
+
+ for (var i = 0; i < text.length; i++) {
+ c = text[i];
+
+ if (c !== '\n' && wasNewLine) {
+ pushAccu();
+ }
+
+ accu += c;
+ wasNewLine = (c === '\n');
+ }
+
+ pushAccu();
+
+ return new Immutable.List(result);
+}
+
+module.exports = textToUnstyledTokens;
diff --git a/lib/prosemirror/__tests__/specs.js b/lib/prosemirror/__tests__/specs.js
new file mode 100644
index 00000000..5f7a224d
--- /dev/null
+++ b/lib/prosemirror/__tests__/specs.js
@@ -0,0 +1,32 @@
+var fs = require('fs');
+var path = require('path');
+
+var encode = require('../encode');
+var JSONUtils = require('../../json');
+
+var FIXTURES = path.resolve(__dirname, 'specs');
+
+var files = fs.readdirSync(FIXTURES);
+
+describe('encode', function() {
+ files.forEach(function(file) {
+ if (path.extname(file) !== '.js') return;
+
+ it(file, function () {
+ var content = require(path.join(FIXTURES, file));
+ var contentState = JSONUtils.decode(content.json);
+
+ encode(contentState).should.deepEqual(content.prosemirror);
+ });
+ });
+});
+
+describe('decode', function() {
+ files.forEach(function(file) {
+ if (path.extname(file) !== '.md') return;
+
+ it(file, function () {
+
+ });
+ });
+});
diff --git a/lib/prosemirror/__tests__/specs/codeBlocks.js b/lib/prosemirror/__tests__/specs/codeBlocks.js
new file mode 100644
index 00000000..30b38aa3
--- /dev/null
+++ b/lib/prosemirror/__tests__/specs/codeBlocks.js
@@ -0,0 +1,69 @@
+module.exports = {
+ json: {
+ 'syntax': 'markdown',
+ 'token': {
+ 'type': 'doc',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'paragraph',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': 'Hello World',
+ 'data': {},
+ 'tokens': []
+ }
+ ]
+ },
+ {
+ 'type': 'code_block',
+ 'data': {
+ 'syntax': 'js'
+ },
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': 'var a = \'test\'\n',
+ 'data': {},
+ 'tokens': []
+ }
+ ]
+ }
+ ]
+ }
+ },
+ prosemirror: {
+ 'type': 'doc',
+ 'attrs': {},
+ 'content': [
+ {
+ 'type': 'paragraph',
+ 'attrs': {},
+ 'content': [
+ {
+ 'type': 'text',
+ 'text': 'Hello World',
+ 'attrs': {},
+ 'marks': []
+ }
+ ]
+ },
+ {
+ 'type': 'code_block',
+ 'attrs': {
+ 'syntax': 'js'
+ },
+ 'content': [
+ {
+ 'type': 'text',
+ 'text': 'var a = \'test\'\n',
+ 'attrs': {},
+ 'marks': []
+ }
+ ]
+ }
+ ]
+ }
+};
\ No newline at end of file
diff --git a/lib/prosemirror/__tests__/specs/imageAndStyles.js b/lib/prosemirror/__tests__/specs/imageAndStyles.js
new file mode 100644
index 00000000..d1382cef
--- /dev/null
+++ b/lib/prosemirror/__tests__/specs/imageAndStyles.js
@@ -0,0 +1,216 @@
+module.exports = {
+ prosemirror: {
+ 'type': 'doc',
+ 'attrs': {},
+ 'content': [
+ {
+ 'type': 'header_one',
+ 'attrs': {
+ 'id': null
+ },
+ 'content': [
+ {
+ 'type': 'text',
+ 'text': 'Hello',
+ 'attrs': {},
+ 'marks': []
+ }
+ ]
+ },
+ {
+ 'type': 'paragraph',
+ 'attrs': {},
+ 'content': [
+ {
+ 'type': 'text',
+ 'text': 'Hello ',
+ 'attrs': {},
+ 'marks': []
+ },
+ {
+ 'type': 'text',
+ 'text': 'Wo',
+ 'attrs': {},
+ 'marks': [
+ {
+ 'href': 'a.md',
+ '_': 'link'
+ },
+ {
+ '_': 'BOLD'
+ }
+ ]
+ },
+ {
+ 'type': 'text',
+ 'text': 'rld',
+ 'attrs': {},
+ 'marks': [
+ {
+ 'href': 'a.md',
+ '_': 'link'
+ },
+ {
+ '_': 'BOLD'
+ },
+ {
+ '_': 'ITALIC'
+ }
+ ]
+ },
+ {
+ 'type': 'text',
+ 'text': ' ',
+ 'attrs': {},
+ 'marks': []
+ },
+ {
+ 'type': 'text',
+ 'text': ' ',
+ 'attrs': {},
+ 'marks': [
+ {
+ '_': 'BOLD'
+ }
+ ]
+ },
+ {
+ 'type': 'image',
+ 'text': '',
+ 'attrs': {
+ 'alt': '',
+ 'src': 'test.png'
+ },
+ 'marks': [
+ {
+ '_': 'BOLD'
+ }
+ ]
+ },
+ {
+ 'type': 'text',
+ 'text': ' ',
+ 'attrs': {},
+ 'marks': [
+ {
+ '_': 'BOLD'
+ }
+ ]
+ },
+ {
+ 'type': 'text',
+ 'text': '.',
+ 'attrs': {},
+ 'marks': []
+ }
+ ]
+ }
+ ]
+ },
+ json: {
+ 'syntax': 'prosemirror',
+ 'token': {
+ 'type': 'doc',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'header_one',
+ 'data': {
+ 'id': null
+ },
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': 'Hello',
+ 'data': {},
+ 'tokens': []
+ }
+ ]
+ },
+ {
+ 'type': 'paragraph',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': 'Hello ',
+ 'data': {},
+ 'tokens': []
+ },
+ {
+ 'type': 'link',
+ 'data': {
+ 'href': 'a.md'
+ },
+ 'tokens': [
+ {
+ 'type': 'BOLD',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': 'Wo',
+ 'data': {},
+ 'tokens': []
+ },
+ {
+ 'type': 'ITALIC',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': 'rld',
+ 'data': {},
+ 'tokens': []
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ {
+ 'type': 'text',
+ 'text': ' ',
+ 'data': {},
+ 'tokens': []
+ },
+ {
+ 'type': 'BOLD',
+ 'data': {},
+ 'tokens': [
+ {
+ 'type': 'text',
+ 'text': ' ',
+ 'data': {},
+ 'tokens': []
+ },
+ {
+ 'type': 'image',
+ 'text': '',
+ 'data': {
+ 'alt': '',
+ 'src': 'test.png'
+ },
+ 'tokens': []
+ },
+ {
+ 'type': 'text',
+ 'text': ' ',
+ 'data': {},
+ 'tokens': []
+ }
+ ]
+ },
+ {
+ 'type': 'text',
+ 'text': '.',
+ 'data': {},
+ 'tokens': []
+ }
+ ]
+ }
+ ]
+ }
+ }
+};
diff --git a/lib/prosemirror/decode.js b/lib/prosemirror/decode.js
new file mode 100644
index 00000000..e8f0f83c
--- /dev/null
+++ b/lib/prosemirror/decode.js
@@ -0,0 +1,67 @@
+var Immutable = require('immutable');
+
+var Content = require('../models/content');
+var Token = require('../models/token');
+
+/**
+ * Decode marks as tokens
+ *
+ * @param {Array} marks
+ * @param {String} text
+ * @return {Token}
+ */
+function decodeMarksAsToken(marks, text) {
+ return marks.reduce(function(child, mark) {
+ return new Token({
+ type: mark._,
+ data: Immutable.Map(mark).delete('_'),
+ tokens: [child]
+ });
+
+ }, Token.createText(text));
+}
+
+/**
+ * Decode a token
+ *
+ * @paran {Object} json
+ * @return {Token}
+ */
+function decodeTokenFromJSON(json) {
+ if (json.marks) {
+ return decodeMarksAsToken(json.marks, json.text);
+ }
+
+ return new Token({
+ type: json.type,
+ text: json.text,
+ raw: json.raw,
+ data: json.attrs,
+ tokens: decodeTokensFromJSON(json.content || [])
+ });
+}
+
+/**
+ * Decode a list of tokens
+ *
+ * @paran {Object} json
+ * @return {List}
+ */
+function decodeTokensFromJSON(json) {
+ return new Immutable.List(json.map(decodeTokenFromJSON));
+}
+
+/**
+ * Decode a JSON into a Content
+ *
+ * @paran {Object} json
+ * @return {Content}
+ */
+function decodeContentFromProseMirror(json) {
+ return Content.createFromToken(
+ 'prosemirror',
+ decodeTokenFromJSON(json)
+ );
+}
+
+module.exports = decodeContentFromProseMirror;
diff --git a/lib/prosemirror/encode.js b/lib/prosemirror/encode.js
new file mode 100644
index 00000000..75a0bada
--- /dev/null
+++ b/lib/prosemirror/encode.js
@@ -0,0 +1,122 @@
+var Immutable = require('immutable');
+
+var BLOCKS = require('../constants/blocks');
+var Token = require('../models/token');
+var MARK_TYPES = require('./markTypes');
+
+/**
+ * Encode data of a token, it ignores undefined value
+ *
+ * @paran {Map} data
+ * @return {Object}
+ */
+function encodeDataToAttrs(data) {
+ return data
+ .filter(function(value, key) {
+ return (value !== undefined);
+ })
+ .toJS();
+}
+
+/**
+ * Encode token as a mark
+ * @param {Token} token
+ * @return {Object}
+ */
+function tokenToMark(token) {
+ var data = token.getData();
+
+ return data
+ .merge({
+ '_': token.getType()
+ })
+ .toJS();
+}
+
+/**
+ * Encode an inline token
+ * @paran {Token} tokens
+ * @param {List}
+ * @return {Array}
+ */
+function encodeInlineTokenToJSON(token, marks) {
+ marks = marks || Immutable.List();
+
+ if (!MARK_TYPES.includes(token.getType())) {
+ return [
+ {
+ type: token.getType(),
+ text: token.getText(),
+ attrs: encodeDataToAttrs(token.getData()),
+ marks: marks.toJS()
+ }
+ ];
+ }
+
+ var mark = tokenToMark(token);
+ var innerMarks = marks.push(mark);
+ var tokens = token.getTokens();
+
+ return tokens
+ .reduce(function(accu, token) {
+ return accu.concat(encodeInlineTokenToJSON(token, innerMarks));
+ }, []);
+}
+
+/**
+ * Encode a block token
+ * @paran {Token} tokens
+ * @return {Object}
+ */
+function encodeBlockTokenToJSON(token) {
+ return {
+ type: token.getType(),
+ attrs: encodeDataToAttrs(token.getData()),
+ content: encodeTokensToJSON(token.getTokens())
+ };
+}
+
+/**
+ * Encode a token to JSON
+ *
+ * @paran {Token} tokens
+ * @return {Array}
+ */
+function encodeTokenToJSON(token) {
+ if (token.isInline()) {
+ return encodeInlineTokenToJSON(token);
+ } else {
+ return [encodeBlockTokenToJSON(token)];
+ }
+}
+
+/**
+ * Encode a list of tokens to JSON
+ *
+ * @paran {List} tokens
+ * @return {Array}
+ */
+function encodeTokensToJSON(tokens) {
+ return tokens.reduce(function(accu, token) {
+ return accu.concat(encodeTokenToJSON(token));
+ }, []);
+}
+
+/**
+ * Encode a Content into a ProseMirror JSON object
+ *
+ * @paran {Content} content
+ * @return {Object}
+ */
+function encodeContentToProseMirror(content) {
+ var doc = encodeTokenToJSON(content.getToken())[0];
+
+ // ProseMirror crash with empty doc node
+ if (doc.content.length === 0) {
+ doc.content = encodeTokenToJSON(Token.create(BLOCKS.PARAGRAPH));
+ }
+
+ return doc;
+}
+
+module.exports = encodeContentToProseMirror;
diff --git a/lib/prosemirror/index.js b/lib/prosemirror/index.js
new file mode 100644
index 00000000..22bb0fda
--- /dev/null
+++ b/lib/prosemirror/index.js
@@ -0,0 +1,5 @@
+
+module.exports = {
+ encode: require('./encode'),
+ decode: require('./decode')
+};
diff --git a/lib/prosemirror/markTypes.js b/lib/prosemirror/markTypes.js
new file mode 100644
index 00000000..b7f8a255
--- /dev/null
+++ b/lib/prosemirror/markTypes.js
@@ -0,0 +1,13 @@
+var Immutable = require('immutable');
+
+var STYLES = require('../constants/styles');
+var ENTITIES = require('../constants/entities');
+
+module.exports = Immutable.List([
+ STYLES.BOLD,
+ STYLES.ITALIC,
+ STYLES.CODE,
+ STYLES.STRIKETHROUGH,
+ ENTITIES.LINK
+]);
+
diff --git a/lib/render/index.js b/lib/render/index.js
index 9fcb3f9f..73f1a5f3 100644
--- a/lib/render/index.js
+++ b/lib/render/index.js
@@ -1,14 +1,4 @@
-/**
- * Return context to describe a token
- */
-function getTokenCtx(token) {
- return {
- type: token.getType(),
- text: token.getText(),
- raw: token.getRaw(),
- data: token.getData().toJS()
- };
-}
+var RenderingState = require('./state');
/**
* Render a Content instance using a syntax
@@ -16,45 +6,11 @@ function getTokenCtx(token) {
* @return {String}
*/
function render(syntax, content) {
- var ctx = {};
-
- function _renderTokens(tokens, depth) {
- depth = depth || 0;
-
- return tokens.map(function(token, i) {
- var tokenType = token.getType();
- var innerTokens = token.getTokens();
- var rule, renderInner, innerText;
-
- if (token.isInline()) {
- rule = syntax.getInlineRule(tokenType);
- } else {
- rule = syntax.getBlockRule(tokenType);
- }
-
- if (rule.getOption('renderInner') && innerTokens.size > 0) {
- renderInner = true;
- }
-
- if (renderInner) {
- innerText = _renderTokens(innerTokens, depth + 1);
- } else {
- innerText = token.getText();
- }
-
- // Create context to describe a token
- var prevToken = i > 0? tokens.get(i - 1) : null;
- var nextToken = i < (tokens.size - 1)? tokens.get(i + 1) : null;
-
- var tokenCtx = getTokenCtx(token);
- tokenCtx.next = nextToken? getTokenCtx(nextToken) : null;
- tokenCtx.prev = prevToken? getTokenCtx(prevToken) : null;
-
- return rule.onToken(ctx, innerText, tokenCtx);
- }).join('');
- }
+ var state = new RenderingState(syntax);
+ var entryRule = syntax.getEntryRule();
+ var token = content.getToken();
- return _renderTokens(content.getTokens());
+ return entryRule.onToken(state, token);
}
module.exports = render;
diff --git a/lib/render/state.js b/lib/render/state.js
new file mode 100644
index 00000000..3208c763
--- /dev/null
+++ b/lib/render/state.js
@@ -0,0 +1,52 @@
+var Token = require('../models/token');
+
+function RenderingState(syntax) {
+ if (!(this instanceof RenderingState)) {
+ return new RenderingState(syntax);
+ }
+
+ this.syntax = syntax;
+}
+
+/**
+ * Render a token using a set of rules
+ * @param {RulesSet} rules
+ * @param {Boolean} isInline
+ * @param {Token|List} tokens
+ * @return {List}
+ */
+RenderingState.prototype.render = function(tokens) {
+ var state = this;
+ var syntax = this.syntax;
+
+ if (tokens instanceof Token) {
+ var token = tokens;
+ tokens = token.getTokens();
+
+ if (tokens.size === 0) {
+ return token.getAsPlainText();
+ }
+ }
+
+ return tokens.reduce(function(text, token) {
+ var tokenType = token.getType();
+ var rule = (token.isInline()? syntax.getInlineRule(tokenType)
+ : syntax.getBlockRule(tokenType));
+
+ if (!rule) {
+ throw new Error('Unexpected error: no rule to render "' + tokenType + '"');
+ }
+
+ return text + rule.onToken(state, token);
+ }, '');
+};
+
+RenderingState.prototype.renderAsInline = function(tokens) {
+ return this.render(tokens);
+};
+
+RenderingState.prototype.renderAsBlock = function(tokens) {
+ return this.render(tokens);
+};
+
+module.exports = RenderingState;
diff --git a/lib/utils/getText.js b/lib/utils/getText.js
deleted file mode 100644
index b71c0dbd..00000000
--- a/lib/utils/getText.js
+++ /dev/null
@@ -1,19 +0,0 @@
-
-/**
- * Extract plain text from a token.
- *
- * @param {Token}
- * @return {String}
- */
-function getText(token) {
- var tokens = token.getTokens();
- if (tokens.size === 0) {
- return token.getText();
- }
-
- return tokens.reduce(function(result, tok) {
- return result + tok.getText();
- }, '');
-}
-
-module.exports = getText;
diff --git a/lib/utils/transform.js b/lib/utils/transform.js
index a1f86cbb..63113ad3 100644
--- a/lib/utils/transform.js
+++ b/lib/utils/transform.js
@@ -1,25 +1,62 @@
+var Immutable = require('immutable');
var Content = require('../models/content');
/**
* Walk throught the children tokens tree, and
* map each token using a transformation
*
- * @param {Token} base
+ * The transformation iterator can return a list, a new token or undefined.
+ *
+ * @param {Token|Content} base
* @param {Function(token, depth)} iter
* @param {Number} depth
* @return {Token}
*/
-function transform(base, iter, depth) {
+function transformToken(base, iter, depth) {
depth = depth || 0;
var tokens = base.getTokens();
+ var newTokens = transformTokens(tokens, iter, depth);
+ base = base.set('tokens', newTokens);
- tokens = tokens.map(function(token) {
- return transform(token, iter, depth + 1);
- });
+ return (base instanceof Content)? base : iter(base, depth);
+}
- base = base.set('tokens', tokens);
+/**
+ * Transform a list of tokens
+ * @param {List} tokens
+ * @param {Function} iter
+ * @param {Number} depth
+ * @return {List}
+ */
+function transformTokens(tokens, iter, depth) {
+ return tokens
+ .reduce(function(list, token) {
+ var result = transform(token, iter, depth + 1);
- return (base instanceof Content)? base : iter(base, depth);
+ if (Immutable.List.isList(result)) {
+ return list.concat(result);
+ }
+ else if (result) {
+ return list.push(result);
+ }
+
+ return list;
+ }, Immutable.List());
+}
+
+
+/**
+ * Transform that works on token or list of tokens
+ * @param {Token|List|Content} base
+ * @param {Function} iter
+ * @return {Token|List|Content}
+ */
+function transform(base, iter) {
+ if (Immutable.List.isList(base)) {
+ return transformTokens(base, iter);
+ } else {
+ return transformToken(base, iter);
+ }
}
module.exports = transform;
diff --git a/lib/utils/walk.js b/lib/utils/walk.js
index e6e0acd8..709766e2 100644
--- a/lib/utils/walk.js
+++ b/lib/utils/walk.js
@@ -15,6 +15,8 @@ function walk(base, iter) {
var tokens = base.getTokens();
if (tokens.size === 0) {
+ iter(base, Range(0, base.getText().length));
+
return base.getText();
}
diff --git a/package.json b/package.json
index a4006c85..c8342a51 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
"main": "lib/index.js",
"scripts": {
"lint": "eslint .",
- "test": "./node_modules/.bin/mocha ./testing/setup.js \"./lib/**/*/__tests__/*.js\" \"./lib/__tests__/*.js\" \"./syntaxes/**/*/__tests__/*.js\" --reporter=list --timeout=10000"
+ "test": "./node_modules/.bin/mocha ./testing/setup.js \"./lib/**/*/__tests__/*.js\" \"./lib/__tests__/*.js\" \"./syntaxes/**/*/__tests__/*.js\" --bail --reporter=list --timeout=10000"
},
"repository": {
"type": "git",
diff --git a/syntaxes/gitbook/__tests__/index.js b/syntaxes/gitbook/__tests__/index.js
index d0668169..70600067 100644
--- a/syntaxes/gitbook/__tests__/index.js
+++ b/syntaxes/gitbook/__tests__/index.js
@@ -7,29 +7,32 @@ describe('GitBook Markdown', function() {
describe('Math', function() {
it('should parse a block', function() {
var content = markup.toContent('$$\na = b\n$$');
- var blocks = content.getTokens();
+ var math = content.getToken().getTokens().get(0);
- blocks.size.should.equal(1);
- blocks.get(0).getText().should.equal('a = b');
- blocks.get(0).getType().should.equal('math');
+ math.getData().get('tex').should.equal('a = b');
+ math.getType().should.equal(MarkupIt.BLOCKS.MATH);
});
it('should parse inline math', function() {
var content = markup.toContent('$$a = b$$');
- var blocks = content.getTokens();
+ var p = content.getToken().getTokens().get(0);
- blocks.size.should.equal(1);
- blocks.get(0).getText().should.equal('a = b');
- blocks.get(0).getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH);
+ p.getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH);
+
+ var math = p.getTokens().get(0);
+ math.getData().get('tex').should.equal('a = b');
+ math.getType().should.equal(MarkupIt.ENTITIES.MATH);
});
it('should parse inline math with text', function() {
var content = markup.toContent('Here are some math $$a = b$$, awesome!');
- var blocks = content.getTokens();
+ var p = content.getToken().getTokens().get(0);
+
+ p.getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH);
- blocks.size.should.equal(1);
- blocks.get(0).getText().should.equal('Here are some math a = b, awesome!');
- blocks.get(0).getType().should.equal(MarkupIt.BLOCKS.PARAGRAPH);
+ var math = p.getTokens().get(1);
+ math.getData().get('tex').should.equal('a = b');
+ math.getType().should.equal(MarkupIt.ENTITIES.MATH);
});
});
});
diff --git a/syntaxes/gitbook/index.js b/syntaxes/gitbook/index.js
index 2bcaa4d1..117036c5 100644
--- a/syntaxes/gitbook/index.js
+++ b/syntaxes/gitbook/index.js
@@ -6,39 +6,41 @@ var reMathBlock = /^\$\$\n([^$]+)\n\$\$/;
var reTpl = /^{([#%{])\s*(.*?)\s*(?=[#%}]})}}/;
var inlineMathRule = markup.Rule(markup.ENTITIES.MATH)
- .setOption('parse', false)
- .setOption('renderInner', false)
- .regExp(reMathInline, function(match) {
+ .regExp(reMathInline, function(state, match) {
var text = match[1];
- if (text.trim().length == 0) return;
+ if (text.trim().length == 0) {
+ return;
+ }
return {
- text: text,
- data: {}
+ data: {
+ tex: text
+ }
};
})
- .toText(function(text, block) {
- return '$$' + text + '$$';
+ .toText(function(state, token) {
+ return '$$' + token.getData().get('tex') + '$$';
});
var blockMathRule = markup.Rule(markup.BLOCKS.MATH)
- .setOption('parse', false)
- .setOption('renderInner', false)
- .regExp(reMathBlock, function(match) {
+ .regExp(reMathBlock, function(state, match) {
var text = match[1];
- if (text.trim().length == 0) return;
+ if (text.trim().length == 0) {
+ return;
+ }
return {
- text: text
+ data: {
+ tex: text
+ }
};
})
- .toText(function(text, block) {
- return '$$\n' + text + '\n$$\n\n';
+ .toText(function(state, token) {
+ return '$$\n' + token.getData().get('tex') + '\n$$\n\n';
});
var tplExpr = markup.Rule(markup.STYLES.TEMPLATE)
- .setOption('parse', false)
- .regExp(reTpl, function(match) {
+ .regExp(reTpl, function(state, match) {
var type = match[0];
var text = match[2];
@@ -47,14 +49,18 @@ var tplExpr = markup.Rule(markup.STYLES.TEMPLATE)
else if (type == '{') type = 'var';
return {
- text: text,
data: {
type: type
- }
+ },
+ tokens: [
+ MarkupIt.Token.createText(text)
+ ]
};
})
- .toText(function(text, block) {
- var type = block.data.type;
+ .toText(function(sttae, token) {
+ var data = token.getData();
+ var text = token.getAsPlainText();
+ var type = data.get('type');
if (type == 'expr') text = '{% ' + text + ' %}';
else if (type == 'comment') text = '{# ' + text + ' #}';
diff --git a/syntaxes/html/blocks.js b/syntaxes/html/blocks.js
index 25dcd17e..583d460f 100644
--- a/syntaxes/html/blocks.js
+++ b/syntaxes/html/blocks.js
@@ -7,17 +7,15 @@ var listRules = require('./list');
var utils = require('./utils');
/*
- Generate an heading rule for a specific level
-*/
+ * Generate an heading rule for a specific level
+ */
function headingRule(n) {
var type = MarkupIt.BLOCKS['HEADING_' + n];
- return HTMLRule(type, 'h'+n);
+ return HTMLRule(type, 'h' + n);
}
module.exports = [
tableRules.block,
- tableRules.header,
- tableRules.body,
tableRules.row,
tableRules.cell,
@@ -36,8 +34,10 @@ module.exports = [
HTMLRule(MarkupIt.BLOCKS.BLOCKQUOTE, 'blockquote'),
MarkupIt.Rule(MarkupIt.BLOCKS.FOOTNOTE)
- .toText(function(text, token) {
- var refname = token.data.id;
+ .toText(function(state, token) {
+ var data = token.getData();
+ var text = state.renderAsInline(token);
+ var refname = data.get('id');
return '\n'
+ '' + refname + ' . '
@@ -50,11 +50,14 @@ module.exports = [
.toText('%s\n\n'),
MarkupIt.Rule(MarkupIt.BLOCKS.CODE)
- .toText(function(text, token) {
- var attr = '';
+ .toText(function(state, token) {
+ var attr = '';
+ var data = token.getData();
+ var text = token.getAsPlainText();
+ var syntax = data.get('syntax');
- if (token.data.syntax) {
- attr = ' class="lang-' + token.data.syntax +'"';
+ if (syntax) {
+ attr = ' class="lang-' + syntax +'"';
}
return '' + utils.escape(text) + '
\n';
diff --git a/syntaxes/html/index.js b/syntaxes/html/index.js
index 122041b0..f20d4577 100644
--- a/syntaxes/html/index.js
+++ b/syntaxes/html/index.js
@@ -1,6 +1,20 @@
-var markup = require('../../');
+var MarkupIt = require('../../');
+var htmlToTokens = require('./parse');
+
+var documentRule = MarkupIt.Rule(MarkupIt.BLOCKS.DOCUMENT)
+ .match(function(state, text) {
+ return {
+ tokens: htmlToTokens(text)
+ };
+ })
+ .toText(function(state, token) {
+ return state.renderAsBlock(token);
+ });
+
+
+module.exports = MarkupIt.Syntax('html', {
+ entryRule: documentRule,
-module.exports = markup.Syntax('html', {
// List of rules for parsing blocks
inline: require('./inline'),
diff --git a/syntaxes/html/inline.js b/syntaxes/html/inline.js
index 67ff100a..77ae0e6e 100644
--- a/syntaxes/html/inline.js
+++ b/syntaxes/html/inline.js
@@ -6,13 +6,14 @@ var HTMLRule = require('./rule');
module.exports = [
// ---- TEXT ----
markup.Rule(markup.STYLES.TEXT)
- .setOption('parse', false)
- .toText(utils.escape),
+ .toText(function(state, token) {
+ return utils.escape(token.getAsPlainText());
+ }),
// ---- CODE ----
markup.Rule(markup.STYLES.CODE)
- .toText(function(text, token) {
- return '' + utils.escape(text) + '
';
+ .toText(function(state, token) {
+ return '' + utils.escape(token.getAsPlainText()) + '
';
}),
// ---- BOLD ----
@@ -31,19 +32,20 @@ module.exports = [
HTMLRule(markup.ENTITIES.LINK, 'a', function(data) {
return {
title: data.title? utils.escape(data.title) : undefined,
- href: utils.escape(data.href || '')
+ href: utils.escape(data.href || '')
};
}),
// ---- FOOTNOTE ----
markup.Rule(markup.ENTITIES.FOOTNOTE_REF)
- .toText(function(refname, token) {
+ .toText(function(state, token) {
+ var refname = token.getAsPlainText();
return '' + refname + ' ';
}),
// ---- HTML ----
markup.Rule(markup.STYLES.HTML)
- .toText(function(text, token) {
- return text;
+ .toText(function(state, token) {
+ return token.getAsPlainText();
})
];
diff --git a/syntaxes/html/list.js b/syntaxes/html/list.js
index 2caea678..81e217da 100644
--- a/syntaxes/html/list.js
+++ b/syntaxes/html/list.js
@@ -1,43 +1,28 @@
var MarkupIt = require('../../');
-/*
- Render a list item
-
- @param {String} text
- @param {Token} token
- @return {String}
-*/
-function renderListItem(text, token) {
- var isOrdered = token.type == MarkupIt.BLOCKS.OL_ITEM;
- var listTag = isOrdered? 'ol' : 'ul';
- var depth = token.data.depth;
-
- var prevToken = token.prev? token.prev.type : null;
- var nextToken = token.next? token.next.type : null;
-
- var prevTokenDepth = token.prev? token.prev.data.depth : 0;
- var nextTokenDepth = token.next? token.next.data.depth : 0;
-
- var output = '';
-
- if (prevToken != token.type || prevTokenDepth < depth) {
- output += '<' + listTag + '>\n';
- }
-
- output += '' + text + ' \n';
-
- if (nextToken != token.type || nextTokenDepth < depth) {
- output += '' + listTag + '>\n';
- }
-
- return output;
+/**
+ * Render a list item
+ *
+ * @param {String} text
+ * @param {Token} token
+ * @return {String}
+ */
+function renderListItem(state, token) {
+ var isOrdered = (token.type == MarkupIt.BLOCKS.OL_LIST);
+ var listTag = isOrdered? 'ol' : 'ul';
+ var items = token.getTokens();
+
+ return '<' + listTag + '>' +
+ items.map(function(item) {
+ return '' + state.render(item) + ' ';
+ }).join('\n')
+ + '' + listTag + '>\n';
}
-
-var ruleOL = MarkupIt.Rule(MarkupIt.BLOCKS.OL_ITEM)
+var ruleOL = MarkupIt.Rule(MarkupIt.BLOCKS.OL_LIST)
.toText(renderListItem);
-var ruleUL = MarkupIt.Rule(MarkupIt.BLOCKS.UL_ITEM)
+var ruleUL = MarkupIt.Rule(MarkupIt.BLOCKS.UL_LIST)
.toText(renderListItem);
module.exports = {
diff --git a/syntaxes/html/parse.js b/syntaxes/html/parse.js
index f7fbf81c..1442646d 100644
--- a/syntaxes/html/parse.js
+++ b/syntaxes/html/parse.js
@@ -1,38 +1,37 @@
var Immutable = require('immutable');
var htmlparser = require('htmlparser2');
-var markup = require('../../');
+var MarkupIt = require('../../');
var TAGS_TO_TYPE = {
- a: markup.ENTITIES.LINK,
- img: markup.ENTITIES.IMAGE,
+ a: MarkupIt.ENTITIES.LINK,
+ img: MarkupIt.ENTITIES.IMAGE,
- h1: markup.BLOCKS.HEADING_1,
- h2: markup.BLOCKS.HEADING_2,
- h3: markup.BLOCKS.HEADING_3,
- h4: markup.BLOCKS.HEADING_4,
- h5: markup.BLOCKS.HEADING_5,
- h6: markup.BLOCKS.HEADING_6,
- pre: markup.BLOCKS.CODE,
- blockquote: markup.BLOCKS.BLOCKQUOTE,
- p: markup.BLOCKS.PARAGRAPH,
- hr: markup.BLOCKS.HR,
+ h1: MarkupIt.BLOCKS.HEADING_1,
+ h2: MarkupIt.BLOCKS.HEADING_2,
+ h3: MarkupIt.BLOCKS.HEADING_3,
+ h4: MarkupIt.BLOCKS.HEADING_4,
+ h5: MarkupIt.BLOCKS.HEADING_5,
+ h6: MarkupIt.BLOCKS.HEADING_6,
+ pre: MarkupIt.BLOCKS.CODE,
+ blockquote: MarkupIt.BLOCKS.BLOCKQUOTE,
+ p: MarkupIt.BLOCKS.PARAGRAPH,
+ hr: MarkupIt.BLOCKS.HR,
- table: markup.BLOCKS.TABLE,
- tr: markup.BLOCKS.TR_ROW,
- td: markup.BLOCKS.TABLE_CELL,
+ table: MarkupIt.BLOCKS.TABLE,
+ tr: MarkupIt.BLOCKS.TR_ROW,
+ td: MarkupIt.BLOCKS.TABLE_CELL,
- b: markup.INLINES.BOLD,
- strike: markup.INLINES.STRIKETHROUGH,
- em: markup.INLINES.ITALIC,
- code: markup.INLINES.CODE
+ b: MarkupIt.STYLES.BOLD,
+ strike: MarkupIt.STYLES.STRIKETHROUGH,
+ em: MarkupIt.STYLES.ITALIC,
+ code: MarkupIt.STYLES.CODE
};
/**
- Parse an HTML string into a token tree
-
- @param {String} str
- @return {List}
-*/
+ * Parse an HTML string into a token tree
+ * @param {String} str
+ * @return {List}
+ */
function htmlToTokens(str) {
var accuText = '';
var result = [];
@@ -50,8 +49,7 @@ function htmlToTokens(str) {
return;
}
- var token = new markup.Token({
- type: type,
+ var token = MarkupIt.Token.create(type, {
text: accuText
});
diff --git a/syntaxes/html/rule.js b/syntaxes/html/rule.js
index 4506cbbf..fd248edc 100644
--- a/syntaxes/html/rule.js
+++ b/syntaxes/html/rule.js
@@ -1,15 +1,14 @@
var is = require('is');
-var markup = require('../../');
+var MarkupIt = require('../../');
var identity = require('../../lib/utils/identity');
var SINGLE_TAG = ['img', 'hr'];
/**
- Convert a map of attributes into a string
-
- @param {Object} attrs
- @return {String}
+ * Convert a map of attributes into a string
+ * @param {Object} attrs
+ * @return {String}
*/
function attrsToString(attrs) {
var output = '', value;
@@ -25,20 +24,22 @@ function attrsToString(attrs) {
} else {
output += ' ' + key + '=' + JSON.stringify(value);
}
-
-
}
return output;
}
+
function HTMLRule(type, tag, getAttrs) {
getAttrs = getAttrs || identity;
var isSingleTag = SINGLE_TAG.indexOf(tag) >= 0;
- return markup.Rule(type)
- .toText(function(text, token) {
- var attrs = getAttrs(token.data, token);
+ return MarkupIt.Rule(type)
+ .toText(function(state, token) {
+ var text = state.render(token);
+ var data = token.getData().toJS();
+ var attrs = getAttrs(data, token);
+
var output = '<' + tag + attrsToString(attrs) + (isSingleTag? '/>' : '>');
if (!isSingleTag) {
diff --git a/syntaxes/html/table.js b/syntaxes/html/table.js
index 2e124a30..a2aec9bc 100644
--- a/syntaxes/html/table.js
+++ b/syntaxes/html/table.js
@@ -1,46 +1,54 @@
var MarkupIt = require('../../');
var blockRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE)
- .toText(function(innerHTML) {
- this._rowIndex = 0;
+ .toText(function(state, token) {
+ state._rowIndex = 0;
- return '\n\n';
- });
+ var data = token.getData();
+ var rows = token.getTokens();
+ var align = data.get('align');
-var headerRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_HEADER)
- .toText(function(innerHTML) {
- return '\n' + innerHTML + ' ';
- });
+ var headerRows = rows.slice(0, 1);
+ var bodyRows = rows.slice(1);
+
+ state._tableAlign = align;
-var bodyRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_BODY)
- .toText(function(innerHTML) {
- return '\n' + innerHTML + ' ';
+ var headerText = state.render(headerRows);
+ var bodyText = state.render(bodyRows);
+
+ return '\n'
+ + '\n' + headerText + ' '
+ + '\n' + bodyText + ' '
+ + '
\n\n';
});
var rowRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_ROW)
- .toText(function(innerHTML) {
- this._rowIndex = (this._rowIndex || 0) + 1;
+ .toText(function(state, token) {
+ var innerContent = state.render(token);
+ state._rowIndex = state._rowIndex + 1;
+ state._cellIndex = 0;
- return '' + innerHTML + ' ';
+ return '' + innerContent + ' ';
});
var cellRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_CELL)
- .toText(function(innerHTML, token) {
- var isHeader = (this._rowIndex || 0) === 0;
- var align = token.data.align;
+ .toText(function(state, token) {
+ var align = state._tableAlign[state._cellIndex];
+ var isHeader = (state._rowIndex || 0) === 0;
+ var innerHTML = state.render(token);
var type = isHeader ? 'th' : 'td';
var tag = align
? '<' + type + ' style="text-align:' + align + '">'
: '<' + type + '>';
+ state._cellIndex = state._cellIndex + 1;
+
return tag + innerHTML + '' + type + '>\n';
});
module.exports = {
block: blockRule,
- header: headerRule,
- body: bodyRule,
row: rowRule,
cell: cellRule
};
\ No newline at end of file
diff --git a/syntaxes/html/utils.js b/syntaxes/html/utils.js
index fb13927a..d1884c4e 100644
--- a/syntaxes/html/utils.js
+++ b/syntaxes/html/utils.js
@@ -1,14 +1,20 @@
var entities = require('html-entities');
var htmlEntities = new entities.AllHtmlEntities();
-// Escape markdown syntax
-// We escape only basic XML entities
+/**
+ * Escape all entities (HTML + XML)
+ * @param {String} str
+ * @return {String}
+ */
function escape(str) {
return htmlEntities.encode(str);
}
-// Unescape markdown syntax
-// We unescape all entities (HTML + XML)
+/**
+ * Unescape all entities (HTML + XML)
+ * @param {String} str
+ * @return {String}
+ */
function unescape(str) {
return htmlEntities.decode(str);
}
diff --git a/syntaxes/markdown/__tests__/specs.js b/syntaxes/markdown/__tests__/specs.js
index a442c388..82d68e19 100644
--- a/syntaxes/markdown/__tests__/specs.js
+++ b/syntaxes/markdown/__tests__/specs.js
@@ -53,11 +53,10 @@ function testMdIdempotence(fixture) {
backToMd = markdown.toText(content2);
var content3 = markdown.toContent(backToMd);
- var jsonContent1 = MarkupIt.JSONUtils.encode(content1);
- var jsonContent2 = MarkupIt.JSONUtils.encode(content2);
- var jsonContent3 = MarkupIt.JSONUtils.encode(content3);
+ var resultHtml2 = html.toText(content2);
+ var resultHtml3 = html.toText(content3);
- jsonContent3.should.eql(jsonContent2);
+ (resultHtml2).should.be.html(resultHtml3);
}
function readFixture(filename) {
diff --git a/syntaxes/markdown/__tests__/specs/blockquote_list_item.html b/syntaxes/markdown/__tests__/specs/blockquote_list_item.html
index d149cbc0..f263f53c 100755
--- a/syntaxes/markdown/__tests__/specs/blockquote_list_item.html
+++ b/syntaxes/markdown/__tests__/specs/blockquote_list_item.html
@@ -1,4 +1,3 @@
-This fails in markdown.pl and upskirt:
\ No newline at end of file
+This fails in markdown.pl and upskirt:
+
+
\ No newline at end of file
diff --git a/syntaxes/markdown/__tests__/specs/def_blocks.html b/syntaxes/markdown/__tests__/specs/def_blocks.html
index 49a7a750..4a5dc40e 100755
--- a/syntaxes/markdown/__tests__/specs/def_blocks.html
+++ b/syntaxes/markdown/__tests__/specs/def_blocks.html
@@ -13,6 +13,10 @@
+
+
+
diff --git a/syntaxes/markdown/__tests__/specs/loose_lists.html b/syntaxes/markdown/__tests__/specs/loose_lists.html
index 6ead82cb..1155c905 100755
--- a/syntaxes/markdown/__tests__/specs/loose_lists.html
+++ b/syntaxes/markdown/__tests__/specs/loose_lists.html
@@ -13,26 +13,50 @@
hi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/syntaxes/markdown/blocks.js b/syntaxes/markdown/blocks.js
index 2b913373..af5ff8bb 100644
--- a/syntaxes/markdown/blocks.js
+++ b/syntaxes/markdown/blocks.js
@@ -1,5 +1,5 @@
var reBlock = require('./re/block');
-var markup = require('../../');
+var MarkupIt = require('../../');
var heading = require('./heading');
var list = require('./list');
@@ -7,90 +7,71 @@ var code = require('./code');
var table = require('./table');
var utils = require('./utils');
-/**
- * Is top block check that a paragraph can be parsed
- * Paragraphs can exists only in loose list or blockquote.
- *
- * @param {List} parents
- * @return {Boolean}
- */
-function isTop(parents) {
- return parents.find(function(token) {
- var isBlockquote = (token.getType() === markup.BLOCKS.BLOCKQUOTE);
- var isLooseList = (token.isListItem() && token.getData().get('loose'));
-
- return (!isBlockquote && !isLooseList);
- }) === undefined;
-}
-
-module.exports = markup.RulesSet([
+module.exports = MarkupIt.RulesSet([
// ---- CODE BLOCKS ----
code.block,
// ---- FOOTNOTES ----
- markup.Rule(markup.BLOCKS.FOOTNOTE)
- .regExp(reBlock.footnote, function(match) {
+ MarkupIt.Rule(MarkupIt.BLOCKS.FOOTNOTE)
+ .regExp(reBlock.footnote, function(state, match) {
var text = match[2];
return {
- text: text,
+ tokens: state.parseAsInline(text),
data: {
id: match[1]
}
};
})
- .toText(function(text, block) {
- return '[^' + block.data.id + ']: ' + text + '\n\n';
+ .toText(function(state, token) {
+ var data = token.getData();
+ var id = data.get('id');
+ var innerContent = state.renderAsInline(token);
+
+ return '[^' + id + ']: ' + innerContent + '\n\n';
}),
// ---- HEADING ----
- heading.rule(6),
- heading.rule(5),
- heading.rule(4),
- heading.rule(3),
- heading.rule(2),
- heading.rule(1),
-
- heading.lrule(2),
- heading.lrule(1),
+ heading(6),
+ heading(5),
+ heading(4),
+ heading(3),
+ heading(2),
+ heading(1),
// ---- TABLE ----
table.block,
- table.header,
- table.body,
table.row,
table.cell,
// ---- HR ----
- markup.Rule(markup.BLOCKS.HR)
- .setOption('parse', false)
- .setOption('renderInner', false)
+ MarkupIt.Rule(MarkupIt.BLOCKS.HR)
.regExp(reBlock.hr, function() {
- return {
- text: ''
- };
+ return {};
})
.toText('---\n\n'),
// ---- BLOCKQUOTE ----
- markup.Rule(markup.BLOCKS.BLOCKQUOTE)
- .setOption('parse', 'block')
- .regExp(reBlock.blockquote, function(match) {
+ MarkupIt.Rule(MarkupIt.BLOCKS.BLOCKQUOTE)
+ .regExp(reBlock.blockquote, function(state, match) {
var inner = match[0].replace(/^ *> ?/gm, '').trim();
- return {
- text: inner
- };
+ return state.toggle('blockquote', function() {
+ return {
+ tokens: state.parseAsBlock(inner)
+ };
+ });
})
- .toText(function(text) {
- var lines = utils.splitLines(text.trim());
+ .toText(function(state, token) {
+ var innerContent = state.renderAsBlock(token);
+ var lines = utils.splitLines(innerContent.trim());
return lines
- .map(function(line) {
- return '> ' + line;
- })
- .join('\n') + '\n\n';
+ .map(function(line) {
+ return '> ' + line;
+ })
+ .join('\n') + '\n\n';
}),
// ---- LISTS ----
@@ -98,9 +79,8 @@ module.exports = markup.RulesSet([
list.ol,
// ---- HTML ----
- markup.Rule(markup.BLOCKS.HTML)
- .setOption('parse', false)
- .regExp(reBlock.html, function(match) {
+ MarkupIt.Rule(MarkupIt.BLOCKS.HTML)
+ .regExp(reBlock.html, function(state, match) {
return {
text: match[0]
};
@@ -108,50 +88,53 @@ module.exports = markup.RulesSet([
.toText('%s'),
// ---- DEFINITION ----
- markup.Rule(markup.BLOCKS.DEFINITION)
- .regExp(reBlock.def, function(match, parents) {
- if (parents.size > 0) {
- return null;
+ MarkupIt.Rule()
+ .regExp(reBlock.def, function(state, match) {
+ if (state.getDepth() > 1) {
+ return;
}
- var id = match[1].toLowerCase();
- var href = match[2];
+ var id = match[1].toLowerCase();
+ var href = match[2];
var title = match[3];
- this.refs = this.refs || {};
- this.refs[id] = {
+ state.refs = state.refs || {};
+ state.refs[id] = {
href: href,
title: title
};
return {
- type: markup.BLOCKS.IGNORE
+ type: 'definition'
};
}),
// ---- PARAGRAPH ----
- markup.Rule(markup.BLOCKS.PARAGRAPH)
- .regExp(reBlock.paragraph, function(match, parents) {
- if (!isTop(parents)) {
+ MarkupIt.Rule(MarkupIt.BLOCKS.PARAGRAPH)
+ .regExp(reBlock.paragraph, function(state, match) {
+ var isInBlocquote = (state.get('blockquote') === state.getParentDepth());
+ var isInLooseList = (state.get('looseList') === state.getParentDepth());
+ var isTop = (state.getDepth() === 1);
+
+ if (!isTop && !isInBlocquote && !isInLooseList) {
return;
}
-
var text = match[1].trim();
return {
- text: text
+ tokens: state.parseAsInline(text)
};
})
.toText('%s\n\n'),
- // ---- PARAGRAPH ----
- markup.Rule(markup.BLOCKS.TEXT)
- .regExp(reBlock.text, function(match, parents) {
- // Top-level should never reach here.
+ // ---- TEXT ----
+ // Top-level should never reach here.
+ MarkupIt.Rule(MarkupIt.BLOCKS.TEXT)
+ .regExp(reBlock.text, function(state, match) {
var text = match[0];
return {
- text: text
+ tokens: state.parseAsInline(text)
};
})
.toText('%s\n')
diff --git a/syntaxes/markdown/code.js b/syntaxes/markdown/code.js
index 23d40b97..92f5280a 100644
--- a/syntaxes/markdown/code.js
+++ b/syntaxes/markdown/code.js
@@ -1,16 +1,17 @@
var reBlock = require('./re/block');
-var markup = require('../../');
+var MarkupIt = require('../../');
var utils = require('./utils');
// Rule for parsing code blocks
-var blockRule = markup.Rule(markup.BLOCKS.CODE)
- .setOption('parse', false)
- .setOption('renderInner', false)
-
+var blockRule = MarkupIt.Rule(MarkupIt.BLOCKS.CODE)
// Fences
- .regExp(reBlock.fences, function(match) {
+ .regExp(reBlock.fences, function(state, match) {
+ var inner = match[3];
+
return {
- text: match[3],
+ tokens: [
+ MarkupIt.Token.createText(inner)
+ ],
data: {
syntax: match[2]
}
@@ -18,7 +19,7 @@ var blockRule = markup.Rule(markup.BLOCKS.CODE)
})
// 4 spaces / Tab
- .regExp(reBlock.code, function(match) {
+ .regExp(reBlock.code, function(state, match) {
var inner = match[0];
// Remove indentation
@@ -28,7 +29,9 @@ var blockRule = markup.Rule(markup.BLOCKS.CODE)
inner = inner.replace(/\n+$/, '');
return {
- text: inner,
+ tokens: [
+ MarkupIt.Token.createText(inner)
+ ],
data: {
syntax: undefined
}
@@ -36,13 +39,16 @@ var blockRule = markup.Rule(markup.BLOCKS.CODE)
})
// Output code blocks
- .toText(function(text, block) {
+ .toText(function(state, token) {
+ var text = token.getAsPlainText();
+ var data = token.getData();
+ var syntax = data.get('syntax') || '';
var hasFences = text.indexOf('`') >= 0;
// Use fences if syntax is set
- if (!hasFences) {
+ if (!hasFences || syntax) {
return (
- '```' + (block.data.syntax || '') + '\n'
+ '```' + syntax + '\n'
+ text
+ '```\n\n'
);
diff --git a/syntaxes/markdown/document.js b/syntaxes/markdown/document.js
new file mode 100644
index 00000000..4051d5ab
--- /dev/null
+++ b/syntaxes/markdown/document.js
@@ -0,0 +1,83 @@
+var Immutable = require('immutable');
+var MarkupIt = require('../../');
+
+/**
+ * Cleanup a text before parsing: normalize newlines and tabs
+ *
+ * @param {String} src
+ * @return {String}
+ */
+function cleanupText(src) {
+ return src
+ .replace(/\r\n|\r/g, '\n')
+ .replace(/\t/g, ' ')
+ .replace(/\u00a0/g, ' ')
+ .replace(/\u2424/g, '\n')
+ .replace(/^ +$/gm, '');
+}
+
+/**
+ * Resolve definition links
+ *
+ * @param {ParsingState} state
+ * @param {Token} token
+ * @return {Token}
+ */
+function resolveLink(state, token) {
+ var tokenType = token.getType();
+ var data = token.getData();
+
+ if (tokenType === 'definition') {
+ return false;
+ }
+ if (tokenType !== MarkupIt.ENTITIES.LINK) {
+ return token;
+ }
+
+ // Normal link
+ if (!data.has('ref')) {
+ return token;
+ }
+
+ // Resolve reference
+ var refs = (state.refs || {});
+ var refId = data.get('ref')
+ .replace(/\s+/g, ' ')
+ .toLowerCase();
+ var ref = refs[refId];
+
+ // Parse reference as text
+ if (!ref) {
+ var rawText = token.getRaw();
+
+ var tokens = Immutable.List([
+ MarkupIt.Token.createText(rawText[0])
+ ])
+ .concat(
+ state.parseAsInline(rawText.slice(1))
+ );
+
+ return MarkupIt.transform(tokens, resolveLink.bind(null, state));
+ }
+
+ // Update link attributes
+ return token.setData(
+ data.merge(ref)
+ );
+}
+
+var documentRule = MarkupIt.Rule(MarkupIt.BLOCKS.DOCUMENT)
+ .match(function(state, text) {
+ text = cleanupText(text);
+
+ var token = MarkupIt.Token.create(MarkupIt.BLOCKS.DOCUMENT, {
+ tokens: state.parseAsBlock(text)
+ });
+
+ return MarkupIt.transform(token, resolveLink.bind(null, state));
+ })
+ .toText(function(state, token) {
+ return state.renderAsBlock(token);
+ });
+
+module.exports = documentRule;
diff --git a/syntaxes/markdown/heading.js b/syntaxes/markdown/heading.js
index f6d5c78a..5b6119fb 100644
--- a/syntaxes/markdown/heading.js
+++ b/syntaxes/markdown/heading.js
@@ -1,8 +1,13 @@
var reHeading = require('./re/heading');
var markup = require('../../');
-// Parse inner text of header to extract ID entity
-function parseHeadingText(text) {
+/**
+ * Parse inner text of header to extract ID entity
+ * @param {ParsingState} state
+ * @param {String} text
+ * @return {TokenLike}
+ */
+function parseHeadingText(state, text) {
var id, match;
reHeading.id.lastIndex = 0;
@@ -17,44 +22,54 @@ function parseHeadingText(text) {
}
return {
- text: text,
+ tokens: state.parseAsInline(text),
data: {
id: id
}
};
}
-// Generator for HEADING_X rules
+/**
+ * Generator for HEADING_X rules
+ * @param {Number} level
+ * @return {Rule}
+ */
function headingRule(level) {
var prefix = Array(level + 1).join('#');
return markup.Rule(markup.BLOCKS['HEADING_' + level])
- .regExp(reHeading.normal, function(match) {
- if (match[1].length != level) return null;
- return parseHeadingText(match[2]);
- })
- .toText(function (text, block) {
- if (block.data.id) {
- text += ' {#' + block.data.id + '}';
+
+ // Normal heading like
+ .regExp(reHeading.normal, function(state, match) {
+ if (match[1].length != level) {
+ return;
}
- return prefix + ' ' + text + '\n\n';
- });
-}
+ return parseHeadingText(state, match[2]);
+ })
-// Generator for HEADING_X rules for line heading
-// Since normal heading are listed first, onText is not required here
-function lheadingRule(level) {
- return markup.Rule(markup.BLOCKS['HEADING_' + level])
- .regExp(reHeading.line, function(match) {
+ // Line heading
+ .regExp(reHeading.line, function(state, match) {
var matchLevel = (match[2] === '=')? 1 : 2;
- if (matchLevel != level) return null;
+ if (matchLevel != level) {
+ return;
+ }
- return parseHeadingText(match[1]);
+ return parseHeadingText(state, match[1]);
+ })
+
+ .toText(function (state, token) {
+ var data = token.getData();
+ var innerContent = state.renderAsInline(token);
+ var id = data.get('id');
+
+ if (id) {
+ innerContent += ' {#' + id + '}';
+ }
+
+ return prefix + ' ' + innerContent + '\n\n';
});
}
-module.exports = {
- rule: headingRule,
- lrule: lheadingRule
-};
+
+module.exports = headingRule;
diff --git a/syntaxes/markdown/index.js b/syntaxes/markdown/index.js
index 5aa40662..44dcf257 100644
--- a/syntaxes/markdown/index.js
+++ b/syntaxes/markdown/index.js
@@ -1,6 +1,8 @@
-var markup = require('../../');
+var MarkupIt = require('../../');
+
+module.exports = MarkupIt.Syntax('markdown', {
+ entryRule: require('./document'),
-module.exports = markup.Syntax('markdown', {
// List of rules for parsing blocks
inline: require('./inline'),
diff --git a/syntaxes/markdown/inline.js b/syntaxes/markdown/inline.js
index ad7ccc6a..1b1452f3 100644
--- a/syntaxes/markdown/inline.js
+++ b/syntaxes/markdown/inline.js
@@ -1,173 +1,152 @@
+var Immutable = require('immutable');
+
var reInline = require('./re/inline');
-var markup = require('../../');
+var MarkupIt = require('../../');
var utils = require('./utils');
var isHTMLBlock = require('./isHTMLBlock');
-/**
- * Test if we are parsing inside a link
- * @param {List} parents
- * @return {Boolean}
- */
-function isInLink(parents, ctx) {
- if (ctx.isLink) {
- return true;
- }
-
- return parents.find(function(tok) {
- if (tok.getType() === markup.ENTITIES.LINK) {
- return true;
- }
-
- return false;
- }) !== undefined;
-}
-
-/**
- * Resolve a reflink
- * @param {Object} ctx
- * @param {String} text
- * @return {Object|null}
- */
-function resolveRefLink(ctx, text) {
- var refs = (ctx.refs || {});
-
- // Normalize the refId
- var refId = (text)
- .replace(/\s+/g, ' ')
- .toLowerCase();
- var ref = refs[refId];
-
- return (ref && ref.href)? ref : null;
-}
-
-var inlineRules = markup.RulesSet([
+var inlineRules = MarkupIt.RulesSet([
// ---- FOOTNOTE REFS ----
- markup.Rule(markup.ENTITIES.FOOTNOTE_REF)
- .regExp(reInline.reffn, function(match) {
+ MarkupIt.Rule(MarkupIt.ENTITIES.FOOTNOTE_REF)
+ .regExp(reInline.reffn, function(state, match) {
return {
- text: match[1],
- data: {}
+ tokens: [
+ MarkupIt.Token.createText(match[1])
+ ]
};
})
- .toText(function(text) {
- return '[^' + text + ']';
+ .toText(function(state, token) {
+ return '[^' + token.getAsPlainText() + ']';
}),
// ---- IMAGES ----
- markup.Rule(markup.ENTITIES.IMAGE)
- .regExp(reInline.link, function(match) {
+ MarkupIt.Rule(MarkupIt.ENTITIES.IMAGE)
+ .regExp(reInline.link, function(state, match) {
var isImage = match[0].charAt(0) === '!';
- if (!isImage) return null;
+ if (!isImage) {
+ return;
+ }
return {
- text: ' ',
data: {
alt: match[1],
src: match[2]
}
};
})
- .toText(function(text, entity) {
- return '![' + entity.data.alt + '](' + entity.data.src + ')';
+ .toText(function(state, token) {
+ var data = token.getData();
+ var alt = data.get('alt', '');
+ var src = data.get('src', '');
+
+ return '![' + alt + '](' + src + ')';
}),
// ---- LINK ----
- markup.Rule(markup.ENTITIES.LINK)
- .regExp(reInline.link, function(match) {
- return {
- text: match[1],
- data: {
- href: match[2],
- title: match[3]
- }
- };
+ MarkupIt.Rule(MarkupIt.ENTITIES.LINK)
+ .regExp(reInline.link, function(state, match) {
+ return state.toggle('link', function() {
+ return {
+ tokens: state.parseAsInline(match[1]),
+ data: {
+ href: match[2],
+ title: match[3]
+ }
+ };
+ });
})
- .regExp(reInline.autolink, function(match) {
- return {
- text: match[1],
- data: {
- href: match[1]
- }
- };
+ .regExp(reInline.autolink, function(state, match) {
+ return state.toggle('link', function() {
+ return {
+ tokens: state.parseAsInline(match[1]),
+ data: {
+ href: match[1]
+ }
+ };
+ });
})
- .regExp(reInline.url, function(match, parents) {
- if (isInLink(parents, this)) {
+ .regExp(reInline.url, function(state, match, parents) {
+ if (state.get('link')) {
return;
}
-
var uri = match[1];
return {
- text: uri,
data: {
href: uri
- }
+ },
+ tokens: [
+ MarkupIt.Token.createText(uri)
+ ]
};
})
- .toText(function(text, entity) {
- var title = entity.data.title? ' "' + entity.data.title + '"' : '';
- return '[' + text + '](' + entity.data.href + title + ')';
- }),
-
- // ---- REF LINKS ----
- // Doesn't render, but match and resolve reference
- markup.Rule(markup.ENTITIES.LINK_REF)
- .regExp(reInline.reflink, function(match) {
- var ref = resolveRefLink(this, (match[2] || match[1]));
-
- if (!ref) {
- return null;
- }
-
- return {
- type: markup.ENTITIES.LINK,
- text: match[1],
- data: ref
- };
+ .regExp(reInline.reflink, function(state, match) {
+ var refId = (match[2] || match[1]);
+ var innerText = match[1];
+
+ return state.toggle('link', function() {
+ return {
+ type: MarkupIt.ENTITIES.LINK,
+ data: {
+ ref: refId
+ },
+ tokens: [
+ MarkupIt.Token.createText(innerText)
+ ]
+ };
+ });
})
- .regExp(reInline.nolink, function(match) {
- var ref = resolveRefLink(this, (match[2] || match[1]));
-
- if (!ref) {
- return null;
- }
-
- return {
- type: markup.ENTITIES.LINK,
- text: match[1],
- data: ref
- };
+ .regExp(reInline.nolink, function(state, match) {
+ var refId = (match[2] || match[1]);
+
+ return state.toggle('link', function() {
+ return {
+ type: MarkupIt.ENTITIES.LINK,
+ tokens: state.parseAsInline(match[1]),
+ data: {
+ ref: refId
+ }
+ };
+ });
})
- .regExp(reInline.reffn, function(match) {
- var ref = resolveRefLink(this, match[1]);
-
- if (!ref) {
- return null;
- }
-
- return {
- text: match[1],
- data: ref
- };
+ .regExp(reInline.reffn, function(state, match) {
+ var refId = match[1];
+
+ return state.toggle('link', function() {
+ return {
+ tokens: state.parseAsInline(match[1]),
+ data: {
+ ref: refId
+ }
+ };
+ });
})
- .toText(function(text, entity) {
- var title = entity.data.title? ' "' + entity.data.title + '"' : '';
- return '[' + text + '](' + entity.data.href + title + ')';
+ .toText(function(state, token) {
+ var data = token.getData();
+ var title = data.get('title');
+ var href = data.get('href');
+ var innerContent = state.renderAsInline(token);
+ title = title? ' "' + title + '"' : '';
+
+ return '[' + innerContent + '](' + href + title + ')';
}),
// ---- CODE ----
- markup.Rule(markup.STYLES.CODE)
- .setOption('parse', false)
- .regExp(reInline.code, function(match) {
+ MarkupIt.Rule(MarkupIt.STYLES.CODE)
+ .regExp(reInline.code, function(state, match) {
return {
- text: match[2]
+ tokens: [
+ MarkupIt.Token.createText(match[2])
+ ]
};
})
- .toText(function(text) {
- // We need to find the right separator not present in the content
+ .toText(function(state, token) {
var separator = '`';
- while(text.indexOf(separator) >= 0) {
+ var text = token.getAsPlainText();
+
+ // We need to find the right separator not present in the content
+ while (text.indexOf(separator) >= 0) {
separator += '`';
}
@@ -175,117 +154,102 @@ var inlineRules = markup.RulesSet([
}),
// ---- BOLD ----
- markup.Rule(markup.STYLES.BOLD)
- .regExp(reInline.strong, function(match) {
+ MarkupIt.Rule(MarkupIt.STYLES.BOLD)
+ .regExp(reInline.strong, function(state, match) {
return {
- text: match[2] || match[1]
+ tokens: state.parseAsInline(match[2] || match[1])
};
})
.toText('**%s**'),
// ---- ITALIC ----
- markup.Rule(markup.STYLES.ITALIC)
- .regExp(reInline.em, function(match) {
+ MarkupIt.Rule(MarkupIt.STYLES.ITALIC)
+ .regExp(reInline.em, function(state, match) {
return {
- text: match[2] || match[1]
+ tokens: state.parseAsInline(match[2] || match[1])
};
})
.toText('_%s_'),
// ---- STRIKETHROUGH ----
- markup.Rule(markup.STYLES.STRIKETHROUGH)
- .regExp(reInline.del, function(match) {
+ MarkupIt.Rule(MarkupIt.STYLES.STRIKETHROUGH)
+ .regExp(reInline.del, function(state, match) {
return {
- text: match[1]
+ tokens: state.parseAsInline(match[1])
};
})
.toText('~~%s~~'),
// ---- HTML ----
- markup.Rule(markup.STYLES.HTML)
- .setOption('parse', false)
- .setOption('renderInline', false)
- .regExp(reInline.html, function(match, parents) {
- var tag = match[0];
- var tagName = match[1];
+ MarkupIt.Rule(MarkupIt.STYLES.HTML)
+ .regExp(reInline.html, function(state, match) {
+ var tag = match[0];
+ var tagName = match[1];
var innerText = match[2] || '';
-
var startTag, endTag;
+ var innerTokens = [];
if (innerText) {
startTag = tag.substring(0, tag.indexOf(innerText));
- endTag = tag.substring(tag.indexOf(innerText) + innerText.length);
+ endTag = tag.substring(tag.indexOf(innerText) + innerText.length);
} else {
startTag = match[0];
- endTag = '';
+ endTag = '';
}
-
- // todo: handle link tags
- /*if (tagName === 'a' && innerText) {
-
- }*/
-
- var innerTokens = [];
-
if (tagName && !isHTMLBlock(tagName) && innerText) {
- var inlineSyntax = markup.Syntax('markdown+html', {
- inline: inlineRules
+ var isLink = (tagName.toLowerCase() === 'a');
+
+ innerTokens = state.toggle(isLink? 'link' : 'html', function() {
+ return state.parseAsInline(innerText);
});
- var oldIsLink = this.isLink;
- this.isLink = this.isLink || (tagName.toLowerCase() === 'a');
- innerTokens = markup.parseInline(inlineSyntax, innerText, this)
- .getTokens()
- .toArray();
- this.isLink = oldIsLink;
} else {
innerTokens = [
{
- type: markup.STYLES.HTML,
+ type: MarkupIt.STYLES.HTML,
text: innerText,
- raw: innerText
+ raw: innerText
}
];
}
- var result = [];
-
- result.push({
- type: markup.STYLES.HTML,
- text: startTag,
- raw: startTag
- });
+ var result = Immutable.List()
+ .push({
+ type: MarkupIt.STYLES.HTML,
+ text: startTag,
+ raw: startTag
+ });
result = result.concat(innerTokens);
if (endTag) {
- result.push({
- type: markup.STYLES.HTML,
+ result = result.push({
+ type: MarkupIt.STYLES.HTML,
text: endTag,
- raw: endTag
+ raw: endTag
});
}
return result;
})
- .toText(function(text, token) {
- return text;
+ .toText(function(state, token) {
+ return token.getAsPlainText();
}),
// ---- ESCAPED ----
- markup.Rule(markup.STYLES.TEXT)
- .setOption('parse', false)
- .regExp(reInline.escape, function(match) {
+ MarkupIt.Rule(MarkupIt.STYLES.TEXT)
+ .regExp(reInline.escape, function(state, match) {
return {
text: match[1]
};
})
- .regExp(reInline.text, function(match) {
+ .regExp(reInline.text, function(state, match) {
return {
text: utils.unescape(match[0])
};
})
- .toText(function(text) {
+ .toText(function(state, token) {
+ var text = token.getAsPlainText();
return utils.escape(text, false);
})
]);
diff --git a/syntaxes/markdown/isHTMLBlock.js b/syntaxes/markdown/isHTMLBlock.js
index a53c31e3..9c8df072 100644
--- a/syntaxes/markdown/isHTMLBlock.js
+++ b/syntaxes/markdown/isHTMLBlock.js
@@ -68,11 +68,10 @@ var htmlBlocks = Immutable.List([
]);
/**
- Test if a tag name is a valid HTML block
-
- @param {String} tag
- @return {Boolean}
-*/
+ * Test if a tag name is a valid HTML block
+ * @param {String} tag
+ * @return {Boolean}
+ */
function isHTMLBlock(tag) {
tag = tag.toLowerCase();
return htmlBlocks.includes(tag);
diff --git a/syntaxes/markdown/list.js b/syntaxes/markdown/list.js
index 266caeb1..e6764fa1 100644
--- a/syntaxes/markdown/list.js
+++ b/syntaxes/markdown/list.js
@@ -1,25 +1,19 @@
var reBlock = require('./re/block');
-var markup = require('../../');
+var MarkupIt = require('../../');
var utils = require('./utils');
var reList = reBlock.list;
-// Return true if block is a list
-function isListItem(type) {
- return (type == markup.BLOCKS.UL_ITEM || type == markup.BLOCKS.OL_ITEM);
-}
-
// Rule for lists, rBlock.list match the whole (multilines) list, we stop at the first item
function listRule(type) {
- return markup.Rule(type)
- .setOption('parse', 'block')
- .regExp(reList.block, function(match) {
+ return MarkupIt.Rule(type)
+ .regExp(reList.block, function(state, match) {
var rawList = match[0];
var bull = match[2];
var ordered = bull.length > 1;
- if (ordered && type === markup.BLOCKS.UL_ITEM) return;
- if (!ordered && type === markup.BLOCKS.OL_ITEM) return;
+ if (ordered && type === MarkupIt.BLOCKS.UL_LIST) return;
+ if (!ordered && type === MarkupIt.BLOCKS.OL_LIST) return;
var item, loose, next = false;
@@ -33,7 +27,6 @@ function listRule(type) {
rawItem = rawList.slice(lastIndex, reList.item.lastIndex);
lastIndex = reList.item.lastIndex;
-
items.push([item, rawItem]);
}
@@ -63,45 +56,56 @@ function listRule(type) {
if (!loose) loose = next;
}
- result.push({
- type: type,
- raw: rawItem,
- text: textItem,
- data:{
- loose: loose
- }
- });
- }
+ var parse = function() {
+ return MarkupIt.Token.create(MarkupIt.BLOCKS.LIST_ITEM, {
+ raw: rawItem,
+ tokens: state.parseAsBlock(textItem),
+ data: {
+ loose: loose
+ }
+ });
+ };
- return result;
- })
- .toText(function(text, block) {
- // Determine which bullet to use
- var bullet = '*';
- if (type == markup.BLOCKS.OL_ITEM) {
- bullet = '1.';
+ result.push(
+ loose? state.toggle('looseList', parse) : parse()
+ );
}
- var nextBlock = block.next? block.next.type : null;
-
- // Prepend text with spacing
- var rows = utils.splitLines(text);
- var head = rows[0];
- var rest = utils.indent(rows.slice(1).join('\n'), ' ');
-
- var eol = rest? '' : '\n';
- if (nextBlock && !isListItem(nextBlock)) {
- eol += '\n';
- }
-
- var result = bullet + ' ' + head + (rest ? '\n' + rest : '') + eol;
+ return {
+ tokens: result
+ };
+ })
+ .toText(function(state, token) {
+ var listType = token.getType();
+ var items = token.getTokens();
+
+ return items.reduce(function(text, item, i) {
+ // Determine which bullet to use
+ var bullet = '*';
+ if (listType == MarkupIt.BLOCKS.OL_LIST) {
+ bullet = (i + 1) + '.';
+ }
- return result;
+ // Prepend text with spacing
+ var innerText = state.renderAsBlock(item);
+ var rows = utils.splitLines(innerText);
+ var head = rows[0];
+ var rest = utils.indent(rows.slice(1).join('\n'), ' ');
+ var eol = rest? '' : '\n';
+ var isLoose = item.getTokens()
+ .find(function(p) {
+ return p.getType() === MarkupIt.BLOCKS.PARAGRAPH;
+ }) !== undefined;
+ //if (isLoose) eol += '\n';
+
+ var itemText = bullet + ' ' + head + (rest ? '\n' + rest : '') + eol;
+ return text + itemText;
+ }, '') + '\n';
});
}
module.exports = {
- ul: listRule(markup.BLOCKS.UL_ITEM),
- ol: listRule(markup.BLOCKS.OL_ITEM)
+ ul: listRule(MarkupIt.BLOCKS.UL_LIST),
+ ol: listRule(MarkupIt.BLOCKS.OL_LIST)
};
diff --git a/syntaxes/markdown/table.js b/syntaxes/markdown/table.js
index 7f4ab646..10c2a781 100644
--- a/syntaxes/markdown/table.js
+++ b/syntaxes/markdown/table.js
@@ -1,48 +1,35 @@
-var reTable = require('./re/table');
-var markup = require('../../');
+var Immutable = require('immutable');
+var MarkupIt = require('../../');
+var reTable = require('./re/table');
var tableRow = require('./tableRow');
-// Create a table entity
+var ALIGN = {
+ LEFT: 'left',
+ RIGHT: 'right',
+ CENTER: 'center'
+};
/**
* Create a table entity from parsed header/rows
*
+ * @param {ParsingState} state
* @param {Array} header
* @param {Array} align
* @param {Array} rows
* @rteturn {Object} tokenMatch
*/
-function Table(header, align, rows) {
- var ctx = this;
-
- var headerRow = tableRow.parse(header, ctx, align);
+function Table(state, header, align, rows) {
+ var headerRow = tableRow.parse(state, header);
var rowTokens = rows.map(function(row) {
- return tableRow.parse(row, ctx, align);
- });
-
- var headerToken = markup.Token.create(markup.BLOCKS.TABLE_HEADER, {
- tokens: [headerRow],
- data: {
- align: align
- }
- });
-
- var bodyToken = markup.Token.create(markup.BLOCKS.TABLE_BODY, {
- tokens: rowTokens,
- data: {
- align: align
- }
+ return tableRow.parse(state, row);
});
return {
data: {
align: align
},
- tokens: [
- headerToken,
- bodyToken
- ]
+ tokens: Immutable.List([headerRow]).concat(rowTokens)
};
}
@@ -55,11 +42,11 @@ function Table(header, align, rows) {
function mapAlign(align) {
return align.map(function(s) {
if (reTable.alignRight.test(s)) {
- return 'right';
+ return ALIGN.RIGHT;
} else if (reTable.alignCenter.test(s)) {
- return 'center';
+ return ALIGN.CENTER;
} else if (reTable.alignLeft.test(s)) {
- return 'left';
+ return ALIGN.LEFT;
} else {
return null;
}
@@ -86,10 +73,10 @@ function alignToText(row) {
}).join('');
}
-var blockRule = markup.Rule(markup.BLOCKS.TABLE)
+var blockRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE)
// Table no leading pipe (gfm)
- .regExp(reTable.nptable, function(match) {
+ .regExp(reTable.nptable, function(state, match) {
var header = match[1];
var align = match[2]
.replace(reTable.trailingPipeAlign, '')
@@ -101,11 +88,11 @@ var blockRule = markup.Rule(markup.BLOCKS.TABLE)
// Align for columns
align = mapAlign(align);
- return Table.call(this, header, align, rows);
+ return Table(state, header, align, rows);
})
// Normal table
- .regExp(reTable.normal, function(match) {
+ .regExp(reTable.normal, function(state, match) {
var header = match[1];
var align = match[2]
.replace(reTable.trailingPipeAlign, '')
@@ -118,37 +105,49 @@ var blockRule = markup.Rule(markup.BLOCKS.TABLE)
// Align for columns
align = mapAlign(align);
- return Table.call(this, header, align, rows);
+ return Table(state, header, align, rows);
})
- .toText(function(text) {
- return text + '\n';
- });
+ .toText(function(state, token) {
+ var data = token.getData();
+ var rows = token.getTokens();
+ var align = data.get('align');
-var headerRule = markup.Rule(markup.BLOCKS.TABLE_HEADER)
- .toText(function(text, tok) {
- return text + alignToText(tok.data.align) + '\n';
- });
+ var headerRows = rows.slice(0, 1);
+ var bodyRows = rows.slice(1);
+ var headerRow = headerRows.get(0);
+ var countCells = headerRow.getTokens().size;
+
+ align = align || [];
+ align = Array
+ .apply(null, Array(countCells))
+ .map(function(v, i){
+ return align[i] || ALIGN.LEFT;
+ });
+
+ var headerText = state.render(headerRows);
+ var bodyText = state.render(bodyRows);
+
+ return (headerText
+ + alignToText(align) + '\n'
+ + bodyText + '\n');
-var bodyRule = markup.Rule(markup.BLOCKS.TABLE_BODY)
- .toText(function(text) {
- return text;
});
-var rowRule = markup.Rule(markup.BLOCKS.TABLE_ROW)
- .toText(function(text) {
- return '|' + text + '\n';
+var rowRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_ROW)
+ .toText(function(state, token) {
+ var innerContent = state.render(token);
+ return '|' + innerContent + '\n';
});
-var cellRule = markup.Rule(markup.BLOCKS.TABLE_CELL)
- .toText(function(text) {
- return ' ' + text.trim() + ' |';
+var cellRule = MarkupIt.Rule(MarkupIt.BLOCKS.TABLE_CELL)
+ .toText(function(state, token) {
+ var innerContent = state.render(token);
+ return ' ' + innerContent.trim() + ' |';
});
module.exports = {
block: blockRule,
- header: headerRule,
- body: bodyRule,
cell: cellRule,
row: rowRule
};
diff --git a/syntaxes/markdown/tableRow.js b/syntaxes/markdown/tableRow.js
index 338506ca..1314e77b 100644
--- a/syntaxes/markdown/tableRow.js
+++ b/syntaxes/markdown/tableRow.js
@@ -7,61 +7,49 @@ var utils = require('./utils');
var CELL_SEPARATOR = 'cell';
/*
- Custom inline syntax to parse each row with custom cell separator tokens
-*/
+ * Custom inline syntax to parse each row with custom cell separator tokens
+ */
var rowRules = inlineRules
.unshift(
MarkupIt.Rule(CELL_SEPARATOR)
- .setOption('parse', false)
.regExp(reTable.cellSeparation, function(match) {
return {
- text: match[0]
+ raw: match[0]
};
})
)
.replace(
MarkupIt.Rule(MarkupIt.STYLES.TEXT)
- .setOption('parse', false)
- .regExp(reTable.cellInlineEscape, function(match) {
- return {
- text: utils.unescape(match[0])
- };
+ .regExp(reTable.cellInlineEscape, function(state, match) {
+ var text = utils.unescape(match[0]);
+ return { text: text };
})
- .regExp(reTable.cellInlineText, function(match) {
- return {
- text: utils.unescape(match[0])
- };
+ .regExp(reTable.cellInlineText, function(state, match) {
+ var text = utils.unescape(match[0]);
+ return { text: text };
})
.toText(utils.escape)
);
-var rowSyntax = MarkupIt.Syntax('markdown+row', {
- inline: rowRules
-});
-
-/*
- Parse a row from a table
-
- @param {String} text
- @param {Object} ctx
- @param {Array} align
-
- @return {Token}
-*/
-function parseRow(text, ctx, align) {
+/**
+ * Parse a row from a table
+ *
+ * @param {ParsingState} state
+ * @param {String} text
+ * @return {Token}
+ */
+function parseRow(state, text) {
var cells = [];
var accu = [];
- var content = MarkupIt.parseInline(rowSyntax, text, ctx);
- var tokens = content.getTokens();
+ var tokens = state.parse(rowRules, true, text);
function pushCell() {
- if (accu.length == 0) return;
+ if (accu.length == 0) {
+ return;
+ }
var cell = MarkupIt.Token.create(MarkupIt.BLOCKS.TABLE_CELL, {
- tokens: accu,
- data: {
- align: align[cells.length]
- }
+ tokens: accu
});
cells.push(cell);
@@ -75,7 +63,6 @@ function parseRow(text, ctx, align) {
accu.push(token);
}
});
-
pushCell();
return MarkupIt.Token.create(MarkupIt.BLOCKS.TABLE_ROW, {
diff --git a/test.md b/test.md
new file mode 100644
index 00000000..b0e81e78
--- /dev/null
+++ b/test.md
@@ -0,0 +1,3 @@
+Some long sentence. [^footnote]
+
+[^footnote]: Test, [Link](https://google.com).
\ No newline at end of file
diff --git a/testing/mock.js b/testing/mock.js
index 9ee2a980..b66fc276 100644
--- a/testing/mock.js
+++ b/testing/mock.js
@@ -32,6 +32,16 @@ var helloWorld = MarkupIt.Token({
});
module.exports = {
- paragraph: MarkupIt.Content.createFromTokens('mysyntax', [helloWorld]),
- titleParagraph: MarkupIt.Content.createFromTokens('mysyntax', [helloTitle, helloWorld])
+ paragraph: MarkupIt.Content.createFromToken(
+ 'mysyntax',
+ MarkupIt.Token.create(MarkupIt.BLOCKS.DOCUMENT, {
+ tokens: [helloWorld]
+ })
+ ),
+ titleParagraph: MarkupIt.Content.createFromToken(
+ 'mysyntax',
+ MarkupIt.Token.create(MarkupIt.BLOCKS.DOCUMENT, {
+ tokens: [helloTitle, helloWorld]
+ })
+ )
};