Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add StringLiteral.parse(), NumberLiteral.parse() #246

Merged
merged 2 commits into from
Feb 27, 2019
Merged

Add StringLiteral.parse(), NumberLiteral.parse() #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c468d46
Rename StringLiteral.raw to value. Add base Literal with parse().
stasm Feb 21, 2019
9f62f4f
Add tests for Literal.parse()
stasm Feb 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@
"generate:fixtures": "make -sC test/fixtures",
"generate": "npm run --silent generate:ebnf && npm run --silent generate:fixtures",
"lint": "eslint **/*.mjs",
"test:fixtures": "node --experimental-modules test/parser.mjs ./test/fixtures",
"test:ebnf": "node --experimental-modules test/ebnf.mjs ./syntax/grammar.mjs ./spec/fluent.ebnf",
"test": "npm run --silent test:fixtures"
"test:fixtures": "node --experimental-modules test/parser.mjs ./test/fixtures",
"test:unit": "node --experimental-modules test/literals.mjs",
"test": "npm run --silent test:fixtures && npm run --silent test:unit"
},
"homepage": "https://projectfluent.org",
"repository": {
Expand Down
9 changes: 9 additions & 0 deletions spec/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@
The `VariantLists` and the `VariantExpression` syntax and AST nodes were
deprecated in Syntax 0.9 and have now been removed.

- Rename `StringLiteral.raw` to `value`. (#243)

`StringLiteral.value` contains the exact contents of the string literal,
character-for-character. Escape sequences are stored verbatim without
processing.

Implementations may decide how to process the raw value. When they do,
however, they must comply with the behavior specified in `Literal.parse`.

- Rename `args` to `arguments`.

The `args` field of `MessageReference`, `TermReference`,
Expand Down
30 changes: 0 additions & 30 deletions syntax/abstract.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,6 @@ export function into(Type) {
}
return always(new Type(expression));
};
case FTL.StringLiteral:
return raw =>
always(new Type(raw, unescape(raw)));
default:
return (...args) =>
always(new Type(...args));
Expand Down Expand Up @@ -202,30 +199,3 @@ function remove_empty_text(element) {
function remove_blank_lines(element) {
return typeof(element) !== "string";
}

// Backslash backslash, backslash double quote, uHHHH, UHHHHHH.
const KNOWN_ESCAPES =
/(?:\\\\|\\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g;

function unescape(raw) {
return raw.replace(KNOWN_ESCAPES, from_escape_sequence);
}

function from_escape_sequence(match, codepoint4, codepoint6) {
switch (match) {
case "\\\\":
return "\\";
case "\\\"":
return "\"";
default:
let codepoint = parseInt(codepoint4 || codepoint6, 16);
if (codepoint <= 0xD7FF || 0xE000 <= codepoint) {
// It's a Unicode scalar value.
return String.fromCodePoint(codepoint);
}
// Escape sequences reresenting surrogate code points are
// well-formed but invalid in Fluent. Replace them with U+FFFD
// REPLACEMENT CHARACTER.
return "�";
}
}
70 changes: 57 additions & 13 deletions syntax/ast.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,26 +73,70 @@ export class Placeable extends PatternElement {
// An abstract base class for Expressions.
export class Expression extends SyntaxNode {}

// The "raw" field contains the exact contents of the string literal,
// character-for-character. Escape sequences are stored verbatim without
// processing. The "value" field contains the same contents with escape
// sequences unescaped to the characters they represent.
// See grammar.mjs for the definitions of well-formed escape sequences and
// abstract.mjs for the validation and unescaping logic.
export class StringLiteral extends Expression {
constructor(raw, value) {
// An abstract base class for Literals.
export class Literal extends Expression {
constructor(value) {
super();
this.type = "StringLiteral";
this.raw = raw;
// The "value" field contains the exact contents of the literal,
// character-for-character.
this.value = value;
}

// Implementations are free to decide how they process the raw value. When
// they do, however, they must comply with the behavior of `Literal.parse`.
parse() {
return {value: this.value};
}
}

export class NumberLiteral extends Expression {
export class StringLiteral extends Literal {
constructor(value) {
super();
super(value);
this.type = "StringLiteral";
}

parse() {
// Backslash backslash, backslash double quote, uHHHH, UHHHHHH.
const KNOWN_ESCAPES =
/(?:\\\\|\\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g;

function from_escape_sequence(match, codepoint4, codepoint6) {
switch (match) {
case "\\\\":
return "\\";
case "\\\"":
return "\"";
default:
let codepoint = parseInt(codepoint4 || codepoint6, 16);
if (codepoint <= 0xD7FF || 0xE000 <= codepoint) {
// It's a Unicode scalar value.
return String.fromCodePoint(codepoint);
}
// Escape sequences reresenting surrogate code points are
// well-formed but invalid in Fluent. Replace them with U+FFFD
// REPLACEMENT CHARACTER.
return "�";
}
}

let value = this.value.replace(KNOWN_ESCAPES, from_escape_sequence);
return {value};
}
}

export class NumberLiteral extends Literal {
constructor(value) {
super(value);
this.type = "NumberLiteral";
this.value = value;
}

parse() {
let value = parseFloat(this.value);
let decimal_position = this.value.indexOf(".");
let precision = decimal_position > 0
? this.value.length - decimal_position - 1
: 0;
return {value, precision};
}
}

Expand Down
2 changes: 2 additions & 0 deletions syntax/grammar.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ let InlineExpression = defer(() =>

/* -------- */
/* Literals */
export
let StringLiteral = defer(() =>
sequence(
string("\""),
Expand All @@ -210,6 +211,7 @@ let StringLiteral = defer(() =>
.map(join)
.chain(into(FTL.StringLiteral)));

export
let NumberLiteral = defer(() =>
sequence(
maybe(string("-")),
Expand Down
10 changes: 3 additions & 7 deletions test/fixtures/astral.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\uD83D\\uDE02",
"value": "��"
"value": "\\uD83D\\uDE02"
}
}
]
Expand All @@ -90,16 +89,14 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\uD83D",
"value": "�"
"value": "\\uD83D"
}
},
{
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\uDE02",
"value": "�"
"value": "\\uDE02"
}
}
]
Expand Down Expand Up @@ -138,7 +135,6 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "A face 😂 with tears of joy.",
"value": "A face 😂 with tears of joy."
}
}
Expand Down
9 changes: 0 additions & 9 deletions test/fixtures/call_expressions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,6 @@
},
{
"type": "StringLiteral",
"raw": "a",
"value": "a"
},
{
Expand Down Expand Up @@ -219,7 +218,6 @@
},
"value": {
"type": "StringLiteral",
"raw": "Y",
"value": "Y"
}
}
Expand Down Expand Up @@ -272,7 +270,6 @@
},
"value": {
"type": "StringLiteral",
"raw": "Y",
"value": "Y"
}
}
Expand Down Expand Up @@ -311,7 +308,6 @@
},
{
"type": "StringLiteral",
"raw": "a",
"value": "a"
},
{
Expand Down Expand Up @@ -343,7 +339,6 @@
},
"value": {
"type": "StringLiteral",
"raw": "Y",
"value": "Y"
}
}
Expand Down Expand Up @@ -400,7 +395,6 @@
"positional": [
{
"type": "StringLiteral",
"raw": "a",
"value": "a"
},
{
Expand Down Expand Up @@ -484,7 +478,6 @@
"positional": [
{
"type": "StringLiteral",
"raw": "a",
"value": "a"
},
{
Expand Down Expand Up @@ -539,7 +532,6 @@
"positional": [
{
"type": "StringLiteral",
"raw": "a",
"value": "a"
},
{
Expand Down Expand Up @@ -657,7 +649,6 @@
"positional": [
{
"type": "StringLiteral",
"raw": "a",
"value": "a"
}
],
Expand Down
26 changes: 8 additions & 18 deletions test/fixtures/escaped_characters.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\\"",
"value": "\""
"value": "\\\""
}
}
]
Expand All @@ -145,8 +144,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\\\",
"value": "\\"
"value": "\\\\"
}
}
]
Expand Down Expand Up @@ -189,8 +187,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\u0041",
"value": "A"
"value": "\\u0041"
}
}
]
Expand All @@ -211,8 +208,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\\\u0041",
"value": "\\u0041"
"value": "\\\\u0041"
}
}
]
Expand All @@ -233,8 +229,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\U01F602",
"value": "😂"
"value": "\\U01F602"
}
}
]
Expand All @@ -255,8 +250,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\\\U01F602",
"value": "\\U01F602"
"value": "\\\\U01F602"
}
}
]
Expand All @@ -277,8 +271,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\u004100",
"value": "A00"
"value": "\\u004100"
}
}
]
Expand All @@ -302,8 +295,7 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "\\U01F60200",
"value": "😂00"
"value": "\\U01F60200"
}
}
]
Expand Down Expand Up @@ -353,7 +345,6 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "{",
"value": "{"
}
},
Expand Down Expand Up @@ -383,7 +374,6 @@
"type": "Placeable",
"expression": {
"type": "StringLiteral",
"raw": "}",
"value": "}"
}
},
Expand Down
Loading