Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce comments support #443

Merged
merged 2 commits into from
Oct 14, 2015
Merged

Introduce comments support #443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ce78b1
Introduce support of comments.
andrusha97 Oct 11, 2015
f7960ac
Comments parsing fixes.
andrusha97 Oct 13, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 63 additions & 12 deletions include/rapidjson/reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ enum ParseFlag {
kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing.
kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error.
kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower).
kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments.
kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS
};

Expand Down Expand Up @@ -398,7 +399,8 @@ class GenericReader {

ClearStackOnExit scope(*this);

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);

if (is.Peek() == '\0') {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell());
Expand All @@ -409,7 +411,8 @@ class GenericReader {
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);

if (!(parseFlags & kParseStopWhenDoneFlag)) {
SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);

if (is.Peek() != '\0') {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell());
Expand Down Expand Up @@ -462,6 +465,40 @@ class GenericReader {
ClearStackOnExit& operator=(const ClearStackOnExit&);
};

template<unsigned parseFlags, typename InputStream>
void SkipWhitespaceAndComments(InputStream& is) {
SkipWhitespace(is);

if (parseFlags & kParseCommentsFlag) {
while (is.Peek() == '/') {
is.Take();

if (is.Peek() == '*') {
is.Take();
while (true) {
if (is.Peek() == '\0')
RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());

if (is.Take() == '*') {
if (is.Peek() == '\0')
RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());

if (is.Take() == '/')
break;
}
}
} else if (is.Peek() == '/') {
is.Take();
while (is.Peek() != '\0' && is.Take() != '\n') { }
} else {
RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
}

SkipWhitespace(is);
}
}
}

// Parse object: { string : value, ... }
template<unsigned parseFlags, typename InputStream, typename Handler>
void ParseObject(InputStream& is, Handler& handler) {
Expand All @@ -471,7 +508,8 @@ class GenericReader {
if (!handler.StartObject())
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

if (is.Peek() == '}') {
is.Take();
Expand All @@ -487,22 +525,28 @@ class GenericReader {
ParseString<parseFlags>(is, handler, true);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

if (is.Take() != ':')
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell());

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

ParseValue<parseFlags>(is, handler);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

++memberCount;

switch (is.Take()) {
case ',': SkipWhitespace(is); break;
case ',':
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
break;
case '}':
if (!handler.EndObject(memberCount))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
Expand All @@ -521,7 +565,8 @@ class GenericReader {
if (!handler.StartArray())
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

if (is.Peek() == ']') {
is.Take();
Expand All @@ -535,10 +580,14 @@ class GenericReader {
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

++elementCount;
SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;

switch (is.Take()) {
case ',': SkipWhitespace(is); break;
case ',':
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
break;
case ']':
if (!handler.EndArray(elementCount))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
Expand Down Expand Up @@ -1404,7 +1453,8 @@ class GenericReader {
ClearStackOnExit scope(*this);
IterativeParsingState state = IterativeParsingStartState;

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
while (is.Peek() != '\0') {
Token t = Tokenize(is.Peek());
IterativeParsingState n = Predict(state, t);
Expand All @@ -1421,7 +1471,8 @@ class GenericReader {
if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState)
break;

SkipWhitespace(is);
SkipWhitespaceAndComments<parseFlags>(is);
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
}

// Handle the end of file.
Expand Down
103 changes: 103 additions & 0 deletions test/unittest/readertest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1349,6 +1349,109 @@ TEST(Reader, ParseTerminationByHandler) {
TEST_TERMINATION(12, "{\"a\":[1]"); // non-empty array
}

TEST(Reader, ParseComments) {
const char* json =
"// Here is a one-line comment.\n"
"{// And here's another one\n"
" /*And here's an in-line one.*/\"hello\" : \"world\","
" \"t\" :/* And one with '*' symbol*/true ,"
"/* A multiline comment\n"
" goes here*/"
" \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3]"
"}/*And the last one to be sure */";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}

TEST(Reader, ParseEmptyInlineComment) {
const char* json = "{/**/\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}

TEST(Reader, ParseEmptyOnelineComment) {
const char* json = "{//\n\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}

TEST(Reader, ParseMultipleCommentsInARow) {
const char* json =
"{/* first comment *//* second */\n"
"/* third */ /*fourth*/// last one\n"
"\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}

TEST(Reader, InlineCommentsAreDisabledByDefault) {
{
const char* json = "{/* Inline comment. */\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_FALSE(reader.Parse<kParseDefaultFlags>(s, h));
}

{
const char* json =
"{\"hello\" : /* Multiline comment starts here\n"
" continues here\n"
" and ends here */\"world\", \"t\" :true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_FALSE(reader.Parse<kParseDefaultFlags>(s, h));
}
}

TEST(Reader, OnelineCommentsAreDisabledByDefault) {
const char* json = "{// One-line comment\n\"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_FALSE(reader.Parse<kParseDefaultFlags>(s, h));
}

TEST(Reader, EofAfterOneLineComment) {
const char* json = "{\"hello\" : \"world\" // EOF is here -->\0 \n}";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_FALSE(reader.Parse<kParseCommentsFlag>(s, h));
EXPECT_EQ(kParseErrorObjectMissCommaOrCurlyBracket, reader.GetParseErrorCode());
}

TEST(Reader, IncompleteMultilineComment) {
const char* json = "{\"hello\" : \"world\" /* EOF is here -->\0 */}";

StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_FALSE(reader.Parse<kParseCommentsFlag>(s, h));
EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode());
}

#ifdef __GNUC__
RAPIDJSON_DIAG_POP
#endif