Newlines: Parse \f and normalize in comments

Also moves the `normalize*` functions and `rtrim` to a separate object
file, so they can be unit tested quickly and easily.

I'm not entirely sure if the new `normalize_newlines` method is called
in the right places.

Refs sass#2843

Makefile.conf

@@ -29,6 +29,7 @@ SOURCES = \
 	bind.cpp \
 	file.cpp \
 	util.cpp \
+	util_string.cpp \
 	json.cpp \
 	units.cpp \
 	values.cpp \

src/eval.cpp

@@ -30,6 +30,7 @@
 #include "expand.hpp"
 #include "color_maps.hpp"
 #include "sass_functions.hpp"
+#include "util_string.hpp"
 
 namespace Sass {
 

src/fn_miscs.cpp

@@ -2,6 +2,7 @@
 #include "expand.hpp"
 #include "fn_utils.hpp"
 #include "fn_miscs.hpp"
+#include "util_string.hpp"
 
 namespace Sass {
 

src/fn_utils.cpp

@@ -4,6 +4,7 @@
 
 #include "parser.hpp"
 #include "fn_utils.hpp"
+#include "util_string.hpp"
 
 namespace Sass {
 

src/inspect.cpp

@@ -14,6 +14,7 @@
 #include "context.hpp"
 #include "listize.hpp"
 #include "color_maps.hpp"
+#include "util_string.hpp"
 #include "utf8/checked.h"
 
 namespace Sass {
@@ -685,13 +686,19 @@ namespace Sass {
 
   void Inspect::operator()(String_Constant* s)
   {
-    append_token(s->value(), s);
+    if (in_comment) {
+      append_token(Util::normalize_newlines(s->value()), s);
+    } else {
+      append_token(s->value(), s);
+    }
   }
 
   void Inspect::operator()(String_Quoted* s)
   {
     if (const char q = s->quote_mark()) {
       append_token(quote(s->value(), q), s);
+    } else if (in_comment) {
+      append_token(Util::normalize_newlines(s->value()), s);
     } else {
       append_token(s->value(), s);
     }

src/lexer.cpp

@@ -153,11 +153,11 @@ namespace Sass {
     // Match word boundary (zero-width lookahead).
     const char* word_boundary(const char* src) { return is_character(*src) || *src == '#' ? 0 : src; }
 
-    // Match linefeed /(?:\n|\r\n?)/
+    // Match linefeed /(?:\n|\r\n?|\f)/
     const char* re_linebreak(const char* src)
     {
       // end of file or unix linefeed return here
-      if (*src == 0 || *src == '\n') return src + 1;
+      if (*src == 0 || *src == '\n' || *src == '\f') return src + 1;
       // a carriage return may optionally be followed by a linefeed
       if (*src == '\r') return *(src + 1) == '\n' ? src + 2 : src + 1;
       // no linefeed
@@ -169,7 +169,7 @@ namespace Sass {
     const char* end_of_line(const char* src)
     {
       // end of file or unix linefeed return here
-      return *src == 0 || *src == '\n' || *src == '\r' ? src : 0;
+      return *src == 0 || *src == '\n' || *src == '\r' || *src == '\f' ? src : 0;
     }
 
     // Assert end_of_file boundary (/\z/)

src/output.cpp

@@ -5,6 +5,7 @@
 #include "ast.hpp"
 #include "output.hpp"
 #include "util.hpp"
+#include "util_string.hpp"
 
 namespace Sass {
 
@@ -92,7 +93,6 @@ namespace Sass {
 
   void Output::operator()(Comment* c)
   {
-    std::string txt = c->text()->to_string(opt);
     // if (indentation && txt == "/**/") return;
     bool important = c->is_important();
     if (output_style() != COMPRESSED || important) {
@@ -320,7 +320,7 @@ namespace Sass {
     } else if (!in_comment) {
       append_token(string_to_output(s->value()), s);
     } else {
-      append_token(s->value(), s);
+      append_token(Util::normalize_newlines(s->value()), s);
     }
   }
 
@@ -333,7 +333,7 @@ namespace Sass {
     if (!in_comment && !in_custom_property) {
       append_token(string_to_output(value), s);
     } else {
-      append_token(value, s);
+      append_token(Util::normalize_newlines(value), s);
     }
   }
 

src/parser.cpp

@@ -11,6 +11,7 @@
 #include "color_maps.hpp"
 #include "sass/functions.h"
 #include "error_handling.hpp"
+#include "util_string.hpp"
 
 // Notes about delayed: some ast nodes can have delayed evaluation so
 // they can preserve their original semantics if needed. This is most

src/prelexer.cpp

@@ -263,7 +263,7 @@ namespace Sass {
       >(src);
     }
 
-    // Match a line comment (/.*?(?=\n|\r\n?|\Z)/.
+    // Match a line comment (/.*?(?=\n|\r\n?|\f|\Z)/.
     const char* line_comment(const char* src)
     {
       return sequence<

src/util.cpp

@@ -183,16 +183,21 @@ namespace Sass {
 
   std::string escape_string(const std::string& str)
   {
-    std::string out("");
-    for (auto i : str) {
-      if (i == '\n') {
-        out += "\\n";
-      } else if (i == '\r') {
-        out += "\\r";
-      } else if (i == '\t') {
-        out += "\\t";
-      } else {
-        out += i;
+    std::string out;
+    out.reserve(str.size());
+    for (char c : str) {
+      switch (c) {
+        case '\n':
+          out.append("\\n");
+          break;
+        case '\r':
+          out.append("\\r");
+          break;
+        case '\f':
+          out.append("\\f");
+          break;
+        default:
+          out += c;
       }
     }
     return out;
@@ -508,33 +513,6 @@ namespace Sass {
   }
 
   namespace Util {
-    using std::string;
-
-    std::string rtrim(const std::string &str) {
-      std::string trimmed = str;
-      size_t pos_ws = trimmed.find_last_not_of(" \t\n\v\f\r");
-      if (pos_ws != std::string::npos)
-      { trimmed.erase(pos_ws + 1); }
-      else { trimmed.clear(); }
-      return trimmed;
-    }
-
-    std::string normalize_underscores(const std::string& str) {
-      std::string normalized = str;
-      for(size_t i = 0, L = normalized.length(); i < L; ++i) {
-        if(normalized[i] == '_') {
-          normalized[i] = '-';
-        }
-      }
-      return normalized;
-    }
-
-    std::string normalize_decimals(const std::string& str) {
-      std::string prefix = "0";
-      std::string normalized = str;
-
-      return normalized[0] == '.' ? normalized.insert(0, prefix) : normalized;
-    }
 
     bool isPrintable(Ruleset* r, Sass_Output_Style style) {
       if (r == NULL) {

src/util.hpp

@@ -91,11 +91,6 @@ namespace Sass {
 
   namespace Util {
 
-    std::string rtrim(const std::string& str);
-
-    std::string normalize_underscores(const std::string& str);
-    std::string normalize_decimals(const std::string& str);
-
     bool isPrintable(Ruleset* r, Sass_Output_Style style = NESTED);
     bool isPrintable(Supports_Block* r, Sass_Output_Style style = NESTED);
     bool isPrintable(Media_Block* r, Sass_Output_Style style = NESTED);

src/util_string.cpp

@@ -0,0 +1,60 @@
+#include "util_string.hpp"
+
+#include <cstring>
+
+namespace Sass {
+namespace Util {
+
+std::string rtrim(const std::string &str) {
+  std::string trimmed = str;
+  size_t pos_ws = trimmed.find_last_not_of(" \t\n\v\f\r");
+  if (pos_ws != std::string::npos) {
+    trimmed.erase(pos_ws + 1);
+  } else {
+    trimmed.clear();
+  }
+  return trimmed;
+}
+
+std::string normalize_newlines(const std::string& str) {
+  std::string result;
+  result.reserve(str.size());
+  const char *cur = str.data();
+  std::size_t len = str.length();
+  while (len > 0) {
+    const char *newline = static_cast<const char *>(std::memchr(cur, '\r', len));
+    const char *next;
+    if (newline != nullptr) {
+      next = newline + (*(newline + 1) == '\n' ? 2 : 1);
+    } else {
+      newline = static_cast<const char *>(std::memchr(cur, '\f', len));
+      if (newline == nullptr) break;
+      next = newline + 1;
+    }
+    result.append(cur, newline - cur).append(1, '\n');
+    len -= (next - cur);
+    cur = next;
+  }
+  result.append(cur, len);
+  return result;
+}
+
+std::string normalize_underscores(const std::string& str) {
+  std::string normalized = str;
+  for(size_t i = 0, L = normalized.length(); i < L; ++i) {
+    if(normalized[i] == '_') {
+      normalized[i] = '-';
+    }
+  }
+  return normalized;
+}
+
+std::string normalize_decimals(const std::string& str) {
+  std::string prefix = "0";
+  std::string normalized = str;
+
+  return normalized[0] == '.' ? normalized.insert(0, prefix) : normalized;
+}
+
+}  // namespace Sass
+}  // namespace Util

src/util_string.hpp

@@ -0,0 +1,17 @@
+#ifndef SASS_UTIL_STRING_H
+#define SASS_UTIL_STRING_H
+
+#include <string>
+
+namespace Sass {
+namespace Util {
+
+std::string rtrim(const std::string& str);
+
+std::string normalize_newlines(const std::string& str);
+std::string normalize_underscores(const std::string& str);
+std::string normalize_decimals(const std::string& str);
+
+}  // namespace Sass
+}  // namespace Util
+#endif  // SASS_UTIL_STRING_H

test/Makefile

@@ -1,16 +1,22 @@
 CXX ?= c++
 CXXFLAGS := -I ../include/ -std=c++11 -fsanitize=address -g -O1 -fno-omit-frame-pointer
 
-test: test_shared_ptr
+test: test_shared_ptr test_util_string
 
 test_shared_ptr: build/test_shared_ptr
 	@ASAN_OPTIONS="symbolize=1" build/test_shared_ptr
 
+test_util_string: build/test_util_string
+	@ASAN_OPTIONS="symbolize=1" build/test_util_string
+
 build:
 	@mkdir build
 
-build/test_shared_ptr: | build
-	@$(CXX) $(CXXFLAGS) -o build/test_shared_ptr test_shared_ptr.cpp ../src/memory/SharedPtr.cpp
+build/test_shared_ptr: test_shared_ptr.cpp ../src/memory/SharedPtr.cpp | build
+	$(CXX) $(CXXFLAGS) -o build/test_shared_ptr test_shared_ptr.cpp ../src/memory/SharedPtr.cpp
+
+build/test_util_string: test_util_string.cpp ../src/util_string.cpp | build
+	$(CXX) $(CXXFLAGS) -o build/test_util_string test_util_string.cpp ../src/util_string.cpp
 
 clean: | build
 	rm -rf build