Merge pull request #1152 from tenderlove/interpolated-heredocs

Fix heredocs inside %W and %w lists

include/yarp/util/yp_char.h

@@ -15,7 +15,7 @@ size_t yp_strspn_whitespace(const char *string, ptrdiff_t length);
 // whitespace while also tracking the location of each newline. Disallows
 // searching past the given maximum number of characters.
 size_t
-yp_strspn_whitespace_newlines(const char *string, long length, yp_newline_list_t *newline_list);
+yp_strspn_whitespace_newlines(const char *string, long length, yp_newline_list_t *newline_list, bool);
 
 // Returns the number of characters at the start of the string that are inline
 // whitespace. Disallows searching past the given maximum number of characters.

src/util/yp_char.c

@@ -75,15 +75,20 @@ yp_strspn_whitespace(const char *string, ptrdiff_t length) {
 // whitespace while also tracking the location of each newline. Disallows
 // searching past the given maximum number of characters.
 size_t
-yp_strspn_whitespace_newlines(const char *string, long length, yp_newline_list_t *newline_list) {
+yp_strspn_whitespace_newlines(const char *string, long length, yp_newline_list_t *newline_list, bool stop_at_newline) {
     if (length <= 0) return 0;
 
     size_t size = 0;
     size_t maximum = (size_t) length;
 
     while (size < maximum && (yp_char_table[(unsigned char) string[size]] & YP_CHAR_BIT_WHITESPACE)) {
         if (string[size] == '\n') {
-            yp_newline_list_append(newline_list, string + size);
+            if (stop_at_newline) {
+                return size + 1;
+            }
+            else {
+                yp_newline_list_append(newline_list, string + size);
+            }
         }
 
         size++;

src/util/yp_newline_list.c

@@ -25,13 +25,15 @@ yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity
 bool
 yp_newline_list_append(yp_newline_list_t *list, const char *cursor) {
     if (list->size == list->capacity) {
-        list->capacity = list->capacity * 3 / 2;
+        list->capacity = (list->capacity * 3) / 2;
         list->offsets = (size_t *) realloc(list->offsets, list->capacity * sizeof(size_t));
         if (list->offsets == NULL) return false;
     }
 
     assert(cursor >= list->start);
-    list->offsets[list->size++] = (size_t) (cursor - list->start + 1);
+    size_t newline_offset = (size_t) (cursor - list->start + 1);
+    assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]);
+    list->offsets[list->size++] = newline_offset;
 
     return true;
 }

src/yarp.c

@@ -6505,14 +6505,26 @@ parser_lex(yp_parser_t *parser) {
             }
         }
         case YP_LEX_LIST:
+            if (parser->next_start != NULL) {
+                parser->current.end = parser->next_start;
+                parser->next_start = NULL;
+            }
+
             // First we'll set the beginning of the token.
             parser->current.start = parser->current.end;
 
             // If there's any whitespace at the start of the list, then we're
             // going to trim it off the beginning and create a new token.
             size_t whitespace;
-            if ((whitespace = yp_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list)) > 0) {
+
+            bool should_stop = parser->heredoc_end;
+
+            if ((whitespace = yp_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list, should_stop)) > 0) {
                 parser->current.end += whitespace;
+                if (parser->current.end[-1] == '\n') {
+                    // mutates next_start
+                    parser_flush_heredoc_end(parser);
+                }
                 LEX(YP_TOKEN_WORDS_SEP);
             }
 

test/parse_test.rb

@@ -28,7 +28,6 @@ def test_empty_string
 
   known_failures = %w[
     seattlerb/heredoc_nested.txt
-    seattlerb/pct_w_heredoc_interp_nested.txt
   ]
 
   def find_source_file_node(node)