Skip to content

Commit

Permalink
LineInfo.extend_* now modify "line" for you.
Browse files Browse the repository at this point in the history 
You pass in the segment you want to move to the relevant
LineInfo field (leading, trailing, or comment), and the
"line" that starts or ends with that segment, and the
method returns the appropriately-modified line.
  • Loading branch information
@larryhastings
larryhastings committed Jul 14, 2024
1 parent 7f883e1 commit 96f5c8b
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 55 deletions.
98 changes: 45 additions & 53 deletions big/text.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2224,20 +2224,28 @@ def __init__(self, lines, line, line_number, column_number, *, leading=None, tra
def detab(self, s):
return self.lines.detab(s)

def extend_leading(self, s, length=None):
def extend_leading(self, s, line):
assert line.startswith(s), f"{line=} doesn't start with {s=}"
self.leading += s
if length == None:
detabbed = self.detab(s)
length = len(detabbed)
line = line[len(s):]
detabbed = self.detab(s)
length = len(detabbed)
self.column_number += length
return line

def extend_trailing(self, s):
def extend_trailing(self, s, line):
assert line.endswith(s)
self.trailing = s + self.trailing
line = line[:-len(s)]
return line

def extend_comment(self, s):
def extend_comment(self, s, line):
empty = b"" if self._is_bytes else ""
assert line.endswith(s)
self.comment = s + self.trailing + self.comment
self.trailing = empty
line = line[:-len(s)]
return line

def __repr__(self):
names = list(self.__dict__)
Expand Down Expand Up @@ -2393,15 +2401,15 @@ def lines_rstrip(li, separators=None):
rstripped = line.rstrip()
if rstripped != line:
trailing = line[len(rstripped):]
info.extend_trailing(trailing)
yield (info, rstripped)
line = info.extend_trailing(trailing, line)
yield (info, line)
return

for info, line in li:
rstripped = multistrip(line, separators, left=False, right=True)
if rstripped != line:
trailing = line[len(rstripped):]
info.extend_trailing(trailing)
line = info.extend_trailing(trailing, line)
yield (info, rstripped)


Expand All @@ -2423,13 +2431,13 @@ def lines_strip(li, separators=None):
leading = trailing = None
if line:
stripped = multistrip(line, separators)
leading, line, trailing = line.partition(stripped)
leading, _, trailing = line.partition(stripped)

if leading:
info.extend_leading(leading)
line = info.extend_leading(leading, line)

if trailing:
info.extend_trailing(trailing)
line = info.extend_trailing(trailing, line)

yield (info, line)

Expand All @@ -2445,7 +2453,6 @@ def lines_strip(li, separators=None):
if not lstripped:
# line was all whitespace.
leading = line
line = lstripped # aka empty
else:
if len(line) != len(lstripped):
# we stripped leading whitespace, preserve it
Expand All @@ -2454,13 +2461,12 @@ def lines_strip(li, separators=None):
rstripped = lstripped.rstrip()
if len(lstripped) != len(rstripped):
trailing = lstripped[len(rstripped):]
line = rstripped

if leading:
info.extend_leading(leading)
line = info.extend_leading(leading, line)

if trailing:
info.extend_trailing(trailing)
line = info.extend_trailing(trailing, line)

yield (info, line)

Expand Down Expand Up @@ -2676,6 +2682,7 @@ def lines_strip_line_comments(li, line_comment_markers, *,
def lines_strip_line_comments(li, line_comment_splitter, quotes, multiline_quotes, escape, rstrip):
state = None
starting_pair_for_state = None

for info, line in li:
# print(f"[!!!] {info=}\n[!!!] {line=}\n[!!!] {state=}\n")

Expand All @@ -2684,35 +2691,27 @@ def lines_strip_line_comments(li, line_comment_splitter, quotes, multiline_quote
else:
i = iter( (('', line, ''),) )

segments = []
append = segments.append

line_comment_segments = None

for leading_quote, segment, trailing_quote in i:
# print(f"-- {leading_quote=} {segment=} {trailing_quote=}")
if leading_quote:
append(leading_quote)
append(segment)
append(trailing_quote)
continue

if state:
# we're still in a quote from a previous line.
assert not leading_quote
append(segment)
# assert not leading_quote
if trailing_quote:
append(trailing_quote)
assert trailing_quote == state
state = None
starting_pair_for_state = None
else:
# we didn't find the ending quote from the previous line,
# so this should be the entire line
assert segment == line
continue

fields = line_comment_splitter(segment, maxsplit=1)
if len(fields) == 1:
append(segment)
continue

# found a comment marker in an unquoted segment!
Expand All @@ -2723,27 +2722,25 @@ def lines_strip_line_comments(li, line_comment_splitter, quotes, multiline_quote
for triplet in i:
line_comment_segments.extend(triplet)
assert line_comment_segments
info.extend_comment(empty_join(line_comment_segments))
line = info.extend_comment(empty_join(line_comment_segments), line)

# do this *after* extend_comment,
# extend_comment also eats trailing
if rstrip:
stripped = leading.rstrip()
if stripped != leading:
info.extend_trailing(leading[len(stripped):])
leading = stripped
append(leading)

line = empty_join(segments)

yield (info, line)
line = info.extend_trailing(leading[len(stripped):], line)
break

if not line_comment_segments:
if leading_quote and not trailing_quote:
if leading_quote not in multiline_quotes:
raise SyntaxError(f"Unterminated quoted string in line {info.line_number}: {line}")
state = leading_quote
starting_pair_for_state = info, line

yield (info, line)

if state:
info, line = starting_pair_for_state
raise SyntaxError(f"Unterminated quoted string in line {info.line_number}: {line}")
Expand Down Expand Up @@ -2811,32 +2808,30 @@ def lines_strip_indent(li):
empty = ''

lstripped = line.lstrip()
original_leading = line[:len(line) - len(lstripped)]
if not lstripped:
info.extend_leading(line)
blank_lines.append((info, empty))
line = info.extend_leading(line, line)
blank_lines.append((info, line))
# print(f"BL+ {info=} {lstripped=}")
continue

leading = li.detab(original_leading)
len_leading = len(leading)
# print(f"{leadings=} {line=} {leading=} {len_leading=}")
if leading.rstrip(space):
raise ValueError(f"lines_strip_indent can't handle leading whitespace character {leading[0]!r}")
if not leading:
line = info.extend_leading(line[:len(line) - len(lstripped)], line)
column_number = info.column_number

if column_number == info.lines.column_number:
# this line doesn't start with whitespace; text is at column 0.
# outdent to zero.
assert not info.leading
indent = 0
leadings.clear()
new_indent = False
# in all the remaining else cases, the line starts with whitespace. and...
elif not leadings:
# this is the first indent.
new_indent = True
elif leadings[-1] == len_leading:
elif leadings[-1] == column_number:
# indent is unchanged.
new_indent = False
elif len_leading > leadings[-1]:
elif column_number > leadings[-1]:
# we are indented further than the previously observed indent.
new_indent = True
else:
Expand All @@ -2847,19 +2842,19 @@ def lines_strip_indent(li):
indent -= 1
while leadings:
l = leadings[-1]
if l >= len_leading:
if l > len_leading:
if l >= column_number:
if l > column_number:
leadings.clear()
break
leadings.pop()
indent -= 1
if not leadings:
raise IndentationError(f"line {info.line_number} column {len_leading + info.column_number}: unindent does not match any outer indentation level")
raise IndentationError(f"line {info.line_number} column {column_number}: unindent doesn't match any outer indentation level")
new_indent = False

# print(f" >> {leadings=} {new_indent=}")
if new_indent:
leadings.append(len_leading)
leadings.append(column_number)
indent += 1

if blank_lines:
Expand All @@ -2869,10 +2864,7 @@ def lines_strip_indent(li):
yield pair
blank_lines.clear()

info.extend_leading(original_leading)
info.indent = indent
line = lstripped

yield (info, line)

# flush trailing blank lines
Expand Down
4 changes: 2 additions & 2 deletions tests/test_text.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3346,8 +3346,8 @@ def LineInfo(lines, line, line_number, column_number, end=_sentinel, **kwargs):
with self.assertRaises(IndentationError):
test(lines, [], tab_width=4)

with self.assertRaises(ValueError):
test("first line\n \u3000 second line\nthird line\n", [])
# with self.assertRaises(ValueError):
# test("first line\n \u3000 second line\nthird line\n", [])

def test_lines_misc(self):
## error handling
Expand Down

0 comments on commit 96f5c8b

Please sign in to comment.