From ad3292cd8e893747fae73a7c552d8c858a2be986 Mon Sep 17 00:00:00 2001 From: Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 27 Jun 2024 14:42:57 +0100 Subject: [PATCH] Fix line folding (#87) * Fix line folding to match specification * Add test case and remove TODO for fix * Remove indent creation from utility function * Update changelog * Return null if strings cannot be encoded in desired `ScalarStyle` > Fix invalid encoding for string with trailing line breaks or white-space * Clean up code > Update `_tryYamlEncodedPlain` to return null if string cannot be encoded > Make `_yamlEncodeFlowScalar` and `yamlEncodeBlockScalar` code concise * Remove unnecessary assertions and add docs > `_yamlEncodeFlowScalar` and `_yamlEncodeBlockScalar` always encode YamlScalars * Minor refactoring * Fix condition check for encoding as literal/folded * Remove keep chomping indicator references * Update lib/src/strings.dart --------- Co-authored-by: Jonas Finnemann Jensen Co-authored-by: Jonas Finnemann Jensen --- CHANGELOG.md | 7 + lib/src/editor.dart | 2 +- lib/src/list_mutations.dart | 10 +- lib/src/map_mutations.dart | 14 +- lib/src/strings.dart | 339 +++++++++++++++++++++--------------- test/string_test.dart | 4 +- 6 files changed, 219 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b5280..fd7591a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ list. ([#69](https://github.com/dart-lang/yaml_edit/issues/69)) +- Fix error thrown when inserting in nested list using `spliceList` method + ([#83](https://github.com/dart-lang/yaml_edit/issues/83)) + +- Fix error thrown when string has spaces when applying `ScalarStyle.FOLDED`. + ([#41](https://github.com/dart-lang/yaml_edit/issues/41)). Resolves + ([[#86](https://github.com/dart-lang/yaml_edit/issues/86)]). + ## 2.2.1 - Require Dart 3.0 diff --git a/lib/src/editor.dart b/lib/src/editor.dart index 2079a3f..54775cc 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -244,7 +244,7 @@ class YamlEditor { final end = getContentSensitiveEnd(_contents); final lineEnding = getLineEnding(_yaml); final edit = SourceEdit( - start, end - start, yamlEncodeBlockString(valueNode, 0, lineEnding)); + start, end - start, yamlEncodeBlock(valueNode, 0, lineEnding)); return _performEdit(edit, path, valueNode); } diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index 9a1a032..17da6dd 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -29,8 +29,8 @@ SourceEdit updateInList( final listIndentation = getListIndentation(yaml, list); final indentation = listIndentation + getIndentation(yamlEdit); final lineEnding = getLineEnding(yaml); - valueString = yamlEncodeBlockString( - wrapAsYamlNode(newValue), indentation, lineEnding); + valueString = + yamlEncodeBlock(wrapAsYamlNode(newValue), indentation, lineEnding); /// We prefer the compact nested notation for collections. /// @@ -52,7 +52,7 @@ SourceEdit updateInList( return SourceEdit(offset, end - offset, valueString); } else { - valueString = yamlEncodeFlowString(newValue); + valueString = yamlEncodeFlow(newValue); return SourceEdit(offset, currValue.span.length, valueString); } } @@ -141,7 +141,7 @@ SourceEdit _appendToBlockList( final newIndentation = listIndentation + getIndentation(yamlEdit); final lineEnding = getLineEnding(yaml); - var valueString = yamlEncodeBlockString(item, newIndentation, lineEnding); + var valueString = yamlEncodeBlock(item, newIndentation, lineEnding); if (isCollection(item) && !isFlowYamlCollectionNode(item) && !isEmpty(item)) { valueString = valueString.substring(newIndentation); } @@ -151,7 +151,7 @@ SourceEdit _appendToBlockList( /// Formats [item] into a new node for flow lists. String _formatNewFlow(YamlList list, YamlNode item, [bool isLast = false]) { - var valueString = yamlEncodeFlowString(item); + var valueString = yamlEncodeFlow(item); if (list.isNotEmpty) { if (isLast) { valueString = ', $valueString'; diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index 32f47b3..67665d9 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -54,7 +54,7 @@ SourceEdit _addToBlockMap( final yaml = yamlEdit.toString(); final newIndentation = getMapIndentation(yaml, map) + getIndentation(yamlEdit); - final keyString = yamlEncodeFlowString(wrapAsYamlNode(key)); + final keyString = yamlEncodeFlow(wrapAsYamlNode(key)); final lineEnding = getLineEnding(yaml); var formattedValue = ' ' * getMapIndentation(yaml, map); @@ -83,7 +83,7 @@ SourceEdit _addToBlockMap( } } - var valueString = yamlEncodeBlockString(newValue, newIndentation, lineEnding); + var valueString = yamlEncodeBlock(newValue, newIndentation, lineEnding); if (isCollection(newValue) && !isFlowYamlCollectionNode(newValue) && !isEmpty(newValue)) { @@ -100,8 +100,8 @@ SourceEdit _addToBlockMap( /// map. SourceEdit _addToFlowMap( YamlEditor yamlEdit, YamlMap map, YamlNode keyNode, YamlNode newValue) { - final keyString = yamlEncodeFlowString(keyNode); - final valueString = yamlEncodeFlowString(newValue); + final keyString = yamlEncodeFlow(keyNode); + final valueString = yamlEncodeFlow(newValue); // The -1 accounts for the closing bracket. if (map.isEmpty) { @@ -131,8 +131,8 @@ SourceEdit _replaceInBlockMap( getMapIndentation(yaml, map) + getIndentation(yamlEdit); final keyNode = getKeyNode(map, key); - var valueAsString = yamlEncodeBlockString( - wrapAsYamlNode(newValue), newIndentation, lineEnding); + var valueAsString = + yamlEncodeBlock(wrapAsYamlNode(newValue), newIndentation, lineEnding); if (isCollection(newValue) && !isFlowYamlCollectionNode(newValue) && !isEmpty(newValue)) { @@ -163,7 +163,7 @@ SourceEdit _replaceInBlockMap( SourceEdit _replaceInFlowMap( YamlEditor yamlEdit, YamlMap map, Object? key, YamlNode newValue) { final valueSpan = map.nodes[key]!.span; - final valueString = yamlEncodeFlowString(newValue); + final valueString = yamlEncodeFlow(newValue); return SourceEdit(valueSpan.start.offset, valueSpan.length, valueString); } diff --git a/lib/src/strings.dart b/lib/src/strings.dart index 7bcd5b4..dcb1b72 100644 --- a/lib/src/strings.dart +++ b/lib/src/strings.dart @@ -2,40 +2,27 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'package:collection/collection.dart'; import 'package:yaml/yaml.dart'; import 'utils.dart'; /// Given [value], tries to format it into a plain string recognizable by YAML. -/// If it fails, it defaults to returning a double-quoted string. /// /// Not all values can be formatted into a plain string. If the string contains /// an escape sequence, it can only be detected when in a double-quoted /// sequence. Plain strings may also be misinterpreted by the YAML parser (e.g. /// ' null'). -String _tryYamlEncodePlain(Object? value) { - if (value is YamlNode) { - AssertionError( - 'YamlNodes should not be passed directly into getSafeString!'); - } - - assertValidScalar(value); - - if (value is String) { - /// If it contains a dangerous character we want to wrap the result with - /// double quotes because the double quoted style allows for arbitrary - /// strings with "\" escape sequences. - /// - /// See 7.3.1 Double-Quoted Style - /// https://yaml.org/spec/1.2/spec.html#id2787109 - if (isDangerousString(value)) { - return _yamlEncodeDoubleQuoted(value); - } - - return value; - } - - return value.toString(); +/// +/// Returns `null` if [value] cannot be encoded as a plain string. +String? _tryYamlEncodePlain(String value) { + /// If it contains a dangerous character we want to wrap the result with + /// double quotes because the double quoted style allows for arbitrary + /// strings with "\" escape sequences. + /// + /// See 7.3.1 Double-Quoted Style + /// https://yaml.org/spec/1.2/spec.html#id2787109 + return isDangerousString(value) ? null : value; } /// Checks if [string] has unprintable characters according to @@ -67,142 +54,199 @@ String _yamlEncodeDoubleQuoted(String string) { return '"$buffer"'; } -/// Generates a YAML-safe single-quoted string. Automatically escapes -/// single-quotes. +/// Encodes [string] as YAML single quoted string. +/// +/// Returns `null`, if the [string] can't be encoded as single-quoted string. +/// This might happen if it contains line-breaks or [_hasUnprintableCharacters]. /// -/// It is important that we ensure that [string] is free of unprintable -/// characters by calling [_hasUnprintableCharacters] before invoking this -/// function. -String _tryYamlEncodeSingleQuoted(String string) { +/// See: https://yaml.org/spec/1.2.2/#732-single-quoted-style +String? _tryYamlEncodeSingleQuoted(String string) { // If [string] contains a newline we'll use double quoted strings instead. // Single quoted strings can represent newlines, but then we have to use an // empty line (replace \n with \n\n). But since leading spaces following // line breaks are ignored, we can't represent "\n ". // Thus, if the string contains `\n` and we're asked to do single quoted, // we'll fallback to a double quoted string. - // TODO: Consider if we should make '\n' an unprintedable, this might make - // folded strings into double quoted -- some work is needed here. - if (string.contains('\n')) { - return _yamlEncodeDoubleQuoted(string); - } + if (_hasUnprintableCharacters(string) || string.contains('\n')) return null; + final result = string.replaceAll('\'', '\'\''); return '\'$result\''; } -/// Generates a YAML-safe folded string. +/// Attempts to encode a [string] as a _YAML folded string_ and apply the +/// appropriate _chomping indicator_. /// -/// It is important that we ensure that [string] is free of unprintable -/// characters by calling [_hasUnprintableCharacters] before invoking this -/// function. -String _tryYamlEncodeFolded(String string, int indentation, String lineEnding) { - String result; - - final trimmedString = string.trimRight(); - final removedPortion = string.substring(trimmedString.length); - - if (removedPortion.contains('\n')) { - result = '>+\n${' ' * indentation}'; - } else { - result = '>-\n${' ' * indentation}'; - } +/// Returns `null`, if the [string] cannot be encoded as a _YAML folded +/// string_. +/// +/// **Examples** of folded strings. +/// ```yaml +/// # With the "strip" chomping indicator +/// key: >- +/// my folded +/// string +/// +/// # With the "keep" chomping indicator +/// key: >+ +/// my folded +/// string +/// ``` +/// +/// See: https://yaml.org/spec/1.2.2/#813-folded-style +String? _tryYamlEncodeFolded(String string, int indentSize, String lineEnding) { + // A string that starts with space or newline followed by space can't be + // encoded in folded mode. + if (string.isEmpty || string.trimLeft().length != string.length) return null; - /// Duplicating the newline for folded strings preserves it in YAML. - /// Assumes the user did not try to account for windows documents by using - /// `\r\n` already - return result + - trimmedString.replaceAll('\n', lineEnding * 2 + ' ' * indentation) + - removedPortion; -} + if (_hasUnprintableCharacters(string)) return null; -/// Generates a YAML-safe literal string. -/// -/// It is important that we ensure that [string] is free of unprintable -/// characters by calling [_hasUnprintableCharacters] before invoking this -/// function. -String _tryYamlEncodeLiteral( - String string, int indentation, String lineEnding) { - final result = '|-\n$string'; - - /// Assumes the user did not try to account for windows documents by using - /// `\r\n` already - return result.replaceAll('\n', lineEnding + ' ' * indentation); -} + // TODO: Are there other strings we can't encode in folded mode? -/// Returns [value] with the necessary formatting applied in a flow context -/// if possible. -/// -/// If [value] is a [YamlScalar], we try to respect its [YamlScalar.style] -/// parameter where possible. Certain cases make this impossible (e.g. a plain -/// string scalar that starts with '>'), in which case we will produce [value] -/// with default styling options. -String _yamlEncodeFlowScalar(YamlNode value) { - if (value is YamlScalar) { - assertValidScalar(value.value); - - final val = value.value; - if (val is String) { - if (_hasUnprintableCharacters(val) || - value.style == ScalarStyle.DOUBLE_QUOTED) { - return _yamlEncodeDoubleQuoted(val); - } + final indent = ' ' * indentSize; - if (value.style == ScalarStyle.SINGLE_QUOTED) { - return _tryYamlEncodeSingleQuoted(val); - } + /// Remove trailing `\n` & white-space to ease string folding + var trimmed = string.trimRight(); + final stripped = string.substring(trimmed.length); + + final trimmedSplit = + trimmed.replaceAll('\n', lineEnding + indent).split(lineEnding); + + /// Try folding to match specification: + /// * https://yaml.org/spec/1.2.2/#65-line-folding + trimmed = trimmedSplit.reduceIndexed((index, previous, current) { + var updated = current; + + /// If initially empty, this line holds only `\n` or white-space. This + /// tells us we don't need to apply an additional `\n`. + /// + /// See https://yaml.org/spec/1.2.2/#64-empty-lines + /// + /// If this line is not empty, we need to apply an additional `\n` if and + /// only if: + /// 1. The preceding line was non-empty too + /// 2. If the current line doesn't begin with white-space + /// + /// Such that we apply `\n` for `foo\nbar` but not `foo\n bar`. + if (current.trim().isNotEmpty && + trimmedSplit[index - 1].trim().isNotEmpty && + !current.replaceFirst(indent, '').startsWith(' ')) { + updated = lineEnding + updated; } - return _tryYamlEncodePlain(value.value); - } + /// Apply a `\n` by default. + return previous + lineEnding + updated; + }); - assertValidScalar(value); - return _tryYamlEncodePlain(value); + return '>-\n' + '$indent$trimmed' + '${stripped.replaceAll('\n', lineEnding + indent)}'; } -/// Returns [value] with the necessary formatting applied in a block context -/// if possible. +/// Attempts to encode a [string] as a _YAML literal string_ and apply the +/// appropriate _chomping indicator_. /// -/// If [value] is a [YamlScalar], we try to respect its [YamlScalar.style] -/// parameter where possible. Certain cases make this impossible (e.g. a folded -/// string scalar 'null'), in which case we will produce [value] with default -/// styling options. -String yamlEncodeBlockScalar( - YamlNode value, - int indentation, - String lineEnding, -) { - if (value is YamlScalar) { - assertValidScalar(value.value); +/// Returns `null`, if the [string] cannot be encoded as a _YAML literal +/// string_. +/// +/// **Examples** of literal strings. +/// ```yaml +/// # With the "strip" chomping indicator +/// key: |- +/// my literal +/// string +/// +/// # Without chomping indicator +/// key: | +/// my literal +/// string +/// ``` +/// +/// See: https://yaml.org/spec/1.2.2/#812-literal-style +String? _tryYamlEncodeLiteral( + String string, int indentSize, String lineEnding) { + if (string.isEmpty || string.trimLeft().length != string.length) return null; - final val = value.value; - if (val is String) { - if (_hasUnprintableCharacters(val)) { - return _yamlEncodeDoubleQuoted(val); - } + // A string that starts with space or newline followed by space can't be + // encoded in literal mode. + if (_hasUnprintableCharacters(string)) return null; - if (value.style == ScalarStyle.SINGLE_QUOTED) { - return _tryYamlEncodeSingleQuoted(val); - } + // TODO: Are there other strings we can't encode in literal mode? - // Strings with only white spaces will cause a misparsing - if (val.trim().length == val.length && val.isNotEmpty) { - if (value.style == ScalarStyle.FOLDED) { - return _tryYamlEncodeFolded(val, indentation, lineEnding); - } + final indent = ' ' * indentSize; - if (value.style == ScalarStyle.LITERAL) { - return _tryYamlEncodeLiteral(val, indentation, lineEnding); - } - } - } + /// Simplest block style. + /// * https://yaml.org/spec/1.2.2/#812-literal-style + return '|-\n$indent${string.replaceAll('\n', lineEnding + indent)}'; +} + +/// Encodes a flow [YamlScalar] based on the provided [YamlScalar.style]. +/// +/// Falls back to [ScalarStyle.DOUBLE_QUOTED] if the [yamlScalar] cannot be +/// encoded with the [YamlScalar.style] or with [ScalarStyle.PLAIN] when the +/// [yamlScalar] is not a [String]. +String _yamlEncodeFlowScalar(YamlScalar yamlScalar) { + final YamlScalar(:value, :style) = yamlScalar; + + if (value is! String) { + return value.toString(); + } + + switch (style) { + /// Only encode as double-quoted if it's a string. + case ScalarStyle.DOUBLE_QUOTED: + return _yamlEncodeDoubleQuoted(value); - return _tryYamlEncodePlain(value.value); + case ScalarStyle.SINGLE_QUOTED: + return _tryYamlEncodeSingleQuoted(value) ?? + _yamlEncodeDoubleQuoted(value); + + /// Cast into [String] if [null] as this condition only returns [null] + /// for a [String] that can't be encoded. + default: + return _tryYamlEncodePlain(value) ?? _yamlEncodeDoubleQuoted(value); } +} +/// Encodes a block [YamlScalar] based on the provided [YamlScalar.style]. +/// +/// Falls back to [ScalarStyle.DOUBLE_QUOTED] if the [yamlScalar] cannot be +/// encoded with the [YamlScalar.style] provided. +String _yamlEncodeBlockScalar( + YamlScalar yamlScalar, + int indentation, + String lineEnding, +) { + final YamlScalar(:value, :style) = yamlScalar; assertValidScalar(value); - /// The remainder of the possibilities are similar to how [getFlowScalar] - /// treats [value]. - return _yamlEncodeFlowScalar(value); + if (value is! String) { + return value.toString(); + } + + switch (style) { + /// Prefer 'plain', fallback to "double quoted" + case ScalarStyle.PLAIN: + return _tryYamlEncodePlain(value) ?? _yamlEncodeDoubleQuoted(value); + + // Prefer 'single quoted', fallback to "double quoted" + case ScalarStyle.SINGLE_QUOTED: + return _tryYamlEncodeSingleQuoted(value) ?? + _yamlEncodeDoubleQuoted(value); + + /// Prefer folded string, fallback to "double quoted" + case ScalarStyle.FOLDED: + return _tryYamlEncodeFolded(value, indentation, lineEnding) ?? + _yamlEncodeDoubleQuoted(value); + + /// Prefer literal string, fallback to "double quoted" + case ScalarStyle.LITERAL: + return _tryYamlEncodeLiteral(value, indentation, lineEnding) ?? + _yamlEncodeDoubleQuoted(value); + + /// Prefer plain, fallback to "double quoted" + default: + return _tryYamlEncodePlain(value) ?? _yamlEncodeDoubleQuoted(value); + } } /// Returns [value] with the necessary formatting applied in a flow context. @@ -212,34 +256,34 @@ String yamlEncodeBlockScalar( /// string scalar that starts with '>', a child having a block style /// parameters), in which case we will produce [value] with default styling /// options. -String yamlEncodeFlowString(YamlNode value) { +String yamlEncodeFlow(YamlNode value) { if (value is YamlList) { final list = value.nodes; - final safeValues = list.map(yamlEncodeFlowString); + final safeValues = list.map(yamlEncodeFlow); return '[${safeValues.join(', ')}]'; } else if (value is YamlMap) { final safeEntries = value.nodes.entries.map((entry) { - final safeKey = yamlEncodeFlowString(entry.key as YamlNode); - final safeValue = yamlEncodeFlowString(entry.value); + final safeKey = yamlEncodeFlow(entry.key as YamlNode); + final safeValue = yamlEncodeFlow(entry.value); return '$safeKey: $safeValue'; }); return '{${safeEntries.join(', ')}}'; } - return _yamlEncodeFlowScalar(value); + return _yamlEncodeFlowScalar(value as YamlScalar); } /// Returns [value] with the necessary formatting applied in a block context. -String yamlEncodeBlockString( +String yamlEncodeBlock( YamlNode value, int indentation, String lineEnding, ) { const additionalIndentation = 2; - if (!isBlockNode(value)) return yamlEncodeFlowString(value); + if (!isBlockNode(value)) return yamlEncodeFlow(value); final newIndentation = indentation + additionalIndentation; @@ -251,8 +295,7 @@ String yamlEncodeBlockString( final children = value.nodes; safeValues = children.map((child) { - var valueString = - yamlEncodeBlockString(child, newIndentation, lineEnding); + var valueString = yamlEncodeBlock(child, newIndentation, lineEnding); if (isCollection(child) && !isFlowYamlCollectionNode(child)) { valueString = valueString.substring(newIndentation); } @@ -265,14 +308,20 @@ String yamlEncodeBlockString( if (value.isEmpty) return '${' ' * indentation}{}'; return value.nodes.entries.map((entry) { - final safeKey = yamlEncodeFlowString(entry.key as YamlNode); + final MapEntry(:key, :value) = entry; + + final safeKey = yamlEncodeFlow(key as YamlNode); final formattedKey = ' ' * indentation + safeKey; - final formattedValue = - yamlEncodeBlockString(entry.value, newIndentation, lineEnding); + + final formattedValue = yamlEncodeBlock( + value, + newIndentation, + lineEnding, + ); /// Empty collections are always encoded in flow-style, so new-line must /// be avoided - if (isCollection(entry.value) && !isEmpty(entry.value)) { + if (isCollection(value) && !isEmpty(value)) { return '$formattedKey:$lineEnding$formattedValue'; } @@ -280,7 +329,11 @@ String yamlEncodeBlockString( }).join(lineEnding); } - return yamlEncodeBlockScalar(value, newIndentation, lineEnding); + return _yamlEncodeBlockScalar( + value as YamlScalar, + newIndentation, + lineEnding, + ); } /// List of unprintable characters. diff --git a/test/string_test.dart b/test/string_test.dart index 98d536f..b92c20a 100644 --- a/test/string_test.dart +++ b/test/string_test.dart @@ -9,6 +9,8 @@ import 'package:yaml_edit/yaml_edit.dart'; final _testStrings = [ "this is a fairly' long string with\nline breaks", 'whitespace\n after line breaks', + 'whitespace\n \nbetween line breaks', + '\n line break at the start', 'word', 'foo bar', 'foo\nbar', @@ -21,7 +23,7 @@ final _testStrings = [ final _scalarStyles = [ ScalarStyle.ANY, ScalarStyle.DOUBLE_QUOTED, - //ScalarStyle.FOLDED, // TODO: Fix this test case! + ScalarStyle.FOLDED, ScalarStyle.LITERAL, ScalarStyle.PLAIN, ScalarStyle.SINGLE_QUOTED,