Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Wire/issues/272 #703

Merged
merged 2 commits into from
Aug 2, 2023
Merged

Wire/issues/272 #703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e247547
In creating tests to show #272 had been fixed, other issues were foun…
peter-lawrey Jul 28, 2023
87ae276
Improve test coverage for decoding different values
peter-lawrey Jul 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/main/java/net/openhft/chronicle/wire/BinaryWire.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1983,8 +1983,12 @@ public <T, K> WireOut sequence(T t, K kls, @NotNull TriConsumer<T, K, ValueOut>
@NotNull
@Override
public WireOut marshallable(@NotNull WriteMarshallable object) throws InvalidMarshallableException {
if (bytes.retainedHexDumpDescription())
bytes.writeHexDumpDescription(object.getClass().getSimpleName());
if (bytes.retainedHexDumpDescription()) {
String simpleName = object.getClass().getSimpleName();
if (simpleName.contains("$$Lambda"))
simpleName = "Marshallable";
bytes.writeHexDumpDescription(simpleName);
}
final BinaryLengthLength binaryLengthLength = object.binaryLengthLength();
long pos = binaryLengthLength.initialise(bytes);

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/net/openhft/chronicle/wire/ValueOut.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -832,6 +832,8 @@ default WireOut object(@Nullable Object value) throws InvalidMarshallableExcepti
return bool(((AtomicBoolean) value).get());
case "java.lang.String":
return text((String) value);
case "java.lang.StringBuilder":
return text((StringBuilder) value);
case "java.lang.Byte":
return fixedInt8((byte) value);
case "java.lang.Boolean":
Expand Down
173 changes: 115 additions & 58 deletions src/main/java/net/openhft/chronicle/wire/YamlWire.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.*;
import java.util.function.*;

Expand Down Expand Up @@ -178,6 +179,79 @@ private static <ACS extends Appendable & CharSequence> void unescape(@NotNull AC
AppendableUtil.setLength(sb, end);
}

static void removeUnderscore(@NotNull StringBuilder s) {
int i = 0;
for (int j = 0; j < s.length(); j++) {
char ch = s.charAt(j);
s.setCharAt(i, ch);
if (ch != '_')
i++;
}
s.setLength(i);
}

@Nullable
static Object readNumberOrTextFrom(char bq, final @Nullable StringBuilder s) {
if (leaveUnparsed(bq, s))
return s;

StringBuilder sb = s;
// YAML octal notation
if (StringUtils.startsWith(s, "0o")) {
sb = new StringBuilder(s);
sb.deleteCharAt(1);
}

if (s.indexOf("_") >= 0) {
sb = new StringBuilder(s);
removeUnderscore(sb);
}

String ss = sb.toString();
try {
return Long.decode(ss);
} catch (NumberFormatException fallback) {
// fallback
}
try {
return Double.parseDouble(ss);
} catch (NumberFormatException fallback) {
// fallback
}
try {
return parseDateOrTime(s, ss);
} catch (DateTimeParseException fallback) {
// fallback
}
// the original string without underscores removed
return s;
}


private static boolean leaveUnparsed(char bq, @Nullable StringBuilder s) {
return s == null
|| bq != 0
|| s.length() < 1
|| s.length() > 40
|| "0123456789.+-".indexOf(s.charAt(0)) < 0;
}

private static TemporalAccessor parseDateOrTime(StringBuilder s, String ss) {
if (s.length() == 7 && s.charAt(1) == ':') {
return LocalTime.parse('0' + ss);
}
if (s.length() == 8 && s.charAt(2) == ':') {
return LocalTime.parse(s);
}
if (s.length() == 10) {
return LocalDate.parse(s);
}
if (s.length() >= 22) {
return ZonedDateTime.parse(s);
}
throw new DateTimeParseException("Unable to parse date or time", s, 0);
}

@Override
public boolean isBinary() {
return false;
Expand Down Expand Up @@ -308,22 +382,23 @@ public void copyTo(@NotNull WireOut wire) throws InvalidMarshallableException {
wire.bytes().write(this.bytes, yt.blockStart(), bytes0.readLimit() - yt.blockStart);
this.bytes.readPosition(this.bytes.readLimit());
} else {
while (!isEmpty()) {
while (!endOfDocument()) {
copyOne(wire, true);
yt.next();
}
}
}

private void copyOne(WireOut wire, boolean nested) throws InvalidMarshallableException {
ValueOut wireValueOut = wire.getValueOut();
switch (yt.current()) {
case NONE:
break;
case COMMENT:
wire.writeComment(yt.text());
break;
case TAG:
wire.getValueOut().typePrefix(yt.text());
wireValueOut.typePrefix(yt.text());
yt.next();
copyOne(wire, true);
yt.next();
Expand All @@ -334,7 +409,7 @@ private void copyOne(WireOut wire, boolean nested) throws InvalidMarshallableExc
break;
case DIRECTIVES_END:
yt.next();
while (!isEmpty()) {
while (!endOfDocument()) {
copyOne(wire, false);
yt.next();
}
Expand All @@ -347,7 +422,7 @@ private void copyOne(WireOut wire, boolean nested) throws InvalidMarshallableExc
case MAPPING_START: {
if (nested) {
yt.next();
wire.getValueOut().marshallable(w -> {
wireValueOut.marshallable(w -> {
while (yt.current() == YamlToken.MAPPING_KEY) {
copyMappingKey(wire, true);
yt.next();
Expand All @@ -363,7 +438,7 @@ private void copyOne(WireOut wire, boolean nested) throws InvalidMarshallableExc
case SEQUENCE_START: {
yt.next();
YamlWire yw = this;
wire.getValueOut().sequence(w -> {
wireValueOut.sequence(w -> {
while (yt.current() != YamlToken.SEQUENCE_END) {
yw.copyOne(w.wireOut(), true);
yw.yt.next();
Expand All @@ -372,10 +447,14 @@ private void copyOne(WireOut wire, boolean nested) throws InvalidMarshallableExc
break;
}
case TEXT:
wire.getValueOut().text(yt.text());
Object o = valueIn.readNumberOrText();
if (o instanceof Long)
wireValueOut.int64((long) o);
else
wireValueOut.object(o);
break;
case LITERAL:
wire.getValueOut().text(yt.text());
wireValueOut.text(yt.text());
break;
case ANCHOR:
break;
Expand All @@ -390,6 +469,20 @@ private void copyOne(WireOut wire, boolean nested) throws InvalidMarshallableExc
}
}

private boolean endOfDocument() {
if (isEmpty())
return true;
switch (yt.current()) {
case STREAM_END:
case STREAM_START:
case DOCUMENT_END:
case NONE:
return true;
default:
return false;
}
}

private void copyMappingKey(WireOut wire, boolean nested) throws InvalidMarshallableException {
yt.next();
if (yt.current() == YamlToken.MAPPING_KEY)
Expand Down Expand Up @@ -466,7 +559,10 @@ public <K> K readEvent(@NotNull Class<K> expectedClass) throws InvalidMarshallab
return readEvent(expectedClass);
}

return valueIn.object(expectedClass);
K object = valueIn.object(expectedClass);
if (object instanceof StringBuilder)
return (K) object.toString();
return object;
case NONE:
return null;
}
Expand Down Expand Up @@ -824,6 +920,7 @@ public void resetState() {
@Override
public String text() {
@Nullable CharSequence cs = textTo0(acquireStringBuilder());
yt.next();
return cs == null ? null : WireInternal.INTERNER.intern(cs);
}

Expand All @@ -832,6 +929,7 @@ public String text() {
public StringBuilder textTo(@NotNull StringBuilder sb) {
sb.setLength(0);
@Nullable CharSequence cs = textTo0(sb);
yt.next();
if (cs == null)
return null;
if (cs != sb) {
Expand Down Expand Up @@ -898,11 +996,11 @@ StringBuilder textTo0(@NotNull StringBuilder a) {
a.append(yt.text());
if (yt.current() == YamlToken.TEXT)
unescape(a, yt.blockQuote());
yt.next();

} else if (yt.current() == YamlToken.TAG) {
if (yt.isText(NULL_TAG)) {
yt.next();
yt.next();

return null;
}

Expand Down Expand Up @@ -1391,7 +1489,7 @@ public boolean hasNextSequenceItem() {
// Perhaps should be negative selection instead of positive
case SEQUENCE_START:
case SEQUENCE_ENTRY:
// Allows scalar value to be converted into singleton array
// Allows scalar value to be converted into singleton array
case TEXT:
return true;
}
Expand Down Expand Up @@ -1809,6 +1907,9 @@ Object objectWithInferredType0(Object using, @NotNull SerializationStrategy stra
case TEXT:
case LITERAL:
Object o = valueIn.readNumberOrText();
yt.next();
if (o instanceof StringBuilder)
o = o.toString();
return ObjectUtils.convertTo(type, o);

case ANCHOR:
Expand Down Expand Up @@ -1850,55 +1951,10 @@ Object objectWithInferredType0(Object using, @NotNull SerializationStrategy stra
@Nullable
protected Object readNumberOrText() {
char bq = yt.blockQuote();
@Nullable String s = text();
@Nullable StringBuilder s = textTo0(acquireStringBuilder());
if (yt.current() == YamlToken.LITERAL)
return s;
if (s == null
|| bq != 0
|| s.length() < 1
|| s.length() > 40
|| "0123456789.+-".indexOf(s.charAt(0)) < 0)
return s;

String ss = s;
if (s.indexOf('_') >= 0)
ss = ss.replace("_", "");

// YAML octal notation
if (s.startsWith("0o"))
ss = "0" + s.substring(2);

try {
return Long.decode(ss);
} catch (NumberFormatException fallback) {
// fallback
}
try {
return Double.parseDouble(ss);
} catch (NumberFormatException fallback) {
// fallback
}
try {
if (s.length() == 7 && s.charAt(1) == ':')
return LocalTime.parse('0' + s);
if (s.length() == 8 && s.charAt(2) == ':')
return LocalTime.parse(s);
} catch (DateTimeParseException fallback) {
// fallback
}
try {
if (s.length() == 10)
return LocalDate.parse(s);
} catch (DateTimeParseException fallback) {
// fallback
}
try {
if (s.length() >= 22)
return ZonedDateTime.parse(s);
} catch (DateTimeParseException fallback) {
// fallback
}
return s;
return readNumberOrTextFrom(bq, s);
}

@NotNull
Expand Down Expand Up @@ -1961,6 +2017,7 @@ private Object decodeBinary(Class type) throws InvalidMarshallableException {
public String toString() {
return YamlWire.this.toString();
}

}

@Override
Expand Down
7 changes: 3 additions & 4 deletions src/test/java/net/openhft/chronicle/wire/BinaryWire2Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,18 +388,17 @@ public void testSequenceContext() {
"48 00 00 00 # msg-length\n" +
"c8 65 6e 74 72 79 53 65 74 # entrySet:\n" +
"82 3a 00 00 00 # sequence\n" +
"82 18 00 00 00 # BinaryWire2Test$$Lambda$XXX\n" +
"82 18 00 00 00 # Marshallable\n" +
"c3 6b 65 79 # key:\n" +
"e5 6b 65 79 2d 31 # key-1\n" +
"c5 76 61 6c 75 65 # value:\n" +
"e7 76 61 6c 75 65 2d 31 # value-1\n" +
"82 18 00 00 00 # BinaryWire2Test$$Lambda$XXX\n" +
"82 18 00 00 00 # Marshallable\n" +
"c3 6b 65 79 # key:\n" +
"e5 6b 65 79 2d 32 # key-2\n" +
"c5 76 61 6c 75 65 # value:\n" +
"e7 76 61 6c 75 65 2d 32 # value-2\n",
wire.bytes().toHexString()
.replaceAll("BinaryWire2Test\\$\\$Lambda.*", "BinaryWire2Test\\$\\$Lambda\\$XXX"));
wire.bytes().toHexString());

@NotNull Wire twire = WireType.TEXT.apply(Bytes.elasticByteBuffer());
writeMessageContext(twire);
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/net/openhft/chronicle/wire/KubernetesYamlTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package net.openhft.chronicle.wire;

import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.annotation.RequiredForClient;
import org.junit.Test;

import java.io.InputStream;
Expand All @@ -29,6 +30,7 @@
import static org.junit.Assert.assertEquals;

@SuppressWarnings("rawtypes")
@RequiredForClient
public class KubernetesYamlTest extends WireTestCommon {
static String DIR = "/yaml/k8s/";

Expand Down
4 changes: 2 additions & 2 deletions src/test/java/net/openhft/chronicle/wire/WireSerializedLambdaTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public void testBinaryWire() {

assertEquals("c3 6f 6e 65 # one:\n" +
"b6 10 53 65 72 69 61 6c 69 7a 65 64 4c 61 6d 62 # SerializedLambda\n" +
"64 61 82 21 01 00 00 # WireSerializedLambda$$Lambda$\n" +
"64 61 82 21 01 00 00 # Marshallable\n" +
"c2 63 63 # cc:\n" +
"bc 33 6e 65 74 2e 6f 70 65 6e 68 66 74 2e 63 68 # net.openhft.chronicle.wire.WireSerializedLambdaTest\n" +
"72 6f 6e 69 63 6c 65 2e 77 69 72 65 2e 57 69 72\n" +
Expand Down Expand Up @@ -147,7 +147,7 @@ public void testBinaryWire() {
"c5 74 68 72 65 65 # three:\n" +
"b6 06 55 70 64 61 74 65 # Update\n" +
"e4 44 45 43 52 # DECR\n",
wire.bytes().toHexString().replaceAll("\\$\\$Lambda.*", "\\$\\$Lambda\\$"));
wire.bytes().toHexString());

@Nullable Function<String, String> function = wire.read().object(Function.class);
assertEquals("HELLO", function.apply("hello"));
Expand Down
Loading