From f11195303f281414fa10b74ca402c0a99bf77313 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 25 Jan 2016 09:34:46 -0800 Subject: [PATCH 1/2] remove check for valid utf-8 cursor --- .../com/google/gcloud/datastore/Cursor.java | 2 -- .../google/gcloud/datastore/DatastoreTest.java | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java index 667f3cc5e427..39963d1a543d 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java @@ -23,7 +23,6 @@ import com.google.api.services.datastore.DatastoreV1.Value; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; -import com.google.common.base.Preconditions; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; @@ -44,7 +43,6 @@ public final class Cursor extends Serializable { private final transient ByteString byteString; Cursor(ByteString byteString) { - Preconditions.checkArgument(byteString.isValidUtf8(), "content is not a valid UTF-8"); this.byteString = byteString; } diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 8cb88f9e7795..e9eed027e8e0 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -39,6 +39,7 @@ import com.google.gcloud.datastore.testing.LocalGcdHelper; import com.google.gcloud.spi.DatastoreRpc; import com.google.gcloud.spi.DatastoreRpcFactory; +import com.google.protobuf.ByteString; import org.easymock.EasyMock; import org.junit.AfterClass; @@ -496,7 +497,7 @@ public void testQueryPaginationWithLimit() throws DatastoreException { } query = query.toBuilder().startCursor(results.cursorAfter()).build(); } - assertEquals(totalCount, 5); + assertEquals(5, totalCount); EasyMock.verify(rpcFactoryMock, rpcMock); } @@ -524,7 +525,8 @@ private List buildResponsesForQueryPaginationWithLimit() { .setMoreResults(QueryResultBatch.MoreResultsType.MORE_RESULTS_AFTER_LIMIT) .clearEntityResult() .addAllEntityResult(queryResultBatchPb.getEntityResultList().subList(1, 2)) - .setEndCursor(queryResultBatchPb.getEntityResultList().get(1).getCursor()) + .setEndCursor( + ByteString.copyFrom(new byte[] {(byte) 0x80})) // test invalid UTF-8 string .build(); responses.add(RunQueryResponse.newBuilder().setBatch(queryResultBatchPb2).build()); QueryResultBatch queryResultBatchPb3 = QueryResultBatch.newBuilder() @@ -546,6 +548,17 @@ private List buildResponsesForQueryPaginationWithLimit() { return responses; } + @Test + public void testToUrlSafe() { + byte[][] invalidUtf8 = + new byte[][] {{(byte) 0xfe}, {(byte) 0xc1, (byte) 0xbf}, {(byte) 0xc0}, {(byte) 0x80}}; + for (byte[] bytes : invalidUtf8) { + assertFalse(ByteString.copyFrom(bytes).isValidUtf8()); + Cursor cursor = new Cursor(ByteString.copyFrom(bytes)); + assertEquals(cursor, Cursor.fromUrlSafe(cursor.toUrlSafe())); + } + } + @Test public void testAllocateId() { KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1); From 03ca1bfbf214900f5c0dbd06fb982fe396425426 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 25 Jan 2016 12:44:49 -0800 Subject: [PATCH 2/2] Simplify to/from url safe and fix access issue in workaround --- .../com/google/gcloud/datastore/Cursor.java | 21 ++++--------------- .../gcloud/datastore/StructuredQuery.java | 2 +- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java index 39963d1a543d..5e577f7feb6c 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java @@ -17,20 +17,14 @@ package com.google.gcloud.datastore; import static com.google.common.base.Preconditions.checkNotNull; -import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.datastore.DatastoreV1; import com.google.api.services.datastore.DatastoreV1.Value; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.io.BaseEncoding; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.TextFormat; -import com.google.protobuf.TextFormat.ParseException; - -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.net.URLEncoder; /** * A Google Cloud Datastore cursor. @@ -74,11 +68,7 @@ ByteString byteString() { * Returns the cursor in an encoded form that can be used as part of a URL. */ public String toUrlSafe() { - try { - return URLEncoder.encode(TextFormat.printToString(toPb()), UTF_8.name()); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Unexpected encoding exception", e); - } + return BaseEncoding.base64Url().encode(byteString.toByteArray()); } /** @@ -86,11 +76,8 @@ public String toUrlSafe() { */ public static Cursor fromUrlSafe(String urlSafe) { try { - String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); - DatastoreV1.Value.Builder builder = DatastoreV1.Value.newBuilder(); - TextFormat.merge(utf8Str, builder); - return fromPb(builder.build()); - } catch (UnsupportedEncodingException | ParseException e) { + return Cursor.copyFrom(BaseEncoding.base64Url().decode(urlSafe)); + } catch (IllegalArgumentException e) { throw new IllegalStateException("Unexpected decoding exception", e); } } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java index bbb73df4a79c..b8d9dfe87902 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java @@ -782,7 +782,7 @@ public StructuredQuery build() { } } - static final class Builder extends BaseBuilder> { + public static final class Builder extends BaseBuilder> { Builder(ResultType resultType) { super(resultType);