From 796700cdb6d9d1193749ed258c450f56bf90be3e Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Thu, 8 Apr 2021 15:05:40 -0700 Subject: [PATCH] Revert "Revert "Add 'Z' to the string format in Timestamp.ToString() to indicate UTC time. (#871)" (#873)" This reverts commit 309f4107d55c3e9ebb48842328750f379a07f7da. --- azure-pipelines.yml | 31 ++++++++++++------- .../UdfTests/UdfSimpleTypesTests.cs | 8 ++--- .../Sql/TimestampTests.cs | 13 ++++++-- .../Microsoft.Spark/Sql/Types/Timestamp.cs | 2 +- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 683d4deb8..6fc28178b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,20 +16,27 @@ variables: backwardCompatibleRelease: '1.0.0' forwardCompatibleRelease: '1.0.0' - backwardCompatibleTestOptions_Windows_2_3: "" - forwardCompatibleTestOptions_Windows_2_3: "" - backwardCompatibleTestOptions_Linux_2_3: "" - forwardCompatibleTestOptions_Linux_2_3: "" - - backwardCompatibleTestOptions_Windows_2_4: "" - forwardCompatibleTestOptions_Windows_2_4: "" - backwardCompatibleTestOptions_Linux_2_4: "" + # Filter UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType and UdfSimpleTypesTests.TestUdfWithTimestampType + # backward and forward compatibility tests due to bug with Timestamp.ToString(). This is not a breaking change. + # Please see https://github.com/dotnet/spark/pull/871 + backwardCompatibleTestOptions_Windows_2_3: "--filter \ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)" + forwardCompatibleTestOptions_Windows_2_3: $(backwardCompatibleTestOptions_Windows_2_3) + backwardCompatibleTestOptions_Linux_2_3: $(backwardCompatibleTestOptions_Windows_2_3) + forwardCompatibleTestOptions_Linux_2_3: $(backwardCompatibleTestOptions_Windows_2_3) + + backwardCompatibleTestOptions_Windows_2_4: $(backwardCompatibleTestOptions_Windows_2_3) + forwardCompatibleTestOptions_Windows_2_4: $(backwardCompatibleTestOptions_Windows_2_3) + backwardCompatibleTestOptions_Linux_2_4: $(backwardCompatibleTestOptions_Windows_2_3) # Filter HyperspaceTests not due to functionality changes, but to incompatible tests running on Linux. # Please see https://github.com/dotnet/spark/pull/737 for the fix. forwardCompatibleTestOptions_Linux_2_4: "--filter \ (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestExplainAPI)&\ (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestIndexCreateAndDelete)&\ - (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures)" + (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)" # Filter DataFrameTests.TestDataFrameGroupedMapUdf and DataFrameTests.TestGroupedMapUdf backwardCompatible # tests due to https://github.com/dotnet/spark/pull/711 @@ -38,8 +45,10 @@ variables: backwardCompatibleTestOptions_Windows_3_0: "--filter \ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDuplicateTimestamps)" - forwardCompatibleTestOptions_Windows_3_0: "" + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDuplicateTimestamps)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)" + forwardCompatibleTestOptions_Windows_3_0: $(backwardCompatibleTestOptions_Windows_2_3) backwardCompatibleTestOptions_Linux_3_0: $(backwardCompatibleTestOptions_Windows_3_0) forwardCompatibleTestOptions_Linux_3_0: $(forwardCompatibleTestOptions_Linux_2_4) diff --git a/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs b/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs index 79f7b8aaa..c73881437 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs @@ -118,8 +118,8 @@ public void TestUdfWithTimestampType() var expected = new string[] { - "2020-01-01 00:00:00.000000", - "2020-01-02 15:30:30.123456" + "2020-01-01 00:00:00.000000Z", + "2020-01-02 15:30:30.123456Z" }; string[] rowsToArray = rows.Select(x => x[0].ToString()).ToArray(); Assert.Equal(expected, rowsToArray); @@ -193,8 +193,8 @@ public void TestUdfWithReturnAsTimestampType() var expected = new string[] { - "2020-01-04 15:30:30.123456", - "2050-01-04 15:30:30.123456" + "2020-01-04 15:30:30.123456Z", + "2050-01-04 15:30:30.123456Z" }; for (int i = 0; i < rows.Length; ++i) { diff --git a/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs b/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs index dfbef74d3..14a766731 100644 --- a/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs +++ b/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs @@ -29,7 +29,7 @@ public void TimestampTest() Assert.Equal(DateTimeKind.Utc, timestamp.ToDateTime().Kind); // Validate ToString(). - Assert.Equal("2020-01-01 08:30:30.000123", timestamp.ToString()); + Assert.Equal("2020-01-01 08:30:30.000123Z", timestamp.ToString()); // Validate ToDateTime(). Assert.Equal(testDate, timestamp.ToDateTime()); @@ -57,7 +57,7 @@ public void TimestampTest() Assert.Equal(DateTimeKind.Utc, timestamp.ToDateTime().Kind); // Validate ToString(). - Assert.Equal("2020-01-02 15:30:30.123456", timestamp.ToString()); + Assert.Equal("2020-01-02 15:30:30.123456Z", timestamp.ToString()); // Validate ToDateTime(). Assert.Equal( @@ -71,5 +71,14 @@ public void TimestampTest() () => new Timestamp(2020, 1, 2, 15, 30, 30, 1234567)); } } + + [Fact] + public void TestTimestampToString() + { + var dateTimeObj = new DateTime(2021, 01, 01); + Assert.Equal( + new Timestamp(DateTime.Parse(new Timestamp(dateTimeObj).ToString())), + new Timestamp(dateTimeObj)); + } } } diff --git a/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs b/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs index d07d05172..992944c35 100644 --- a/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs +++ b/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs @@ -94,7 +94,7 @@ public Timestamp( /// /// Readable string representation for this type. /// - public override string ToString() => _dateTime.ToString("yyyy-MM-dd HH:mm:ss.ffffff"); + public override string ToString() => _dateTime.ToString("yyyy-MM-dd HH:mm:ss.ffffffZ"); /// /// Checks if the given object is same as the current object.