Skip to content
This repository has been archived by the owner on Jan 23, 2023. It is now read-only.
/ corefx Public archive

SqlClient Manual Test fixes and documentation update #34546

Merged
merged 10 commits into from
Feb 13, 2019
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,33 @@ public static bool IsDatabasePresent(string name)

public static bool IsUsingManagedSNI() => (bool)(s_useManagedSNI?.GetValue(null) ?? false);

public static bool IsUsingNativeSNI() => !IsUsingManagedSNI();

public static bool IsUTF8Supported()
{
bool retval = false;
if (AreConnStringsSetup())
{
using (SqlConnection connection = new SqlConnection(DataTestUtility.TcpConnStr))
using (SqlCommand command = new SqlCommand())
{
command.Connection = connection;
command.CommandText = "SELECT CONNECTIONPROPERTY('SUPPORT_UTF8')";
connection.Open();

using (SqlDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
// CONNECTIONPROPERTY('SUPPORT_UTF8') returns NULL in SQLServer versions that don't support UTF-8.
retval = !reader.IsDBNull(0);
}
}
}
}
return retval;
}

// the name length will be no more then (16 + prefix.Length + escapeLeft.Length + escapeRight.Length)
// some providers does not support names (Oracle supports up to 30)
public static string GetUniqueName(string prefix, string escapeLeft, string escapeRight)
Expand Down
41 changes: 38 additions & 3 deletions src/System.Data.SqlClient/tests/ManualTests/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,41 @@
# SqlClient Manual Tests

These tests require dedicated test servers, so they're designed to be run manually using a custom set of connection strings. These connection strings should be added as environment variables: TEST\_NP\_CONN\_STR & TEST\_TCP\_CONN\_STR. TEST\_NP\_CONN\_STR is a named pipes connection string to the test server, and TEST\_TCP\_CONN\_STR is a TCP connection string. Each protocol can be specified in the Server name parameter. These tests also assume the sample database "Northwind" exists in the target server. This sample database can be found [here](https://msdn.microsoft.com/en-us/library/mt710790.aspx).
These tests require dedicated test servers, so they're designed to be run manually using a custom set of connection strings.

Instructions for running tests: [Unix](https://github.com/dotnet/corefx/blob/master/Documentation/building/cross-platform-testing.md) and [Windows](https://github.com/dotnet/corefx/blob/master/Documentation/building/windows-instructions.md).
Documentation for connection string parameters: [SqlConnection.ConnectionString](https://msdn.microsoft.com/en-us/library/system.data.sqlclient.sqlconnection.connectionstring.aspx).
## Prerequisites

- CoreFX building. you need to be able to do a successful build and run the standard tests, [Unix](https://github.com/dotnet/corefx/blob/master/Documentation/building/cross-platform-testing.md) or [Windows](https://github.com/dotnet/corefx/blob/master/Documentation/building/windows-instructions.md) Use build.cmd for windows and build.sh for Linux to build CoreFX.

**N.B.** if you want to run the EFCore tests later you will need to build -allconfigurations to generate the NuGet packages, build -allconfigurations works only on windows.

- an [MS SQL Server](https://www.microsoft.com/en-us/sql-server/sql-server-editions-express) (any edition) 2012 or later that you can connect to with tcp and named pipes,

**N.B**. if you want to run the EFCore tests it should be a dedicated instance because they create a lot of databases.

- The [Northwind Sample Database](https://msdn.microsoft.com/en-us/library/mt710790.aspx)

- The [UDT Test Database](https://github.com/dotnet/corefx/tree/master/src/System.Data.SqlClient/tests/ManualTests/createUdtTestDb_corefx.sql)

- TCP and Named Pipe [connection strings](https://msdn.microsoft.com/en-us/library/system.data.sqlclient.sqlconnection.connectionstring.aspx) to your instance with Northwind set as the initial catalog



## Running All Tests

1. run `build src\System.Data.SqlClient -allconfigurations` and make sure the builds all work

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1 & 4 can be combined into a single statement
dotnet msbuild .\src\System.Data.SqlClient\tests\ManualTests\System.Data.SqlClient.ManualTesting.Tests.csproj /t:RebuildAndTest
This would build and run only the Manual Tests as compared to running all tests with command build src\System.Data.SqlClient -debug /p:ForceRunTests=true

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you combine then you get test infrastructure failures:

  Executing in E:\Programming\csharp7\corefx\artifacts\bin\tests\System.Data.SqlClient.Tests\netcoreapp-Windows_NT-Debug-x64\
  ----- start 22:39:13.28 ===============  To repro directly: =====================================================
  pushd E:\Programming\csharp7\corefx\artifacts\bin\tests\System.Data.SqlClient.Tests\netcoreapp-Windows_NT-Debug-x64\
  popd
  ===========================================================================================================
  A JSON parsing exception occurred in [E:\Programming\csharp7\corefx\artifacts\bin\tests\System.Data.SqlClient.Tests\netcoreapp-Windows_NT-Debug-x64\xunit.console.runtimeconfig.json]: * Line 1, Column 2 Syntax error: Malformed token
  Invalid runtimeconfig.json [E:\Programming\csharp7\corefx\artifacts\bin\tests\System.Data.SqlClient.Tests\netcoreapp-Windows_NT-Debug-x64\xunit.console.runtimeconfig.json] [E:\Programming\csharp7\corefx\artifacts\bin\tests\System.Data.SqlClient.Tests\netcoreapp-Windows_NT-Debug-x64\xunit.console.runtimeconfig.dev.json]
  ----- end 22:39:13.31 ----- exit code -2147450733 ----------------------------------------------------------

Running the specific build you want to test one at a time works.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is my console output for the command dotnet msbuild .\src\System.Data.SqlClient\tests\ManualTests\System.Data.SqlClient.ManualTesting.Tests.csproj /t:RebuildAndTest

PS D:\dotNetCore\corefx> dotnet msbuild .\src\System.Data.SqlClient\tests\ManualTests\System.Data.SqlClient.ManualTesting.Tests.csproj /t:RebuildAndTest
Microsoft (R) Build Engine version 15.9.20+g88f5fadfbe for .NET Core
Copyright (C) Microsoft Corporation. All rights reserved.

  Address -> D:\dotNetCore\corefx\artifacts\bin\Address\netstandard-AnyOS-Debug\Address.dll
  Utf8String -> D:\dotNetCore\corefx\artifacts\bin\Utf8String\netstandard-AnyOS-Debug\Utf8String.dll
  Shapes -> D:\dotNetCore\corefx\artifacts\bin\Shapes\netstandard-AnyOS-Debug\Shapes.dll
  Circle -> D:\dotNetCore\corefx\artifacts\bin\Circle\netstandard-AnyOS-Debug\Circle.dll
  System.Data.SqlClient.ManualTesting.Tests -> D:\dotNetCore\corefx\artifacts\bin\System.Data.SqlClient.ManualTesting.Tests\netcoreapp-AnyOS-Debug\System.Data.SqlClient.ManualTesting.Tests.dll
  Using D:\dotNetCore\corefx\artifacts\bin\testhost\netcoreapp-Windows_NT-Debug-x64 as the test runtime folder.
  Executing in D:\dotNetCore\corefx\artifacts\bin\tests\System.Data.SqlClient.ManualTesting.Tests\netcoreapp-Windows_NT-Debug-x64\
  ----- start 15:13:58.80 ===============  To repro directly: =====================================================
  pushd D:\dotNetCore\corefx\artifacts\bin\tests\System.Data.SqlClient.ManualTesting.Tests\netcoreapp-Windows_NT-Debug-x64\
  D:\dotNetCore\corefx\artifacts\bin\testhost\netcoreapp-Windows_NT-Debug-x64\dotnet.exe xunit.console.dll System.Data.SqlClient.ManualTesting.Tests.dll -xml testResults.xml -nologo -notrait category=nonnetcoreapptests -notrait category=nonwindowstests -notrait category=failing -notrait category=Outerloop
  popd
  ===========================================================================================================
    Discovering: System.Data.SqlClient.ManualTesting.Tests (method display = ClassAndMethod, method display options = None)
    Discovered:  System.Data.SqlClient.ManualTesting.Tests (found 206 test cases)
    Starting:    System.Data.SqlClient.ManualTesting.Tests (parallel test collections = on, max threads = 8)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will get the same thing if I use msbuild in the directory or build with either release or debug. If I run an allconfigurations build with force run tests 4 of the error I showed above occur as it tries to run the tests for configurations that don't work.

So allconfigurations with tests doesn't work, a single config does. Since it's just supposed to get people started I wanted to break the build and test steps out individually.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So running the above command will only build and test on platform based on the RID picked up automatically, which is netcoreapp-Windows_NT-Debug-x64 in the case above. allconfigurations will not work, since this is not building all configurations. The thing is that build src\System.Data.SqlClient chooses the csproj in the folder src\System.Data.SqlClient which means it is running all the other tests in SqlClient along with the manual tests.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've tweaked it a bit. See if that's better. I also recently found you don't need to use msbuild to run a specific test, you can do it straight from build.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see two potential issues:

  • Using build src\System.Data.SqlClient -allconfigurations builds all the tests and SqlClient and not just manual tests.
  • Kindly confirm this, as per my understanding build might not on unix, as on windows, the build command picks up build.cmd implicitly, whereas on unix it will not pickup build.sh implicitly.
    The suggested command dotnet msbuild will work across all platforms and will build only manual tests as per the csproj path specified.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, it does. I was also crediting linux users with enough intelligence to realise that the shell script needed to be executable if they're trying to do development work. I've changed it anyway.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you, changes look better, you can get rid of the Step 1, since Step 4 is taking care of building the Manual Tests.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.


2. set the environment variables needed for the tests you want. At the minimum you need to set
`TEST_NP_CONN_STR` and `TEST_TCP_CONN_STR` to the connection strings.

3. Optionally you may also want to setup other environment variables to test specific optional features such as [TEST_LOCALDB_INSTALLED](https://github.com/dotnet/corefx/blob/8f7b490ca874ee2a9f11f0163412f7c95811298b/src/System.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs#L96) or [TEST_INTEGRATEDSECURITY_SETUP](https://github.com/dotnet/corefx/blob/8f7b490ca874ee2a9f11f0163412f7c95811298b/src/System.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs#L98). Other scenarios lke azure tests may need configuration so if you see those being skipped and you want to run them invesigate the skipped test code to identify how to configure it.

4. run `dotnet msbuild .\src\System.Data.SqlClient\tests\ManualTests\System.Data.SqlClient.ManualTesting.Tests.csproj /t:Rebuild` to build the debug version with all the assertions and run the tests.

5. If you need to re-run the test suite without having changed the build (e.g. if you've changed the exnvironment variables) you can use `dotnet msbuild .\src\System.Data.SqlClient\tests\ManualTests\System.Data.SqlClient.ManualTesting.Tests.csproj /t:Test`



## Running A Specific Test

Once you have all tests running you may need to debug a single failing test. To do this navigate into the manual tests project directory `cd src\System.Data.SqlClient\tests\ManualTests` then run dotnet msbuild and specify the name of the test you want to execute, like this:
`dotnet msbuild /t:RebuildAndTest /p:XunitMethodName=System.Data.SqlClient.ManualTesting.Tests.DDDataTypesTest.MaxTypesTest`
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@ public static class ConnectionPoolTest
private static readonly string _tcpConnStr = (new SqlConnectionStringBuilder(DataTestUtility.TcpConnStr) { MultipleActiveResultSets = false, Pooling = true }).ConnectionString;
private static readonly string _tcpMarsConnStr = (new SqlConnectionStringBuilder(DataTestUtility.TcpConnStr) { MultipleActiveResultSets = true, Pooling = true }).ConnectionString;

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)]: */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void ConnectionPool_NonMars()
{
RunDataTestForSingleConnString(_tcpConnStr);
}

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void ConnectionPool_Mars()
{
RunDataTestForSingleConnString(_tcpMarsConnStr);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,8 @@ public class PoolBlockPeriodTest
private const string InstanceName = "InstanceName";
private const int ConnectionTimeout = 15;
private const int CompareMargin = 2;
private static bool AreConnectionStringsSetup() => DataTestUtility.AreConnStringsSetup();

[ConditionalTheory(nameof(AreConnectionStringsSetup))]
[ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
[InlineData("Azure with Default Policy must Disable blocking (*.database.windows.net)", new object[] { AzureEndpointSample })]
[InlineData("Azure with Default Policy must Disable blocking (*.database.chinacloudapi.cn)", new object[] { AzureChinaEnpointSample })]
[InlineData("Azure with Default Policy must Disable blocking (*.database.usgovcloudapi.net)", new object[] { AzureUSGovernmentEndpointSample })]
Expand All @@ -46,7 +45,7 @@ public void TestAzureBlockingPeriod(string description, object[] Params)
PoolBlockingPeriodAzureTest(connString, policy);
}

[ConditionalTheory(nameof(AreConnectionStringsSetup))]
[ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
[InlineData("NonAzure with Default Policy must Enable blocking", new object[] { NonExistentServer })]
[InlineData("NonAzure with Auto Policy must Enable Blocking", new object[] { NonExistentServer, PoolBlockingPeriod.Auto })]
[InlineData("NonAzure with Always Policy must Enable Blocking", new object[] { NonExistentServer, PoolBlockingPeriod.AlwaysBlock })]
Expand All @@ -67,7 +66,7 @@ public void TestNonAzureBlockingPeriod(string description, object[] Params)
PoolBlockingPeriodNonAzureTest(connString, policy);
}

[ConditionalTheory(nameof(AreConnectionStringsSetup))]
[ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
[InlineData("Test policy with Auto (lowercase)", "auto")]
[InlineData("Test policy with Auto (PascalCase)", "Auto")]
[InlineData("Test policy with Always (lowercase)", "alwaysblock")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@ private static void RunAllTestsForSingleServer(string connectionString, bool usi
// These tests fail with named pipes, since they try to do DNS lookups on named pipe paths.
if (!usingNamePipes)
{
TimeoutDuringReadAsyncWithClosedReaderTest(connectionString);
//if (DataTestUtility.IsUsingNativeSNI()) /* [ActiveIssue(33930)] */
//{
// TimeoutDuringReadAsyncWithClosedReaderTest(connectionString);
//}
NonFatalTimeoutDuringRead(connectionString);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public static void NamedPipesMARSTest()
}

#if DEBUG
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void MARSAsyncTimeoutTest()
{
using (SqlConnection connection = new SqlConnection(_connStr))
Expand Down Expand Up @@ -73,7 +73,7 @@ public static void MARSAsyncTimeoutTest()
}
}

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void MARSSyncTimeoutTest()
{
using (SqlConnection connection = new SqlConnection(_connStr))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public static void CodeCoverageSqlClient()
Assert.False(((IList)opc).IsReadOnly, "FAILED: Expected collection to NOT be read only.");
Assert.False(((IList)opc).IsFixedSize, "FAILED: Expected collection to NOT be fixed size.");
Assert.False(((IList)opc).IsSynchronized, "FAILED: Expected collection to NOT be synchronized.");
DataTestUtility.AssertEqualsWithDescription("Object", ((IList)opc).SyncRoot.GetType().Name, "FAILED: Incorrect SyncRoot Name");
DataTestUtility.AssertEqualsWithDescription("List`1", ((IList)opc).SyncRoot.GetType().Name, "FAILED: Incorrect SyncRoot Name");

{
string failValue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace System.Data.SqlClient.ManualTesting.Tests
public static class SqlCredentialTest
{

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void CreateSqlConnectionWithCredential()
{
var user = "u" + Guid.NewGuid().ToString().Replace("-", "");
Expand Down Expand Up @@ -49,7 +49,7 @@ public static void CreateSqlConnectionWithCredential()
}
}

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void SqlConnectionChangePasswordPlaintext()
{
var user = "u" + Guid.NewGuid().ToString().Replace("-", "");
Expand Down Expand Up @@ -82,7 +82,7 @@ public static void SqlConnectionChangePasswordPlaintext()
}
}

[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void SqlConnectionChangePasswordSecureString()
{
var user = "u" + Guid.NewGuid().ToString().Replace("-", "");
Expand Down Expand Up @@ -122,7 +122,7 @@ public static void SqlConnectionChangePasswordSecureString()
}
}

[ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), /* [ActiveIssue(33930)] */ nameof(DataTestUtility.IsUsingNativeSNI))]
public static void OldCredentialsShouldFail()
{
String user = "u" + Guid.NewGuid().ToString().Replace("-", "");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ namespace System.Data.SqlClient.ManualTesting.Tests
{
public static class Utf8SupportTest
{
[ConditionalFact(typeof(DataTestUtility),nameof(DataTestUtility.AreConnStringsSetup))]
[ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsUTF8Supported))]
public static void CheckSupportUtf8ConnectionProperty()
{
using (SqlConnection connection = new SqlConnection(DataTestUtility.TcpConnStr))
Expand All @@ -18,15 +18,7 @@ public static void CheckSupportUtf8ConnectionProperty()
{
while (reader.Read())
{
// CONNECTIONPROPERTY('SUPPORT_UTF8') returns NULL in SQLServer versions that don't support UTF-8.
if (!reader.IsDBNull(0))
{
Assert.Equal(1, reader.GetInt32(0));
}
else
{
Console.WriteLine("CONNECTIONPROPERTY('SUPPORT_UTF8') is not supported on this SQLServer");
}
Assert.Equal(1, reader.GetInt32(0));
}
}
}
Expand Down
Loading