Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Release 2.1] Tests | Fix randomly failing tests #942

Merged
merged 1 commit into from
Mar 2, 2021
Merged

[Release 2.1] Tests | Fix randomly failing tests #942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 61 additions & 52 deletions ...ft.Data.SqlClient/tests/FunctionalTests/AlwaysEncryptedTests/ExceptionsAlgorithmErrors.cs
View file
Open in desktop

Large diffs are not rendered by default.

10 changes: 8 additions & 2 deletions ...icrosoft.Data.SqlClient/tests/FunctionalTests/AlwaysEncryptedTests/ExceptionsCertStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,19 @@ public class ExceptionCertFixture : IDisposable

public ExceptionCertFixture()
{
certificate = Utility.CreateCertificate();
if(certificate == null)
{
certificate = Utility.CreateCertificate();
}
thumbprint = certificate.Thumbprint;
certificatePath = string.Format("CurrentUser/My/{0}", thumbprint);
cek = Utility.GenerateRandomBytes(32);
encryptedCek = certStoreProvider.EncryptColumnEncryptionKey(certificatePath, "RSA_OAEP", cek);
#if NET46
masterKeyCertificateNPK = Utility.CreateCertificateWithNoPrivateKey();
if(masterKeyCertificateNPK == null)
{
masterKeyCertificateNPK = Utility.CreateCertificateWithNoPrivateKey();
}
thumbprintNPK = masterKeyCertificateNPK.Thumbprint;
masterKeyPathNPK = "CurrentUser/My/" + thumbprintNPK;
#endif
Expand Down
7 changes: 5 additions & 2 deletions src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ConversionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class ConversionTests : IDisposable
private const decimal SmallMoneyMinValue = -214748.3648M;
private const int MaxLength = 10000;
private int NumberOfRows = DataTestUtility.EnclaveEnabled ? 10 : 100;
private readonly X509Certificate2 certificate;
private static X509Certificate2 certificate;
private ColumnMasterKey columnMasterKey;
private ColumnEncryptionKey columnEncryptionKey;
private SqlColumnEncryptionCertificateStoreProvider certStoreProvider = new SqlColumnEncryptionCertificateStoreProvider();
Expand All @@ -55,7 +55,10 @@ public ColumnMetaData(SqlDbType columnType, int columnSize, int precision, int s

public ConversionTests()
{
certificate = CertificateUtility.CreateCertificate();
if(certificate == null)
{
certificate = CertificateUtility.CreateCertificate();
}
columnMasterKey = new CspColumnMasterKey(DatabaseHelper.GenerateUniqueName("CMK"), certificate.Thumbprint, certStoreProvider, DataTestUtility.EnclaveEnabled);
databaseObjects.Add(columnMasterKey);

Expand Down
7 changes: 5 additions & 2 deletions ...crosoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public class SQLSetupStrategy : IDisposable
{
internal const string ColumnEncryptionAlgorithmName = @"AEAD_AES_256_CBC_HMAC_SHA256";

protected internal readonly X509Certificate2 certificate;
protected static X509Certificate2 certificate;
public string keyPath { get; internal set; }
public Table ApiTestTable { get; private set; }
public Table BulkCopyAEErrorMessageTestTable { get; private set; }
Expand Down Expand Up @@ -56,7 +56,10 @@ public class SQLSetupStrategy : IDisposable

public SQLSetupStrategy()
{
certificate = CertificateUtility.CreateCertificate();
if(certificate == null)
{
certificate = CertificateUtility.CreateCertificate();
}
}

protected SQLSetupStrategy(string customKeyPath) => keyPath = customKeyPath;
Expand Down
29 changes: 19 additions & 10 deletions src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/AADConnectionTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ public static void ActiveDirectoryManagedIdentityWithCredentialsMustFail()
Assert.Contains(expectedMessage, e.Message);
}

[ConditionalFact(nameof(IsAADConnStringsSetup))]
[ConditionalFact(nameof(IsAADConnStringsSetup), nameof(IsManagedIdentitySetup))]
public static void ActiveDirectoryManagedIdentityWithPasswordMustFail()
{
// connection fails with expected error message.
Expand All @@ -385,7 +385,7 @@ public static void ActiveDirectoryManagedIdentityWithPasswordMustFail()

[InlineData("2445343 2343253")]
[InlineData("2445343$#^@@%2343253")]
[ConditionalTheory(nameof(IsAADConnStringsSetup))]
[ConditionalTheory(nameof(IsAADConnStringsSetup), nameof(IsManagedIdentitySetup))]
public static void ActiveDirectoryManagedIdentityWithInvalidUserIdMustFail(string userId)
{
// connection fails with expected error message.
Expand Down Expand Up @@ -444,33 +444,42 @@ public static void ADInteractiveUsingSSPI()
ConnectAndDisconnect(connStr);
}

// Test passes locally everytime, but in pieplines fails randomly with uncertainity.
// e.g. Second AAD connection too slow (802ms)! (More than 30% of the first (576ms).)
[ActiveIssue(16058)]
[ConditionalFact(nameof(IsAADConnStringsSetup))]
public static void ConnectionSpeed()
{
var connString = DataTestUtility.AADPasswordConnectionString;

//Ensure server endpoints are warm
using (var connectionDrill = new SqlConnection(DataTestUtility.AADPasswordConnectionString))
using (var connectionDrill = new SqlConnection(connString))
{
connectionDrill.Open();
}

SqlConnection.ClearAllPools();
ActiveDirectoryAuthenticationProvider.ClearUserTokenCache();

Stopwatch firstConnectionTime = new Stopwatch();
Stopwatch secondConnectionTime = new Stopwatch();

using (var connectionDrill = new SqlConnection(DataTestUtility.AADPasswordConnectionString))
using (var connectionDrill = new SqlConnection(connString))
{
Stopwatch firstConnectionTime = new Stopwatch();
firstConnectionTime.Start();
connectionDrill.Open();
firstConnectionTime.Stop();
using (var connectionDrill2 = new SqlConnection(DataTestUtility.AADPasswordConnectionString))
using (var connectionDrill2 = new SqlConnection(connString))
{
Stopwatch secondConnectionTime = new Stopwatch();
secondConnectionTime.Start();
connectionDrill2.Open();
secondConnectionTime.Stop();
// Subsequent AAD connections within a short timeframe should use an auth token cached from the connection pool
// Second connection speed in tests was typically 10-15% of the first connection time. Using 30% since speeds may vary.
Assert.True(secondConnectionTime.ElapsedMilliseconds / firstConnectionTime.ElapsedMilliseconds < .30, $"Second AAD connection too slow ({secondConnectionTime.ElapsedMilliseconds}ms)! (More than 30% of the first ({firstConnectionTime.ElapsedMilliseconds}ms).)");
}
}

// Subsequent AAD connections within a short timeframe should use an auth token cached from the connection pool
// Second connection speed in tests was typically 10-15% of the first connection time. Using 30% since speeds may vary.
Assert.True(((double)secondConnectionTime.ElapsedMilliseconds / firstConnectionTime.ElapsedMilliseconds) < 0.30, $"Second AAD connection too slow ({secondConnectionTime.ElapsedMilliseconds}ms)! (More than 30% of the first ({firstConnectionTime.ElapsedMilliseconds}ms).)");
}

#region Managed Identity Authentication tests
Expand Down
19 changes: 16 additions & 3 deletions src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ExceptionTest/ExceptionTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,25 @@ public static void WarningsBeforeRowsTest()
private static bool CheckThatExceptionsAreDistinctButHaveSameData(SqlException e1, SqlException e2)
{
Assert.True(e1 != e2, "FAILED: verification of exception cloning in subsequent connection attempts");

Assert.False((e1 == null) || (e2 == null), "FAILED: One of exceptions is null, another is not");

bool equal = (e1.Message == e2.Message) && (e1.HelpLink == e2.HelpLink) && (e1.InnerException == e2.InnerException)
&& (e1.Source == e2.Source) && (e1.Data.Count == e2.Data.Count) && (e1.Errors == e2.Errors);
&& (e1.Source == e2.Source) && (e1.Data.Count == e2.Data.Count) && (e1.Errors.Count == e2.Errors.Count);

for (int i = 0; i < e1.Errors.Count; i++)
{
equal = e1.Errors[i].Number == e2.Errors[i].Number
&& e1.Errors[i].Message == e2.Errors[i].Message
&& e1.Errors[i].LineNumber == e2.Errors[i].LineNumber
&& e1.Errors[i].State == e2.Errors[i].State
&& e1.Errors[i].Class == e2.Errors[i].Class
&& e1.Errors[i].Server == e2.Errors[i].Server
&& e1.Errors[i].Procedure == e2.Errors[i].Procedure
&& e1.Errors[i].Source == e2.Errors[i].Source;
if (!equal)
break;
}

IDictionaryEnumerator enum1 = e1.Data.GetEnumerator();
IDictionaryEnumerator enum2 = e2.Data.GetEnumerator();
while (equal)
Expand All @@ -147,7 +161,6 @@ private static bool CheckThatExceptionsAreDistinctButHaveSameData(SqlException e
}

Assert.True(equal, string.Format("FAILED: exceptions do not contain the same data (besides call stack):\nFirst: {0}\nSecond: {1}\n", e1, e2));

return true;
}

Expand Down
79 changes: 47 additions & 32 deletions src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/UdtTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,48 +242,63 @@ public void NullTest()
p.UdtTypeName = "Utf8String";
p.Value = DBNull.Value;

using (SqlTransaction trans = conn.BeginTransaction())
bool rerun = false;
do
{
com.Transaction = trans;
using (SqlDataReader reader = com.ExecuteReader())
try
{

Utf8String[] expectedValues = {
new Utf8String("this"),
new Utf8String("is"),
new Utf8String("a"),
new Utf8String("test")
};

int currentValue = 0;
do
using (SqlTransaction trans = conn.BeginTransaction())
{
while (reader.Read())
com.Transaction = trans;
using (SqlDataReader reader = com.ExecuteReader())
{
DataTestUtility.AssertEqualsWithDescription(1, reader.FieldCount, "Unexpected FieldCount.");
if (currentValue < expectedValues.Length)
Utf8String[] expectedValues = {
new Utf8String("this"),
new Utf8String("is"),
new Utf8String("a"),
new Utf8String("test")
};

int currentValue = 0;
do
{
DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetValue(0), "Unexpected Value.");
DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetSqlValue(0), "Unexpected SQL Value.");
}
else
{
DataTestUtility.AssertEqualsWithDescription(DBNull.Value, reader.GetValue(0), "Unexpected Value.");

Utf8String sqlValue = (Utf8String)reader.GetSqlValue(0);
INullable iface = sqlValue as INullable;
Assert.True(iface != null, "Expected interface cast to return a non-null value.");
Assert.True(iface.IsNull, "Expected interface cast to have IsNull==true.");
while (reader.Read())
{
DataTestUtility.AssertEqualsWithDescription(1, reader.FieldCount, "Unexpected FieldCount.");
if (currentValue < expectedValues.Length)
{
DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetValue(0), "Unexpected Value.");
DataTestUtility.AssertEqualsWithDescription(expectedValues[currentValue], reader.GetSqlValue(0), "Unexpected SQL Value.");
}
else
{
DataTestUtility.AssertEqualsWithDescription(DBNull.Value, reader.GetValue(0), "Unexpected Value.");

Utf8String sqlValue = (Utf8String)reader.GetSqlValue(0);
INullable iface = sqlValue as INullable;
Assert.True(iface != null, "Expected interface cast to return a non-null value.");
Assert.True(iface.IsNull, "Expected interface cast to have IsNull==true.");
}

currentValue++;
Assert.True(currentValue <= (expectedValues.Length + 1), "Expected to only hit one extra result.");
}
}

currentValue++;
Assert.True(currentValue <= (expectedValues.Length + 1), "Expected to only hit one extra result.");
while (reader.NextResult());
DataTestUtility.AssertEqualsWithDescription(currentValue, (expectedValues.Length + 1), "Did not hit all expected values.");
rerun = false;
}
}
while (reader.NextResult());
DataTestUtility.AssertEqualsWithDescription(currentValue, (expectedValues.Length + 1), "Did not hit all expected values.");
}
catch (SqlException e)
{
if(e.Message.Contains("Rerun the transaction"))
rerun = true;
else
throw e;
}
}
while(rerun);
}
}
}
Expand Down