Skip to content

Commit f2392d8

Browse files
apoorvdeshmukhapoorvdeshmukh
committed
Port #3399 to release/5.1
1 parent 6af24bc commit f2392d8

File tree

5 files changed

+222
-8
lines changed

5 files changed

+222
-8
lines changed

‎src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ internal struct SNIErrorDetails
3939
// and surfacing objects to the user.
4040
internal sealed partial class TdsParser
4141
{
42+
private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
4243
private static int _objectTypeCount; // EventSource counter
4344
private readonly SqlClientLogger _logger = new SqlClientLogger();
4445

@@ -2767,7 +2768,7 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
27672768
// UTF8 collation
27682769
if (env._newCollation.IsUTF8)
27692770
{
2770-
_defaultEncoding = Encoding.UTF8;
2771+
_defaultEncoding = s_utf8EncodingWithoutBom;
27712772
}
27722773
else
27732774
{
@@ -4171,7 +4172,7 @@ internal bool TryProcessReturnValue(int length, TdsParserStateObject stateObj, o
41714172
// UTF8 collation
41724173
if (rec.collation.IsUTF8)
41734174
{
4174-
rec.encoding = Encoding.UTF8;
4175+
rec.encoding = s_utf8EncodingWithoutBom;
41754176
}
41764177
else
41774178
{
@@ -4955,7 +4956,7 @@ private bool TryProcessTypeInfo(TdsParserStateObject stateObj, SqlMetaDataPriv c
49554956
// UTF8 collation
49564957
if (col.collation.IsUTF8)
49574958
{
4958-
col.encoding = Encoding.UTF8;
4959+
col.encoding = s_utf8EncodingWithoutBom;
49594960
}
49604961
else
49614962
{
@@ -10681,7 +10682,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
1068110682
// Replace encoding if it is UTF8
1068210683
if (metadata.collation.IsUTF8)
1068310684
{
10684-
_defaultEncoding = Encoding.UTF8;
10685+
_defaultEncoding = s_utf8EncodingWithoutBom;
1068510686
}
1068610687

1068710688
_defaultCollation = metadata.collation;

‎src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ namespace Microsoft.Data.SqlClient
3232
// and surfacing objects to the user.
3333
sealed internal class TdsParser
3434
{
35+
private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
3536
private static int _objectTypeCount; // EventSource Counter
3637
private readonly SqlClientLogger _logger = new SqlClientLogger();
3738

@@ -3221,7 +3222,7 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
32213222
// UTF8 collation
32223223
if (env._newCollation.IsUTF8)
32233224
{
3224-
_defaultEncoding = Encoding.UTF8;
3225+
_defaultEncoding = s_utf8EncodingWithoutBom;
32253226
}
32263227
else
32273228
{
@@ -4739,7 +4740,7 @@ internal bool TryProcessReturnValue(int length,
47394740

47404741
if (rec.collation.IsUTF8)
47414742
{ // UTF8 collation
4742-
rec.encoding = Encoding.UTF8;
4743+
rec.encoding = s_utf8EncodingWithoutBom;
47434744
}
47444745
else
47454746
{
@@ -5636,7 +5637,7 @@ private bool TryProcessTypeInfo(TdsParserStateObject stateObj, SqlMetaDataPriv c
56365637

56375638
if (col.collation.IsUTF8)
56385639
{ // UTF8 collation
5639-
col.encoding = Encoding.UTF8;
5640+
col.encoding = s_utf8EncodingWithoutBom;
56405641
}
56415642
else
56425643
{
@@ -11670,7 +11671,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
1167011671
// Replace encoding if it is UTF8
1167111672
if (metadata.collation.IsUTF8)
1167211673
{
11673-
_defaultEncoding = Encoding.UTF8;
11674+
_defaultEncoding = s_utf8EncodingWithoutBom;
1167411675
}
1167511676

1167611677
_defaultCollation = metadata.collation;

‎src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,17 @@ public static bool IsSupportingDistributedTransactions()
470470
#endif
471471
}
472472

473+
public static void CreateTable(SqlConnection sqlConnection, string tableName, string createBody)
474+
{
475+
DropTable(sqlConnection, tableName);
476+
string tableCreate = "CREATE TABLE " + tableName + createBody;
477+
using (SqlCommand command = sqlConnection.CreateCommand())
478+
{
479+
command.CommandText = tableCreate;
480+
command.ExecuteNonQuery();
481+
}
482+
}
483+
473484
public static void DropTable(SqlConnection sqlConnection, string tableName)
474485
{
475486
ResurrectConnection(sqlConnection);

‎src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@
279279
<Compile Include="SQL\Common\SystemDataInternals\DataReaderHelper.cs" />
280280
<Compile Include="SQL\Common\SystemDataInternals\TdsParserHelper.cs" />
281281
<Compile Include="SQL\Common\SystemDataInternals\TdsParserStateObjectHelper.cs" />
282+
<Compile Include="SQL\SqlBulkCopyTest\TestBulkCopyWithUTF8.cs" />
282283
<Compile Include="SQL\SqlCommand\SqlCommandStoredProcTest.cs" />
283284
<Compile Include="TracingTests\TestTdsServer.cs" />
284285
<Compile Include="XUnitAssemblyAttributes.cs" />

‎src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/TestBulkCopyWithUTF8.cs

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
using System;
2+
using System.Data;
3+
using System.Threading.Tasks;
4+
using Microsoft.Extensions.Options;
5+
using Xunit;
6+
7+
namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.SqlBulkCopyTest
8+
{
9+
/// <summary>
10+
/// Validates SqlBulkCopy functionality when working with UTF-8 encoded data.
11+
/// Ensures that data copied from a UTF-8 source table to a destination table retains its encoding and content integrity.
12+
/// </summary>
13+
public sealed class TestBulkCopyWithUtf8 : IDisposable
14+
{
15+
private static bool IsAzureServer() => !DataTestUtility.IsNotAzureServer();
16+
private static bool IsNotAzureSynapse => DataTestUtility.IsNotAzureSynapse();
17+
private static bool AreConnectionStringsSetup() => DataTestUtility.AreConnStringsSetup();
18+
private static string s_sourceTable = DataTestUtility.GetUniqueName("SourceTableForUTF8Data");
19+
private static string s_destinationTable = DataTestUtility.GetUniqueName("DestinationTableForUTF8Data");
20+
private static string s_testValue = "test";
21+
private static byte[] s_testValueInUtf8Bytes = new byte[] { 0x74, 0x65, 0x73, 0x74 };
22+
private static readonly string s_insertQuery = $"INSERT INTO {s_sourceTable} VALUES('{s_testValue}')";
23+
24+
/// <summary>
25+
/// Constructor: Initializes and populates source and destination tables required for the tests.
26+
/// </summary>
27+
public TestBulkCopyWithUtf8()
28+
{
29+
using SqlConnection sourceConnection = new SqlConnection(GetConnectionString(true));
30+
sourceConnection.Open();
31+
SetupTables(sourceConnection, s_sourceTable, s_destinationTable, s_insertQuery);
32+
}
33+
34+
/// <summary>
35+
/// Cleanup method to drop tables after test completion.
36+
/// </summary>
37+
public void Dispose()
38+
{
39+
using SqlConnection connection = new SqlConnection(GetConnectionString(true));
40+
connection.Open();
41+
DataTestUtility.DropTable(connection, s_sourceTable);
42+
DataTestUtility.DropTable(connection, s_destinationTable);
43+
connection.Close();
44+
}
45+
46+
/// <summary>
47+
/// Builds a connection string with or without Multiple Active Result Sets (MARS) property.
48+
/// </summary>
49+
private string GetConnectionString(bool enableMars)
50+
{
51+
return new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString)
52+
{
53+
MultipleActiveResultSets = enableMars
54+
}.ConnectionString;
55+
}
56+
57+
/// <summary>
58+
/// Creates source and destination tables with a varchar(max) column with a collation setting
59+
/// that stores the data in UTF8 encoding and inserts the data in the source table.
60+
/// </summary>
61+
private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable, string insertQuery)
62+
{
63+
string columnDefinition = "(str_col varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8)";
64+
DataTestUtility.CreateTable(connection, sourceTable, columnDefinition);
65+
DataTestUtility.CreateTable(connection, destinationTable, columnDefinition);
66+
using SqlCommand insertCommand = connection.CreateCommand();
67+
insertCommand.CommandText = insertQuery;
68+
Helpers.TryExecute(insertCommand, insertQuery);
69+
}
70+
71+
/// <summary>
72+
/// Synchronous test case: Validates that data copied using SqlBulkCopy matches UTF-8 byte sequence for test value.
73+
/// Tested with MARS enabled and disabled, and with streaming enabled and disabled.
74+
/// </summary>
75+
[ConditionalTheory(typeof(DataTestUtility),
76+
nameof(DataTestUtility.AreConnStringsSetup),
77+
nameof(DataTestUtility.IsNotAzureServer),
78+
nameof(DataTestUtility.IsNotAzureSynapse))]
79+
[InlineData(true, true)]
80+
[InlineData(false, true)]
81+
[InlineData(true, false)]
82+
[InlineData(false, false)]
83+
public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableStreaming)
84+
{
85+
// Setup connections for source and destination tables
86+
string connectionString = GetConnectionString(isMarsEnabled);
87+
using SqlConnection sourceConnection = new SqlConnection(connectionString);
88+
sourceConnection.Open();
89+
using SqlConnection destinationConnection = new SqlConnection(connectionString);
90+
destinationConnection.Open();
91+
92+
// Read data from source table
93+
using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection);
94+
using SqlDataReader reader = sourceDataCommand.ExecuteReader(CommandBehavior.SequentialAccess);
95+
96+
// Verify that the destination table is empty before bulk copy
97+
using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection);
98+
Assert.Equal(0, Convert.ToInt16(countCommand.ExecuteScalar()));
99+
100+
// Initialize bulk copy configuration
101+
using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection)
102+
{
103+
EnableStreaming = enableStreaming,
104+
DestinationTableName = s_destinationTable
105+
};
106+
107+
try
108+
{
109+
// Perform bulk copy from source to destination table
110+
bulkCopy.WriteToServer(reader);
111+
}
112+
catch (Exception ex)
113+
{
114+
// If bulk copy fails, fail the test with the exception message
115+
Assert.Fail($"Bulk copy failed: {ex.Message}");
116+
}
117+
118+
// Verify that the 1 row from the source table has been copied into our destination table.
119+
Assert.Equal(1, Convert.ToInt16(countCommand.ExecuteScalar()));
120+
121+
// Read the data from destination table as varbinary to verify the UTF-8 byte sequence
122+
using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection);
123+
using SqlDataReader verifyReader = verifyCommand.ExecuteReader(CommandBehavior.SequentialAccess);
124+
125+
// Verify that we have data in the destination table
126+
Assert.True(verifyReader.Read(), "No data found in destination table after bulk copy.");
127+
128+
// Read the value of the column as SqlBinary.
129+
byte[] actualBytes = verifyReader.GetSqlBinary(0).Value;
130+
131+
// Verify that the byte array matches the expected UTF-8 byte sequence
132+
Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length);
133+
Assert.Equal(s_testValueInUtf8Bytes, actualBytes);
134+
}
135+
136+
/// <summary>
137+
/// Asynchronous version of the testcase BulkCopy_Utf8Data_ShouldMatchSource
138+
/// </summary>
139+
[ConditionalTheory(typeof(DataTestUtility),
140+
nameof(DataTestUtility.AreConnStringsSetup),
141+
nameof(DataTestUtility.IsNotAzureServer),
142+
nameof(DataTestUtility.IsNotAzureSynapse))]
143+
[InlineData(true, true)]
144+
[InlineData(false, true)]
145+
[InlineData(true, false)]
146+
[InlineData(false, false)]
147+
public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, bool enableStreaming)
148+
{
149+
// Setup connections for source and destination tables
150+
string connectionString = GetConnectionString(isMarsEnabled);
151+
using SqlConnection sourceConnection = new SqlConnection(connectionString);
152+
await sourceConnection.OpenAsync();
153+
using SqlConnection destinationConnection = new SqlConnection(connectionString);
154+
await destinationConnection.OpenAsync();
155+
156+
// Read data from source table
157+
using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection);
158+
using SqlDataReader reader = await sourceDataCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
159+
160+
// Verify that the destination table is empty before bulk copy
161+
using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection);
162+
Assert.Equal(0, Convert.ToInt16(await countCommand.ExecuteScalarAsync()));
163+
164+
// Initialize bulk copy configuration
165+
using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection)
166+
{
167+
EnableStreaming = enableStreaming,
168+
DestinationTableName = s_destinationTable
169+
};
170+
171+
try
172+
{
173+
// Perform bulk copy from source to destination table
174+
await bulkCopy.WriteToServerAsync(reader);
175+
}
176+
catch (Exception ex)
177+
{
178+
// If bulk copy fails, fail the test with the exception message
179+
Assert.Fail($"Bulk copy failed: {ex.Message}");
180+
}
181+
182+
// Verify that the 1 row from the source table has been copied into our destination table.
183+
Assert.Equal(1, Convert.ToInt16(await countCommand.ExecuteScalarAsync()));
184+
185+
// Read the data from destination table as varbinary to verify the UTF-8 byte sequence
186+
using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection);
187+
using SqlDataReader verifyReader = await verifyCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
188+
189+
// Verify that we have data in the destination table
190+
Assert.True(await verifyReader.ReadAsync(), "No data found in destination table after bulk copy.");
191+
192+
// Read the value of the column as SqlBinary.
193+
byte[] actualBytes = verifyReader.GetSqlBinary(0).Value;
194+
195+
// Verify that the byte array matches the expected UTF-8 byte sequence
196+
Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length);
197+
Assert.Equal(s_testValueInUtf8Bytes, actualBytes);
198+
}
199+
}
200+
}

0 commit comments

Comments
 (0)