Skip to content

Commit 1ccd0fa

Browse files
apoorvdeshmukhapoorvdeshmukh
committed
Port #3399 to release/5.1
1 parent 6af24bc commit 1ccd0fa

File tree

5 files changed

+218
-8
lines changed

5 files changed

+218
-8
lines changed

‎src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ internal struct SNIErrorDetails
3939
// and surfacing objects to the user.
4040
internal sealed partial class TdsParser
4141
{
42+
private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
4243
private static int _objectTypeCount; // EventSource counter
4344
private readonly SqlClientLogger _logger = new SqlClientLogger();
4445

@@ -2767,7 +2768,7 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
27672768
// UTF8 collation
27682769
if (env._newCollation.IsUTF8)
27692770
{
2770-
_defaultEncoding = Encoding.UTF8;
2771+
_defaultEncoding = s_utf8EncodingWithoutBom;
27712772
}
27722773
else
27732774
{
@@ -4171,7 +4172,7 @@ internal bool TryProcessReturnValue(int length, TdsParserStateObject stateObj, o
41714172
// UTF8 collation
41724173
if (rec.collation.IsUTF8)
41734174
{
4174-
rec.encoding = Encoding.UTF8;
4175+
rec.encoding = s_utf8EncodingWithoutBom;
41754176
}
41764177
else
41774178
{
@@ -4955,7 +4956,7 @@ private bool TryProcessTypeInfo(TdsParserStateObject stateObj, SqlMetaDataPriv c
49554956
// UTF8 collation
49564957
if (col.collation.IsUTF8)
49574958
{
4958-
col.encoding = Encoding.UTF8;
4959+
col.encoding = s_utf8EncodingWithoutBom;
49594960
}
49604961
else
49614962
{
@@ -10681,7 +10682,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
1068110682
// Replace encoding if it is UTF8
1068210683
if (metadata.collation.IsUTF8)
1068310684
{
10684-
_defaultEncoding = Encoding.UTF8;
10685+
_defaultEncoding = s_utf8EncodingWithoutBom;
1068510686
}
1068610687

1068710688
_defaultCollation = metadata.collation;

‎src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ namespace Microsoft.Data.SqlClient
3232
// and surfacing objects to the user.
3333
sealed internal class TdsParser
3434
{
35+
private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
3536
private static int _objectTypeCount; // EventSource Counter
3637
private readonly SqlClientLogger _logger = new SqlClientLogger();
3738

@@ -3221,7 +3222,7 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
32213222
// UTF8 collation
32223223
if (env._newCollation.IsUTF8)
32233224
{
3224-
_defaultEncoding = Encoding.UTF8;
3225+
_defaultEncoding = s_utf8EncodingWithoutBom;
32253226
}
32263227
else
32273228
{
@@ -4739,7 +4740,7 @@ internal bool TryProcessReturnValue(int length,
47394740

47404741
if (rec.collation.IsUTF8)
47414742
{ // UTF8 collation
4742-
rec.encoding = Encoding.UTF8;
4743+
rec.encoding = s_utf8EncodingWithoutBom;
47434744
}
47444745
else
47454746
{
@@ -5636,7 +5637,7 @@ private bool TryProcessTypeInfo(TdsParserStateObject stateObj, SqlMetaDataPriv c
56365637

56375638
if (col.collation.IsUTF8)
56385639
{ // UTF8 collation
5639-
col.encoding = Encoding.UTF8;
5640+
col.encoding = s_utf8EncodingWithoutBom;
56405641
}
56415642
else
56425643
{
@@ -11670,7 +11671,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
1167011671
// Replace encoding if it is UTF8
1167111672
if (metadata.collation.IsUTF8)
1167211673
{
11673-
_defaultEncoding = Encoding.UTF8;
11674+
_defaultEncoding = s_utf8EncodingWithoutBom;
1167411675
}
1167511676

1167611677
_defaultCollation = metadata.collation;

‎src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,17 @@ public static bool IsSupportingDistributedTransactions()
470470
#endif
471471
}
472472

473+
public static void CreateTable(SqlConnection sqlConnection, string tableName, string createBody)
474+
{
475+
DropTable(sqlConnection, tableName);
476+
string tableCreate = "CREATE TABLE " + tableName + createBody;
477+
using (SqlCommand command = sqlConnection.CreateCommand())
478+
{
479+
command.CommandText = tableCreate;
480+
command.ExecuteNonQuery();
481+
}
482+
}
483+
473484
public static void DropTable(SqlConnection sqlConnection, string tableName)
474485
{
475486
ResurrectConnection(sqlConnection);

‎src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@
279279
<Compile Include="SQL\Common\SystemDataInternals\DataReaderHelper.cs" />
280280
<Compile Include="SQL\Common\SystemDataInternals\TdsParserHelper.cs" />
281281
<Compile Include="SQL\Common\SystemDataInternals\TdsParserStateObjectHelper.cs" />
282+
<Compile Include="SQL\SqlBulkCopyTest\TestBulkCopyWithUTF8.cs" />
282283
<Compile Include="SQL\SqlCommand\SqlCommandStoredProcTest.cs" />
283284
<Compile Include="TracingTests\TestTdsServer.cs" />
284285
<Compile Include="XUnitAssemblyAttributes.cs" />

‎src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/TestBulkCopyWithUTF8.cs

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
using System;
2+
using System.Data;
3+
using System.Threading.Tasks;
4+
using Xunit;
5+
6+
namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.SqlBulkCopyTest
7+
{
8+
/// <summary>
9+
/// Validates SqlBulkCopy functionality when working with UTF-8 encoded data.
10+
/// Ensures that data copied from a UTF-8 source table to a destination table retains its encoding and content integrity.
11+
/// </summary>
12+
public sealed class TestBulkCopyWithUtf8 : IDisposable
13+
{
14+
private static string s_sourceTable = DataTestUtility.GetUniqueName("SourceTableForUTF8Data");
15+
private static string s_destinationTable = DataTestUtility.GetUniqueName("DestinationTableForUTF8Data");
16+
private static string s_testValue = "test";
17+
private static byte[] s_testValueInUtf8Bytes = new byte[] { 0x74, 0x65, 0x73, 0x74 };
18+
private static readonly string s_insertQuery = $"INSERT INTO {s_sourceTable} VALUES('{s_testValue}')";
19+
20+
/// <summary>
21+
/// Constructor: Initializes and populates source and destination tables required for the tests.
22+
/// </summary>
23+
public TestBulkCopyWithUtf8()
24+
{
25+
using SqlConnection sourceConnection = new SqlConnection(GetConnectionString(true));
26+
sourceConnection.Open();
27+
SetupTables(sourceConnection, s_sourceTable, s_destinationTable, s_insertQuery);
28+
}
29+
30+
/// <summary>
31+
/// Cleanup method to drop tables after test completion.
32+
/// </summary>
33+
public void Dispose()
34+
{
35+
using SqlConnection connection = new SqlConnection(GetConnectionString(true));
36+
connection.Open();
37+
DataTestUtility.DropTable(connection, s_sourceTable);
38+
DataTestUtility.DropTable(connection, s_destinationTable);
39+
connection.Close();
40+
}
41+
42+
/// <summary>
43+
/// Builds a connection string with or without Multiple Active Result Sets (MARS) property.
44+
/// </summary>
45+
private string GetConnectionString(bool enableMars)
46+
{
47+
return new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString)
48+
{
49+
MultipleActiveResultSets = enableMars
50+
}.ConnectionString;
51+
}
52+
53+
/// <summary>
54+
/// Creates source and destination tables with a varchar(max) column with a collation setting
55+
/// that stores the data in UTF8 encoding and inserts the data in the source table.
56+
/// </summary>
57+
private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable, string insertQuery)
58+
{
59+
string columnDefinition = "(str_col varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8)";
60+
DataTestUtility.CreateTable(connection, sourceTable, columnDefinition);
61+
DataTestUtility.CreateTable(connection, destinationTable, columnDefinition);
62+
using SqlCommand insertCommand = connection.CreateCommand();
63+
insertCommand.CommandText = insertQuery;
64+
Helpers.TryExecute(insertCommand, insertQuery);
65+
}
66+
67+
/// <summary>
68+
/// Synchronous test case: Validates that data copied using SqlBulkCopy matches UTF-8 byte sequence for test value.
69+
/// Tested with MARS enabled and disabled, and with streaming enabled and disabled.
70+
/// </summary>
71+
[ConditionalTheory(typeof(DataTestUtility),
72+
nameof(DataTestUtility.AreConnStringsSetup),
73+
nameof(DataTestUtility.IsNotAzureServer),
74+
nameof(DataTestUtility.IsNotAzureSynapse))]
75+
[InlineData(true, true)]
76+
[InlineData(false, true)]
77+
[InlineData(true, false)]
78+
[InlineData(false, false)]
79+
public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableStreaming)
80+
{
81+
// Setup connections for source and destination tables
82+
string connectionString = GetConnectionString(isMarsEnabled);
83+
using SqlConnection sourceConnection = new SqlConnection(connectionString);
84+
sourceConnection.Open();
85+
using SqlConnection destinationConnection = new SqlConnection(connectionString);
86+
destinationConnection.Open();
87+
88+
// Read data from source table
89+
using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection);
90+
using SqlDataReader reader = sourceDataCommand.ExecuteReader(CommandBehavior.SequentialAccess);
91+
92+
// Verify that the destination table is empty before bulk copy
93+
using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection);
94+
Assert.Equal(0, Convert.ToInt16(countCommand.ExecuteScalar()));
95+
96+
// Initialize bulk copy configuration
97+
using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection)
98+
{
99+
EnableStreaming = enableStreaming,
100+
DestinationTableName = s_destinationTable
101+
};
102+
103+
try
104+
{
105+
// Perform bulk copy from source to destination table
106+
bulkCopy.WriteToServer(reader);
107+
}
108+
catch (Exception ex)
109+
{
110+
// If bulk copy fails, fail the test with the exception message
111+
Assert.Fail($"Bulk copy failed: {ex.Message}");
112+
}
113+
114+
// Verify that the 1 row from the source table has been copied into our destination table.
115+
Assert.Equal(1, Convert.ToInt16(countCommand.ExecuteScalar()));
116+
117+
// Read the data from destination table as varbinary to verify the UTF-8 byte sequence
118+
using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection);
119+
using SqlDataReader verifyReader = verifyCommand.ExecuteReader(CommandBehavior.SequentialAccess);
120+
121+
// Verify that we have data in the destination table
122+
Assert.True(verifyReader.Read(), "No data found in destination table after bulk copy.");
123+
124+
// Read the value of the column as SqlBinary.
125+
byte[] actualBytes = verifyReader.GetSqlBinary(0).Value;
126+
127+
// Verify that the byte array matches the expected UTF-8 byte sequence
128+
Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length);
129+
Assert.Equal(s_testValueInUtf8Bytes, actualBytes);
130+
}
131+
132+
/// <summary>
133+
/// Asynchronous version of the testcase BulkCopy_Utf8Data_ShouldMatchSource
134+
/// </summary>
135+
[ConditionalTheory(typeof(DataTestUtility),
136+
nameof(DataTestUtility.AreConnStringsSetup),
137+
nameof(DataTestUtility.IsNotAzureServer),
138+
nameof(DataTestUtility.IsNotAzureSynapse))]
139+
[InlineData(true, true)]
140+
[InlineData(false, true)]
141+
[InlineData(true, false)]
142+
[InlineData(false, false)]
143+
public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, bool enableStreaming)
144+
{
145+
// Setup connections for source and destination tables
146+
string connectionString = GetConnectionString(isMarsEnabled);
147+
using SqlConnection sourceConnection = new SqlConnection(connectionString);
148+
await sourceConnection.OpenAsync();
149+
using SqlConnection destinationConnection = new SqlConnection(connectionString);
150+
await destinationConnection.OpenAsync();
151+
152+
// Read data from source table
153+
using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection);
154+
using SqlDataReader reader = await sourceDataCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
155+
156+
// Verify that the destination table is empty before bulk copy
157+
using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection);
158+
Assert.Equal(0, Convert.ToInt16(await countCommand.ExecuteScalarAsync()));
159+
160+
// Initialize bulk copy configuration
161+
using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection)
162+
{
163+
EnableStreaming = enableStreaming,
164+
DestinationTableName = s_destinationTable
165+
};
166+
167+
try
168+
{
169+
// Perform bulk copy from source to destination table
170+
await bulkCopy.WriteToServerAsync(reader);
171+
}
172+
catch (Exception ex)
173+
{
174+
// If bulk copy fails, fail the test with the exception message
175+
Assert.Fail($"Bulk copy failed: {ex.Message}");
176+
}
177+
178+
// Verify that the 1 row from the source table has been copied into our destination table.
179+
Assert.Equal(1, Convert.ToInt16(await countCommand.ExecuteScalarAsync()));
180+
181+
// Read the data from destination table as varbinary to verify the UTF-8 byte sequence
182+
using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection);
183+
using SqlDataReader verifyReader = await verifyCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
184+
185+
// Verify that we have data in the destination table
186+
Assert.True(await verifyReader.ReadAsync(), "No data found in destination table after bulk copy.");
187+
188+
// Read the value of the column as SqlBinary.
189+
byte[] actualBytes = verifyReader.GetSqlBinary(0).Value;
190+
191+
// Verify that the byte array matches the expected UTF-8 byte sequence
192+
Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length);
193+
Assert.Equal(s_testValueInUtf8Bytes, actualBytes);
194+
}
195+
}
196+
}

0 commit comments

Comments
 (0)