Use UTF8 instance that doesn't emit BOM

Author: apoorvdeshmukh

src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs

@@ -52,6 +52,7 @@ internal static void Assert(string message)
             }
         }
 
+        private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
         private static int _objectTypeCount; // EventSource counter
         private readonly SqlClientLogger _logger = new SqlClientLogger();
 
@@ -2791,7 +2792,7 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
                             // UTF8 collation
                             if (env._newCollation.IsUTF8)
                             {
-                                _defaultEncoding = Encoding.UTF8;
+                                _defaultEncoding = s_utf8EncodingWithoutBom;
                             }
                             else
                             {
@@ -4199,7 +4200,7 @@ internal bool TryProcessReturnValue(int length, TdsParserStateObject stateObj, o
                 // UTF8 collation
                 if (rec.collation.IsUTF8)
                 {
-                    rec.encoding = Encoding.UTF8;
+                    rec.encoding = s_utf8EncodingWithoutBom;
                 }
                 else
                 {
@@ -4986,7 +4987,7 @@ private bool TryProcessTypeInfo(TdsParserStateObject stateObj, SqlMetaDataPriv c
                 // UTF8 collation
                 if (col.collation.IsUTF8)
                 {
-                    col.encoding = Encoding.UTF8;
+                    col.encoding = s_utf8EncodingWithoutBom;
                 }
                 else
                 {
@@ -10801,7 +10802,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
                     // Replace encoding if it is UTF8
                     if (metadata.collation.IsUTF8)
                     {
-                        _defaultEncoding = Encoding.UTF8;
+                        _defaultEncoding = s_utf8EncodingWithoutBom;
                     }
 
                     _defaultCollation = metadata.collation;

src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs

@@ -33,6 +33,7 @@ namespace Microsoft.Data.SqlClient
     // and surfacing objects to the user.
     sealed internal class TdsParser
     {
+        private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
         private static int _objectTypeCount; // EventSource Counter
         private readonly SqlClientLogger _logger = new SqlClientLogger();
 
@@ -3235,7 +3236,7 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
                             // UTF8 collation
                             if (env._newCollation.IsUTF8)
                             {
-                                _defaultEncoding = Encoding.UTF8;
+                                _defaultEncoding = s_utf8EncodingWithoutBom;
                             }
                             else
                             {
@@ -4757,7 +4758,7 @@ internal bool TryProcessReturnValue(int length,
 
                 if (rec.collation.IsUTF8)
                 { // UTF8 collation
-                    rec.encoding = Encoding.UTF8;
+                    rec.encoding = s_utf8EncodingWithoutBom;
                 }
                 else
                 {
@@ -5657,7 +5658,7 @@ private bool TryProcessTypeInfo(TdsParserStateObject stateObj, SqlMetaDataPriv c
 
                 if (col.collation.IsUTF8)
                 { // UTF8 collation
-                    col.encoding = Encoding.UTF8;
+                    col.encoding = s_utf8EncodingWithoutBom;
                 }
                 else
                 {
@@ -11740,7 +11741,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
                     // Replace encoding if it is UTF8
                     if (metadata.collation.IsUTF8)
                     {
-                        _defaultEncoding = Encoding.UTF8;
+                        _defaultEncoding = s_utf8EncodingWithoutBom;
                     }
 
                     _defaultCollation = metadata.collation;

src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs

@@ -590,6 +590,17 @@ public static string GetUniqueNameForSqlServer(string prefix, bool withBracket =
             return name;
         }
 
+        public static void CreateTable(SqlConnection sqlConnection, string tableName, string createBody)
+        {
+            DropTable(sqlConnection, tableName);
+            string tableCreate = "CREATE TABLE " + tableName + createBody;
+            using (SqlCommand command = sqlConnection.CreateCommand())
+            {
+                command.CommandText = tableCreate;
+                command.ExecuteNonQuery();
+            }
+        }
+
         public static void DropTable(SqlConnection sqlConnection, string tableName)
         {
             ResurrectConnection(sqlConnection);

src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj

@@ -285,6 +285,7 @@
     <Compile Include="SQL\Common\SystemDataInternals\TdsParserHelper.cs" />
     <Compile Include="SQL\Common\SystemDataInternals\TdsParserStateObjectHelper.cs" />
     <Compile Include="SQL\ConnectionTestWithSSLCert\CertificateTest.cs" />
+    <Compile Include="SQL\SqlBulkCopyTest\TestBulkCopyWithUTF8.cs" />
     <Compile Include="SQL\SqlCommand\SqlCommandStoredProcTest.cs" />
     <Compile Include="TracingTests\TestTdsServer.cs" />
     <Compile Include="XUnitAssemblyAttributes.cs" />

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/TestBulkCopyWithUTF8.cs

@@ -0,0 +1,197 @@
+using System;
+using System.Data;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.SqlBulkCopyTest
+{
+    /// <summary>
+    /// Validates SqlBulkCopy functionality when working with UTF-8 encoded data.
+    /// Ensures that data copied from a UTF-8 source table to a destination table retains its encoding and content integrity.
+    /// </summary>
+    public sealed class TestBulkCopyWithUtf8 : IDisposable
+    {
+        private static bool IsAzureServer() => !DataTestUtility.IsNotAzureServer();
+        private static bool IsNotAzureSynapse => DataTestUtility.IsNotAzureSynapse();
+        private static bool AreConnectionStringsSetup() => DataTestUtility.AreConnStringsSetup();
+
+        private static string s_sourceTable = DataTestUtility.GetUniqueName("SourceTableForUTF8Data");
+        private static string s_destinationTable = DataTestUtility.GetUniqueName("DestinationTableForUTF8Data");
+        private static string s_testValue = "test";
+        private static byte[] s_testValueInUtf8Bytes = new byte[] { 0x74, 0x65, 0x73, 0x74 };
+        private static readonly string s_insertQuery = $"INSERT INTO {s_sourceTable} VALUES('{s_testValue}')";
+
+        /// <summary>
+        /// Constructor: Initializes and populates source and destination tables required for the tests.
+        /// </summary>
+        public TestBulkCopyWithUtf8()
+        {
+            using SqlConnection sourceConnection = new SqlConnection(GetConnectionString(true));
+            sourceConnection.Open();
+            SetupTables(sourceConnection, s_sourceTable, s_destinationTable, s_insertQuery);
+        }
+
+        /// <summary>
+        /// Cleanup method to drop tables after test completion.
+        /// </summary>
+        public void Dispose()
+        {
+            using SqlConnection connection = new SqlConnection(GetConnectionString(true));
+            connection.Open();
+            DataTestUtility.DropTable(connection, s_sourceTable);
+            DataTestUtility.DropTable(connection, s_destinationTable);
+            connection.Close();
+        }
+
+        /// <summary>
+        /// Builds a connection string with or without Multiple Active Result Sets (MARS) property.
+        /// </summary>
+        private string GetConnectionString(bool enableMars)
+        {
+            return new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString)
+            {
+                MultipleActiveResultSets = enableMars
+            }.ConnectionString;
+        }
+
+        /// <summary>
+        /// Creates source and destination tables with a varchar(max) column with a collation setting
+        /// that stores the data in UTF8 encoding and inserts the data in the source table. 
+        /// </summary>
+        private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable, string insertQuery)
+        {
+            string columnDefinition = "(str_col varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8)";
+            DataTestUtility.CreateTable(connection, sourceTable, columnDefinition);
+            DataTestUtility.CreateTable(connection, destinationTable, columnDefinition);
+            using SqlCommand insertCommand = connection.CreateCommand();
+            insertCommand.CommandText = insertQuery;
+            Helpers.TryExecute(insertCommand, insertQuery);
+        }
+
+        /// <summary>
+        /// Synchronous test case: Validates that data copied using SqlBulkCopy matches UTF-8 byte sequence for test value.
+        /// Tested with MARS enabled and disabled, and with streaming enabled and disabled.
+        /// </summary>
+        [Theory]
+        [InlineData(true, true)]
+        [InlineData(false, true)]
+        [InlineData(true, false)]
+        [InlineData(false, false)]
+        public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableStreaming)
+        {
+            // Setup connections for source and destination tables
+            string connectionString = GetConnectionString(isMarsEnabled);
+            using SqlConnection sourceConnection = new SqlConnection(connectionString);
+            sourceConnection.Open();
+            using SqlConnection destinationConnection = new SqlConnection(connectionString);
+            destinationConnection.Open();
+
+            // Read data from source table
+            using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection);
+            using SqlDataReader reader = sourceDataCommand.ExecuteReader(CommandBehavior.SequentialAccess);
+
+            // Verify that the destination table is empty before bulk copy
+            using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection);
+            Assert.Equal(0, Convert.ToInt16(countCommand.ExecuteScalar()));
+
+            // Initialize bulk copy configuration
+            using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection)
+            {
+                EnableStreaming = enableStreaming,
+                DestinationTableName = s_destinationTable
+            };
+
+            try
+            {
+                // Perform bulk copy from source to destination table
+                bulkCopy.WriteToServer(reader);
+            }
+            catch (Exception ex)
+            {
+                // If bulk copy fails, fail the test with the exception message
+                Assert.Fail($"Bulk copy failed: {ex.Message}");
+            }
+
+            // Verify that the 1 row from the source table has been copied into our destination table.
+            Assert.Equal(1, Convert.ToInt16(countCommand.ExecuteScalar()));
+
+            // Read the data from destination table as varbinary to verify the UTF-8 byte sequence
+            using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection);
+            using SqlDataReader verifyReader = verifyCommand.ExecuteReader(CommandBehavior.SequentialAccess);
+
+            // Verify that we have data in the destination table
+            Assert.True(verifyReader.Read(), "No data found in destination table after bulk copy.");
+
+            // Read the value of the column as SqlBinary.
+            byte[] actualBytes = verifyReader.GetSqlBinary(0).Value;
+
+            // Verify that the byte array matches the expected UTF-8 byte sequence
+            Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length);
+            Assert.Equal(s_testValueInUtf8Bytes, actualBytes);
+        }
+
+        /// <summary>
+        /// Asynchronous version of the testcase BulkCopy_Utf8Data_ShouldMatchSource
+        /// </summary>
+        [ConditionalTheory(typeof(DataTestUtility),
+         nameof(DataTestUtility.AreConnStringsSetup),
+         nameof(DataTestUtility.IsNotAzureServer),
+         nameof(DataTestUtility.IsNotAzureSynapse))]
+        [InlineData(true, true)]
+        [InlineData(false, true)]
+        [InlineData(true, false)]
+        [InlineData(false, false)]
+        public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, bool enableStreaming)
+        {
+            // Setup connections for source and destination tables
+            string connectionString = GetConnectionString(isMarsEnabled);
+            using SqlConnection sourceConnection = new SqlConnection(connectionString);
+            await sourceConnection.OpenAsync();
+            using SqlConnection destinationConnection = new SqlConnection(connectionString);
+            await destinationConnection.OpenAsync();
+
+            // Read data from source table
+            using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection);
+            using SqlDataReader reader = await sourceDataCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
+
+            // Verify that the destination table is empty before bulk copy
+            using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection);
+            Assert.Equal(0, Convert.ToInt16(await countCommand.ExecuteScalarAsync()));
+
+            // Initialize bulk copy configuration
+            using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection)
+            {
+                EnableStreaming = enableStreaming,
+                DestinationTableName = s_destinationTable
+            };
+
+            try
+            {
+                // Perform bulk copy from source to destination table
+                await bulkCopy.WriteToServerAsync(reader);
+            }
+            catch (Exception ex)
+            {
+                // If bulk copy fails, fail the test with the exception message
+                Assert.Fail($"Bulk copy failed: {ex.Message}");
+            }
+
+            // Verify that the 1 row from the source table has been copied into our destination table.
+            Assert.Equal(1, Convert.ToInt16(await countCommand.ExecuteScalarAsync()));
+
+            // Read the data from destination table as varbinary to verify the UTF-8 byte sequence
+            using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection);
+            using SqlDataReader verifyReader = await verifyCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
+
+            // Verify that we have data in the destination table
+            Assert.True(await verifyReader.ReadAsync(), "No data found in destination table after bulk copy.");
+
+            // Read the value of the column as SqlBinary.
+            byte[] actualBytes = verifyReader.GetSqlBinary(0).Value;
+
+            // Verify that the byte array matches the expected UTF-8 byte sequence
+            Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length);
+            Assert.Equal(s_testValueInUtf8Bytes, actualBytes);
+        }
+    }
+}