adda validate count to tests

Author: Nick-Nuon

src/UTF8.cs

@@ -17,7 +17,7 @@ public static class UTF8
             for (int i = 0; i <= howFarBack; i++)
             {
                 byte b = buf[0 - i];
-                foundLeadingBytes = ((b & 0b11000000) != 0b10000000);
+                foundLeadingBytes = (b & 0b11000000) != 0b10000000;
                 if (foundLeadingBytes)
                 {
                     buf -= i;
@@ -647,22 +647,23 @@ public unsafe static void AdjustForSkippedBytes(byte* pInputBuffer,// int skippe
                             int candidateByte = pInputBuffer[processedLength + k];
                             if ((pInputBuffer[processedLength + k] & 0b11000000) == 0b11000000)
                             {
-                                if ((candidateByte & 0b11110000) == 0b11100000) // Start of a 3-byte sequence
+                                if ((candidateByte & 0b11100000) == 0b11000000) // Start of a 2-byte sequence
                                 {
-                                    TempUtf16CodeUnitCountAdjustment += 1; // Still adjusts for a single UTF-16 unit
+                                    TempUtf16CodeUnitCountAdjustment += 1; 
                                 }
-                                if ((candidateByte & 0b11111000) == 0b11110000) // Start of a 4-byte sequence
+                                if ((candidateByte & 0b11110000) == 0b11100000) // Start of a 3-byte sequence
                                 {
-                                    TempUtf16CodeUnitCountAdjustment += 1; // Adjusts for two UTF-16 units (surrogate pair)
-                                    TempScalarCountAdjustment += 1; // Adjust for one scalar value
+                                    TempUtf16CodeUnitCountAdjustment += 2; 
                                 }
-                                if ((candidateByte & 0b11100000) == 0b11000000) // Start of a 2-byte sequence
+                                if ((candidateByte & 0b11111000) == 0b11110000) // Start of a 4-byte sequence
                                 {
-                                    TempUtf16CodeUnitCountAdjustment += 1; // Adjust for a single UTF-16 unit
+                                    TempUtf16CodeUnitCountAdjustment += 2;
+                                    TempScalarCountAdjustment += 1;
                                 }
 
+
                                 processedLength += k;
-                                break;
+                                // break;
 
                             }
 
@@ -677,7 +678,7 @@ public unsafe static void AdjustForSkippedBytes(byte* pInputBuffer,// int skippe
             // Process the remaining bytes with the scalar function
             if (processedLength < inputLength)
             {
-                // We need to possibly backtrack to the start of the last code point
+                // // We need to possibly backtrack to the start of the last code point
                 while (processedLength > 0 && (sbyte)pInputBuffer[processedLength] <= -65)
                 {
                     processedLength -= 1;

test/UTF8ValidationTests.cs

@@ -188,6 +188,7 @@ public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
                 fixed (byte* pInput = input)
                 {
                     ValidateUtf8(input,utf8ValidationDelegate);
+                    ValidateCount(input,utf8ValidationDelegate);
                 }
             }
         }
@@ -249,6 +250,7 @@ public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
                 bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);
                 string utf8HexString = BitConverter.ToString(utf8).Replace("-", " ");
                 Assert.True(isValidUtf8, $"Failure NoErrorTest. Sequence: {utf8HexString}");
+                ValidateCount(utf8,utf8ValidationDelegate);
             }
         }
     }
@@ -310,6 +312,7 @@ private void RunTestForByteLength(int byteLength,Utf8ValidationDelegate utf8Vali
                 byte[] utf8 = generator.Generate(outputLength, byteLength).ToArray();
                 bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);
                 Assert.True(isValidUtf8, $"Failure for {byteLength}-byte UTF8 of length {outputLength} in trial {trial}");
+                ValidateCount(utf8,utf8ValidationDelegate);
             }
         }
     }
@@ -369,6 +372,7 @@ public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
                         utf8[i] = 0b11111000; // Forcing a header bits error
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = oldByte; // Restore the original byte
                     }
                 }
@@ -432,6 +436,7 @@ public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
                         utf8[i] = 0b11100000; // Forcing a too short error
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = oldByte; // Restore the original byte
                     }
                 }
@@ -498,6 +503,7 @@ public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
                         utf8[i] = 0b10000000; // Forcing a too long error
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = oldByte; // Restore the original byte
                     }
                 }
@@ -577,6 +583,7 @@ public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
 
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
 
                         utf8[i] = old;
                         utf8[i + 1] = secondOld;
@@ -665,6 +672,8 @@ public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
 
                             byte* dotnetResult = DotnetRuntime.Utf8Utility.GetPointerToFirstInvalidByte(pInput, i + offset, out SimdUnicodeUtf16Adjustment, out SimdUnicodeScalarCountAdjustment);
                             Assert.True(dotnetResult == pInput + i + offset);
+
+                            ValidateCount(utf8,utf8ValidationDelegate);
                             }
 
                     }    
@@ -737,6 +746,7 @@ public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
 
                     Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate)); // Expect the validation to fail due to the invalid byte
                     Assert.True(InvalidateUtf8(utf8,position,utf8ValidationDelegate));
+                    ValidateCount(utf8,utf8ValidationDelegate);
                 }
             }
         }
@@ -819,6 +829,7 @@ public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
 
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = old;
                     }
                 }
@@ -888,6 +899,7 @@ public void TooLargeErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
 
                     Assert.False(ValidateUtf8(filler,utf8ValidationDelegate));
                     Assert.True(InvalidateUtf8(filler, outputLength -1,utf8ValidationDelegate));
+                    ValidateCount(filler,utf8ValidationDelegate);
                 }
 
 
@@ -963,6 +975,7 @@ public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
 
                             Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                             Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                            ValidateCount(utf8,utf8ValidationDelegate);
                         }
 
                         utf8[i] = old;
@@ -1044,6 +1057,7 @@ public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
                     // Validate the modified sequence with both methods
                     bool isValidPrimary = ValidateUtf8(modifiedUtf8,utf8ValidationDelegate);
                     bool isValidFuschia = ValidateUtf8Fuschia(modifiedUtf8);
+                    ValidateCount(modifiedUtf8,utf8ValidationDelegate);
 
                     // Ensure both methods agree on the validation result
                     Assert.Equal(isValidPrimary, isValidFuschia);