adda validate count to tests

Nick-Nuon · Nick-Nuon · commit fe73718b142b · 2024-04-02T17:01:12.000-04:00
diff --git a/src/UTF8.cs b/src/UTF8.cs
@@ -17,7 +17,7 @@ public static class UTF8
             for (int i = 0; i <= howFarBack; i++)
             {
                 byte b = buf[0 - i];
-                foundLeadingBytes = ((b & 0b11000000) != 0b10000000);
+                foundLeadingBytes = (b & 0b11000000) != 0b10000000;
                 if (foundLeadingBytes)
                 {
                     buf -= i;
@@ -647,22 +647,23 @@ public unsafe static void AdjustForSkippedBytes(byte* pInputBuffer,// int skippe
                             int candidateByte = pInputBuffer[processedLength + k];
                             if ((pInputBuffer[processedLength + k] & 0b11000000) == 0b11000000)
                             {
-                                if ((candidateByte & 0b11110000) == 0b11100000) // Start of a 3-byte sequence
+                                if ((candidateByte & 0b11100000) == 0b11000000) // Start of a 2-byte sequence
                                 {
-                                    TempUtf16CodeUnitCountAdjustment += 1; // Still adjusts for a single UTF-16 unit
+                                    TempUtf16CodeUnitCountAdjustment += 1; 
                                 }
-                                if ((candidateByte & 0b11111000) == 0b11110000) // Start of a 4-byte sequence
+                                if ((candidateByte & 0b11110000) == 0b11100000) // Start of a 3-byte sequence
                                 {
-                                    TempUtf16CodeUnitCountAdjustment += 1; // Adjusts for two UTF-16 units (surrogate pair)
-                                    TempScalarCountAdjustment += 1; // Adjust for one scalar value
+                                    TempUtf16CodeUnitCountAdjustment += 2; 
                                 }
-                                if ((candidateByte & 0b11100000) == 0b11000000) // Start of a 2-byte sequence
+                                if ((candidateByte & 0b11111000) == 0b11110000) // Start of a 4-byte sequence
                                 {
-                                    TempUtf16CodeUnitCountAdjustment += 1; // Adjust for a single UTF-16 unit
+                                    TempUtf16CodeUnitCountAdjustment += 2;
+                                    TempScalarCountAdjustment += 1;
                                 }
 
+
                                 processedLength += k;
-                                break;
+                                // break;
 
                             }
 
@@ -677,7 +678,7 @@ public unsafe static void AdjustForSkippedBytes(byte* pInputBuffer,// int skippe
             // Process the remaining bytes with the scalar function
             if (processedLength < inputLength)
             {
-                // We need to possibly backtrack to the start of the last code point
+                // // We need to possibly backtrack to the start of the last code point
                 while (processedLength > 0 && (sbyte)pInputBuffer[processedLength] <= -65)
                 {
                     processedLength -= 1;
diff --git a/test/UTF8ValidationTests.cs b/test/UTF8ValidationTests.cs
@@ -188,6 +188,7 @@ public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
                 fixed (byte* pInput = input)
                 {
                     ValidateUtf8(input,utf8ValidationDelegate);
+                    ValidateCount(input,utf8ValidationDelegate);
                 }
             }
         }
@@ -249,6 +250,7 @@ public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
                 bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);
                 string utf8HexString = BitConverter.ToString(utf8).Replace("-", " ");
                 Assert.True(isValidUtf8, $"Failure NoErrorTest. Sequence: {utf8HexString}");
+                ValidateCount(utf8,utf8ValidationDelegate);
             }
         }
     }
@@ -310,6 +312,7 @@ private void RunTestForByteLength(int byteLength,Utf8ValidationDelegate utf8Vali
                 byte[] utf8 = generator.Generate(outputLength, byteLength).ToArray();
                 bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);
                 Assert.True(isValidUtf8, $"Failure for {byteLength}-byte UTF8 of length {outputLength} in trial {trial}");
+                ValidateCount(utf8,utf8ValidationDelegate);
             }
         }
     }
@@ -369,6 +372,7 @@ public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
                         utf8[i] = 0b11111000; // Forcing a header bits error
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = oldByte; // Restore the original byte
                     }
                 }
@@ -432,6 +436,7 @@ public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
                         utf8[i] = 0b11100000; // Forcing a too short error
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = oldByte; // Restore the original byte
                     }
                 }
@@ -498,6 +503,7 @@ public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
                         utf8[i] = 0b10000000; // Forcing a too long error
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = oldByte; // Restore the original byte
                     }
                 }
@@ -577,6 +583,7 @@ public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
 
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
 
                         utf8[i] = old;
                         utf8[i + 1] = secondOld;
@@ -665,6 +672,8 @@ public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
 
                             byte* dotnetResult = DotnetRuntime.Utf8Utility.GetPointerToFirstInvalidByte(pInput, i + offset, out SimdUnicodeUtf16Adjustment, out SimdUnicodeScalarCountAdjustment);
                             Assert.True(dotnetResult == pInput + i + offset);
+
+                            ValidateCount(utf8,utf8ValidationDelegate);
                             }
 
                     }    
@@ -737,6 +746,7 @@ public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
 
                     Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate)); // Expect the validation to fail due to the invalid byte
                     Assert.True(InvalidateUtf8(utf8,position,utf8ValidationDelegate));
+                    ValidateCount(utf8,utf8ValidationDelegate);
                 }
             }
         }
@@ -819,6 +829,7 @@ public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
 
                         Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                         Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                        ValidateCount(utf8,utf8ValidationDelegate);
                         utf8[i] = old;
                     }
                 }
@@ -888,6 +899,7 @@ public void TooLargeErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
 
                     Assert.False(ValidateUtf8(filler,utf8ValidationDelegate));
                     Assert.True(InvalidateUtf8(filler, outputLength -1,utf8ValidationDelegate));
+                    ValidateCount(filler,utf8ValidationDelegate);
                 }
 
 
@@ -963,6 +975,7 @@ public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
 
                             Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
                             Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
+                            ValidateCount(utf8,utf8ValidationDelegate);
                         }
 
                         utf8[i] = old;
@@ -1044,6 +1057,7 @@ public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
                     // Validate the modified sequence with both methods
                     bool isValidPrimary = ValidateUtf8(modifiedUtf8,utf8ValidationDelegate);
                     bool isValidFuschia = ValidateUtf8Fuschia(modifiedUtf8);
+                    ValidateCount(modifiedUtf8,utf8ValidationDelegate);
 
                     // Ensure both methods agree on the validation result
                     Assert.Equal(isValidPrimary, isValidFuschia);

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ public static class UTF8`
`17`	`17`	`for (int i = 0; i <= howFarBack; i++)`
`18`	`18`	`{`
`19`	`19`	`byte b = buf[0 - i];`
`20`		`- foundLeadingBytes = ((b & 0b11000000) != 0b10000000);`
	`20`	`+ foundLeadingBytes = (b & 0b11000000) != 0b10000000;`
`21`	`21`	`if (foundLeadingBytes)`
`22`	`22`	`{`
`23`	`23`	`buf -= i;`
`@@ -647,22 +647,23 @@ public unsafe static void AdjustForSkippedBytes(byte* pInputBuffer,// int skippe`
`647`	`647`	`int candidateByte = pInputBuffer[processedLength + k];`
`648`	`648`	`if ((pInputBuffer[processedLength + k] & 0b11000000) == 0b11000000)`
`649`	`649`	`{`
`650`		`- if ((candidateByte & 0b11110000) == 0b11100000) // Start of a 3-byte sequence`
	`650`	`+ if ((candidateByte & 0b11100000) == 0b11000000) // Start of a 2-byte sequence`
`651`	`651`	`{`
`652`		`- TempUtf16CodeUnitCountAdjustment += 1; // Still adjusts for a single UTF-16 unit`
	`652`	`+ TempUtf16CodeUnitCountAdjustment += 1;`
`653`	`653`	`}`
`654`		`- if ((candidateByte & 0b11111000) == 0b11110000) // Start of a 4-byte sequence`
	`654`	`+ if ((candidateByte & 0b11110000) == 0b11100000) // Start of a 3-byte sequence`
`655`	`655`	`{`
`656`		`- TempUtf16CodeUnitCountAdjustment += 1; // Adjusts for two UTF-16 units (surrogate pair)`
`657`		`- TempScalarCountAdjustment += 1; // Adjust for one scalar value`
	`656`	`+ TempUtf16CodeUnitCountAdjustment += 2;`
`658`	`657`	`}`
`659`		`- if ((candidateByte & 0b11100000) == 0b11000000) // Start of a 2-byte sequence`
	`658`	`+ if ((candidateByte & 0b11111000) == 0b11110000) // Start of a 4-byte sequence`
`660`	`659`	`{`
`661`		`- TempUtf16CodeUnitCountAdjustment += 1; // Adjust for a single UTF-16 unit`
	`660`	`+ TempUtf16CodeUnitCountAdjustment += 2;`
	`661`	`+ TempScalarCountAdjustment += 1;`
`662`	`662`	`}`
`663`	`663`
	`664`	`+`
`664`	`665`	`processedLength += k;`
`665`		`- break;`
	`666`	`+ // break;`
`666`	`667`
`667`	`668`	`}`
`668`	`669`
`@@ -677,7 +678,7 @@ public unsafe static void AdjustForSkippedBytes(byte* pInputBuffer,// int skippe`
`677`	`678`	`// Process the remaining bytes with the scalar function`
`678`	`679`	`if (processedLength < inputLength)`
`679`	`680`	`{`
`680`		`- // We need to possibly backtrack to the start of the last code point`
	`681`	`+ // // We need to possibly backtrack to the start of the last code point`
`681`	`682`	`while (processedLength > 0 && (sbyte)pInputBuffer[processedLength] <= -65)`
`682`	`683`	`{`
`683`	`684`	`processedLength -= 1;`
Original file line number	Diff line number	Diff line change
`@@ -188,6 +188,7 @@ public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)`
`188`	`188`	`fixed (byte* pInput = input)`
`189`	`189`	`{`
`190`	`190`	`ValidateUtf8(input,utf8ValidationDelegate);`
	`191`	`+ ValidateCount(input,utf8ValidationDelegate);`
`191`	`192`	`}`
`192`	`193`	`}`
`193`	`194`	`}`
`@@ -249,6 +250,7 @@ public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)`
`249`	`250`	`bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);`
`250`	`251`	`string utf8HexString = BitConverter.ToString(utf8).Replace("-", " ");`
`251`	`252`	`Assert.True(isValidUtf8, $"Failure NoErrorTest. Sequence: {utf8HexString}");`
	`253`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`252`	`254`	`}`
`253`	`255`	`}`
`254`	`256`	`}`
`@@ -310,6 +312,7 @@ private void RunTestForByteLength(int byteLength,Utf8ValidationDelegate utf8Vali`
`310`	`312`	`byte[] utf8 = generator.Generate(outputLength, byteLength).ToArray();`
`311`	`313`	`bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);`
`312`	`314`	`Assert.True(isValidUtf8, $"Failure for {byteLength}-byte UTF8 of length {outputLength} in trial {trial}");`
	`315`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`313`	`316`	`}`
`314`	`317`	`}`
`315`	`318`	`}`
`@@ -369,6 +372,7 @@ public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)`
`369`	`372`	`utf8[i] = 0b11111000; // Forcing a header bits error`
`370`	`373`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));`
`371`	`374`	`Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));`
	`375`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`372`	`376`	`utf8[i] = oldByte; // Restore the original byte`
`373`	`377`	`}`
`374`	`378`	`}`
`@@ -432,6 +436,7 @@ public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)`
`432`	`436`	`utf8[i] = 0b11100000; // Forcing a too short error`
`433`	`437`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));`
`434`	`438`	`Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));`
	`439`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`435`	`440`	`utf8[i] = oldByte; // Restore the original byte`
`436`	`441`	`}`
`437`	`442`	`}`
`@@ -498,6 +503,7 @@ public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)`
`498`	`503`	`utf8[i] = 0b10000000; // Forcing a too long error`
`499`	`504`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));`
`500`	`505`	`Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));`
	`506`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`501`	`507`	`utf8[i] = oldByte; // Restore the original byte`
`502`	`508`	`}`
`503`	`509`	`}`
`@@ -577,6 +583,7 @@ public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)`
`577`	`583`
`578`	`584`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));`
`579`	`585`	`Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));`
	`586`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`580`	`587`
`581`	`588`	`utf8[i] = old;`
`582`	`589`	`utf8[i + 1] = secondOld;`
`@@ -665,6 +672,8 @@ public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)`
`665`	`672`
`666`	`673`	`byte* dotnetResult = DotnetRuntime.Utf8Utility.GetPointerToFirstInvalidByte(pInput, i + offset, out SimdUnicodeUtf16Adjustment, out SimdUnicodeScalarCountAdjustment);`
`667`	`674`	`Assert.True(dotnetResult == pInput + i + offset);`
	`675`	`+`
	`676`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`668`	`677`	`}`
`669`	`678`
`670`	`679`	`}`
`@@ -737,6 +746,7 @@ public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)`
`737`	`746`
`738`	`747`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate)); // Expect the validation to fail due to the invalid byte`
`739`	`748`	`Assert.True(InvalidateUtf8(utf8,position,utf8ValidationDelegate));`
	`749`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`740`	`750`	`}`
`741`	`751`	`}`
`742`	`752`	`}`
`@@ -819,6 +829,7 @@ public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)`
`819`	`829`
`820`	`830`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));`
`821`	`831`	`Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));`
	`832`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`822`	`833`	`utf8[i] = old;`
`823`	`834`	`}`
`824`	`835`	`}`
`@@ -888,6 +899,7 @@ public void TooLargeErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)`
`888`	`899`
`889`	`900`	`Assert.False(ValidateUtf8(filler,utf8ValidationDelegate));`
`890`	`901`	`Assert.True(InvalidateUtf8(filler, outputLength -1,utf8ValidationDelegate));`
	`902`	`+ ValidateCount(filler,utf8ValidationDelegate);`
`891`	`903`	`}`
`892`	`904`
`893`	`905`
`@@ -963,6 +975,7 @@ public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)`
`963`	`975`
`964`	`976`	`Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));`
`965`	`977`	`Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));`
	`978`	`+ ValidateCount(utf8,utf8ValidationDelegate);`
`966`	`979`	`}`
`967`	`980`
`968`	`981`	`utf8[i] = old;`
`@@ -1044,6 +1057,7 @@ public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)`
`1044`	`1057`	`// Validate the modified sequence with both methods`
`1045`	`1058`	`bool isValidPrimary = ValidateUtf8(modifiedUtf8,utf8ValidationDelegate);`
`1046`	`1059`	`bool isValidFuschia = ValidateUtf8Fuschia(modifiedUtf8);`
	`1060`	`+ ValidateCount(modifiedUtf8,utf8ValidationDelegate);`
`1047`	`1061`
`1048`	`1062`	`// Ensure both methods agree on the validation result`
`1049`	`1063`	`Assert.Equal(isValidPrimary, isValidFuschia);`