Skip to content

Commit 202a996

Browse files
Nick-NuonNick-Nuon
committed
further clean up
1 parent 8e84627 commit 202a996

File tree

1 file changed

+2
-73
lines changed

1 file changed

+2
-73
lines changed

‎test/UTF8ValidationTests.cs

Lines changed: 2 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,6 @@ public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
260260
}
261261
catch (Xunit.Sdk.XunitException)
262262
{
263-
// Console.WriteLine($"Assertion failed at index: ");
264263
PrintHexAndBinary(utf8);
265264
throw; // Rethrow the exception to fail the test.
266265
}
@@ -390,16 +389,11 @@ public void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDe
390389
// var allAscii = generator.Generate(outputLength,1);
391390
var allAscii = new List<byte>(Enumerable.Repeat((byte)0, 256));
392391
int firstcodeLength = rand.Next(2,5);
393-
int secondcodeLength = rand.Next(2,5);
394392
List<byte> singlebytes = generator.Generate(1,firstcodeLength);//recall:generate a utf8 code between 2 and 4 bytes
395-
List<byte> secondbyte = generator.Generate(1,secondcodeLength);
396-
singlebytes.AddRange(secondbyte);
397-
398-
int incompleteLocation = 127 - rand.Next(1,firstcodeLength + secondcodeLength - 1);
393+
int incompleteLocation = 128 - rand.Next(1,firstcodeLength - 1);
399394
allAscii.InsertRange(incompleteLocation,singlebytes);
400395

401396
var utf8 = allAscii.ToArray();
402-
// PrintHexAndBinary(utf8,incompleteLocation);
403397

404398
bool isValidUtf8 = ValidateUtf8(utf8,utf8ValidationDelegate);
405399
string utf8HexString = BitConverter.ToString(utf8).Replace("-", " ");
@@ -411,7 +405,6 @@ public void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDe
411405
}
412406
catch (Xunit.Sdk.XunitException)
413407
{
414-
// Console.WriteLine($"Assertion failed at index: ");
415408
PrintHexAndBinary(utf8,incompleteLocation);
416409
throw; // Rethrow the exception to fail the test.
417410
}
@@ -473,9 +466,6 @@ public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
473466
{
474467
byte oldByte = utf8[i];
475468
utf8[i] = 0b11111000; // Forcing a header bits error
476-
// Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
477-
// Assert.True(InvalidateUtf8(utf8, i,utf8ValidationDelegate));
478-
// ValidateCount(utf8,utf8ValidationDelegate);
479469
try
480470
{
481471
Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate));
@@ -615,10 +605,6 @@ public void TooShortErrorAVX()
615605
public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
616606
{
617607

618-
// int[] outputLengths = { 128, 256, 512, 1024 }; // Example lengths
619-
// int[] outputLengths = { 128, 256,345, 512,968, 1024, 1000 }; // Example lengths
620-
621-
622608
foreach (int outputLength in outputLengths)
623609
{
624610
for (int trial = 0; trial < NumTrials; trial++)
@@ -769,7 +755,6 @@ public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
769755
{
770756
foreach (int outputLength in outputLengths)
771757
{
772-
// List<byte> oneUTFunit = generator.Generate( howManyUnits:1 ,byteCountInUnit: 2);
773758
byte[] utf8 = generator.Generate(outputLength,byteCountInUnit: 1).ToArray();
774759

775760
unsafe
@@ -779,7 +764,6 @@ public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
779764

780765
for (int i = 0; i < utf8.Length; i++)
781766
{
782-
// int DotnetUtf16Adjustment, DotnetScalarCountAdjustment;
783767
int SimdUnicodeUtf16Adjustment, SimdUnicodeScalarCountAdjustment;
784768
byte currentByte = utf8[i];
785769
int offset = 0;
@@ -789,7 +773,6 @@ public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
789773
}
790774
if ((currentByte & 0b11110000) == 0b11100000) {
791775
// This is a header byte of a 3-byte sequence
792-
793776
offset = rand.Next(0, 3);
794777
}
795778
if ((currentByte & 0b11111000) == 0b11110000) {
@@ -872,8 +855,6 @@ public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
872855
foreach (var invalidByte in invalidBytes)
873856
{
874857
utf8[position] = invalidByte;
875-
// PrintHexAndBinary(utf8);
876-
877858
Assert.False(ValidateUtf8(utf8,utf8ValidationDelegate)); // Expect the validation to fail due to the invalid byte
878859
Assert.True(InvalidateUtf8(utf8,position,utf8ValidationDelegate));
879860
ValidateCount(utf8,utf8ValidationDelegate);
@@ -927,6 +908,7 @@ public void Invalid0xf50xffAvx2()
927908
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
928909
}
929910

911+
// helper function for debugging: it prints a green byte every 32 bytes and a red byte at a given index
930912
static void PrintHexAndBinary(byte[] bytes, int highlightIndex = -1)
931913
{
932914
int chunkSize = 16; // 128 bits = 16 bytes
@@ -1441,7 +1423,6 @@ public void ValidateCount(byte[] utf8, Utf8ValidationDelegate utf8ValidationDele
14411423
}
14421424
catch (Exception)
14431425
{
1444-
// Upon failure, print the utf8 array for inspection
14451426
Console.WriteLine("ValidateCount Assertion failed. Inspecting utf8 array:");
14461427
// PrintHexAndBinary(utf8,failureIndex);
14471428
throw; // Re-throw the exception to preserve the failure state
@@ -1450,58 +1431,6 @@ public void ValidateCount(byte[] utf8, Utf8ValidationDelegate utf8ValidationDele
14501431
}
14511432
}
14521433

1453-
1454-
// [Fact]
1455-
// [Trait("Category", "Scalar")]
1456-
// public void DotnetUTF16Count()
1457-
// {
1458-
// int[] outputLengths = { 10, 15, 11,12 ,15,15,1, 3, 5, 8, 10, 12, 15, 18 };
1459-
// int DotnetUtf16Adjustment, DotnetScalarCountAdjustment;
1460-
// int SimdUnicodeUtf16Adjustment, SimdUnicodeScalarCountAdjustment;
1461-
1462-
1463-
// foreach (int outputLength in outputLengths)
1464-
// {
1465-
// // Generate a UTF-8 sequence with 3 units, each 2 bytes long, presumed to be valid.
1466-
// // byte[] utf8 = generator.Generate(howManyUnits: 11, byteCountInUnit: 3).ToArray();
1467-
// byte[] utf8 = generator.Generate(howManyUnits: outputLength).ToArray();
1468-
// PrintHexAndBinary(utf8);
1469-
// var (offset, length) = (0, utf8.Length);
1470-
1471-
// unsafe
1472-
// {
1473-
// fixed (byte* pInput = utf8)
1474-
// {
1475-
// byte* startPtr = pInput + offset;
1476-
// // Invoke the method under test.
1477-
1478-
// DotnetUtf16Adjustment= 0;
1479-
// DotnetScalarCountAdjustment= 0;
1480-
// DotnetRuntime.Utf8Utility.GetPointerToFirstInvalidByte(pInput, length, out DotnetUtf16Adjustment, out DotnetScalarCountAdjustment);
1481-
1482-
// SimdUnicodeUtf16Adjustment= 0;
1483-
// SimdUnicodeScalarCountAdjustment= 0;
1484-
// SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar(pInput, length, out SimdUnicodeUtf16Adjustment, out SimdUnicodeScalarCountAdjustment);
1485-
1486-
// Console.WriteLine("Lenght:" + utf8.Length);
1487-
1488-
// Console.WriteLine("DotnetScalar:" + DotnetScalarCountAdjustment);
1489-
// Console.WriteLine("OurScalar:" + SimdUnicodeScalarCountAdjustment);
1490-
1491-
// Console.WriteLine("Dotnetutf16:" + DotnetUtf16Adjustment);
1492-
// Console.WriteLine("Ourutf16:" + SimdUnicodeUtf16Adjustment);
1493-
// Console.WriteLine("___________________________________________________");
1494-
1495-
1496-
// Assert.True(DotnetUtf16Adjustment == SimdUnicodeUtf16Adjustment, $"Expected UTF16 Adjustment: {DotnetUtf16Adjustment}, but got: {SimdUnicodeUtf16Adjustment}.");
1497-
// Assert.True(DotnetScalarCountAdjustment == SimdUnicodeScalarCountAdjustment, $"Expected Scalar Count Adjustment: {DotnetScalarCountAdjustment}, but got: {SimdUnicodeScalarCountAdjustment}.");
1498-
1499-
// }
1500-
// }
1501-
// }
1502-
// }
1503-
1504-
15051434
}
15061435

15071436

0 commit comments

Comments
 (0)