@@ -10,12 +10,6 @@ namespace SimdUnicode
10
10
public static class UTF8
11
11
{
12
12
13
- public static void ToString ( Vector128 < byte > v )
14
- {
15
- Span < byte > b = stackalloc byte [ 16 ] ;
16
- v . CopyTo ( b ) ;
17
- // Console .WriteLine(Convert.ToHexString(b));
18
- }
19
13
20
14
// Returns &inputBuffer[inputLength] if the input buffer is valid.
21
15
/// <summary>
@@ -42,11 +36,10 @@ public static void ToString(Vector128<byte> v)
42
36
{
43
37
return GetPointerToFirstInvalidByteAvx512(pInputBuffer, inputLength);
44
38
}*/
45
- // if (Ssse3.IsSupported)
46
- // {
47
- // return GetPointerToFirstInvalidByteSse(pInputBuffer, inputLength);
48
- // }
49
- // return GetPointerToFirstInvalidByteScalar(pInputBuffer, inputLength);
39
+ if ( Ssse3 . IsSupported )
40
+ {
41
+ return GetPointerToFirstInvalidByteSse ( pInputBuffer , inputLength , out Utf16CodeUnitCountAdjustment , out ScalarCodeUnitCountAdjustment ) ;
42
+ }
50
43
51
44
return GetPointerToFirstInvalidByteScalar ( pInputBuffer , inputLength , out Utf16CodeUnitCountAdjustment , out ScalarCodeUnitCountAdjustment ) ;
52
45
@@ -480,7 +473,6 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
480
473
481
474
public unsafe static byte * GetPointerToFirstInvalidByteSse ( byte * pInputBuffer , int inputLength , out int utf16CodeUnitCountAdjustment , out int scalarCountAdjustment )
482
475
{
483
- // Console .WriteLine("-------------------Calling function--------------------");
484
476
int processedLength = 0 ;
485
477
if ( pInputBuffer == null || inputLength <= 0 )
486
478
{
@@ -510,7 +502,6 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
510
502
511
503
if ( processedLength + 16 < inputLength )
512
504
{
513
- // Console .WriteLine("-------SIMD kicking in-------");
514
505
Vector128 < byte > prevInputBlock = Vector128 < byte > . Zero ;
515
506
516
507
Vector128 < byte > maxValue = Vector128 . Create (
@@ -591,23 +582,17 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
591
582
int n4 = 0 ; // number of 4-byte sequences that start in this block
592
583
for ( ; processedLength + 16 <= inputLength ; processedLength += 16 )
593
584
{
594
- // Console .WriteLine($"Processing {processedLength} bytes", processedLength);
595
585
596
586
Vector128 < byte > currentBlock = Avx . LoadVector128 ( pInputBuffer + processedLength ) ;
597
587
int mask = Sse42 . MoveMask ( currentBlock ) ;
598
- // ToString (currentBlock);
599
- // // Console .WriteLine($"{mask}");
600
588
if ( mask == 0 )
601
589
{
602
590
// We have an ASCII block, no need to process it, but
603
591
// we need to check if the previous block was incomplete.
604
592
//
605
593
606
- // Console .WriteLine("Found all ASCII");
607
- // ToString(prevIncomplete);
608
594
if ( ! Sse41 . TestZ ( prevIncomplete , prevIncomplete ) )
609
595
{
610
- // // Console .WriteLine($"Found all-ascii but previous is incomplete.Triggering Rewind");
611
596
int off = processedLength >= 3 ? processedLength - 3 : processedLength ;
612
597
byte * invalidBytePointer = SimdUnicode . UTF8 . SimpleRewindAndValidateWithErrors ( 16 - 3 , pInputBuffer + processedLength - 3 , inputLength - processedLength + 3 ) ;
613
598
// So the code is correct up to invalidBytePointer
@@ -627,7 +612,6 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
627
612
}
628
613
else // Contains non-ASCII characters, we need to do non-trivial processing
629
614
{
630
- // Console .WriteLine("Contains ASCII characters, firing standard SIMD routine");
631
615
// Use SubtractSaturate to effectively compare if bytes in block are greater than markers.
632
616
// Contains non-ASCII characters, we need to do non-trivial processing
633
617
Vector128 < byte > prev1 = Ssse3 . AlignRight ( currentBlock , prevInputBlock , ( byte ) ( 16 - 1 ) ) ;
@@ -638,8 +622,6 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
638
622
Vector128 < byte > prev2 = Ssse3 . AlignRight ( currentBlock , prevInputBlock , ( byte ) ( 16 - 2 ) ) ;
639
623
Vector128 < byte > prev3 = Ssse3 . AlignRight ( currentBlock , prevInputBlock , ( byte ) ( 16 - 3 ) ) ;
640
624
prevInputBlock = currentBlock ;
641
- // // Console .WriteLine("PrevIncomplete:");
642
- // ToString(prevIncomplete);
643
625
644
626
Vector128 < byte > isThirdByte = Sse2 . SubtractSaturate ( prev2 , thirdByte ) ;
645
627
Vector128 < byte > isFourthByte = Sse2 . SubtractSaturate ( prev3 , fourthByte ) ;
@@ -649,7 +631,6 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
649
631
650
632
if ( ! Sse42 . TestZ ( error , error ) )
651
633
{
652
- // Console .WriteLine($"Found error! rewinding...");
653
634
654
635
byte * invalidBytePointer ;
655
636
if ( processedLength == 0 )
@@ -686,7 +667,6 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust
686
667
asciibytes += ( int ) ( 16 - Popcnt . PopCount ( ( uint ) mask ) ) ;
687
668
}
688
669
689
- // Console .WriteLine($"-----SIMD finished , calling scalar:");
690
670
691
671
// We may still have an error.
692
672
if ( processedLength < inputLength || ! Sse42 . TestZ ( prevIncomplete , prevIncomplete ) )
0 commit comments