Actually fix net48 tests/DidWeEmitOneUnkWhereHuggingFaceJustSkippedS...

georg-jung · Apr 29, 2024 · 824effa · 824effa
1 parent 7cbbd26
commit 824effa
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/FastBertTokenizer.Tests/AsyncBatchEnumeratorVsHuggingface.cs b/src/FastBertTokenizer.Tests/AsyncBatchEnumeratorVsHuggingface.cs
@@ -332,16 +332,16 @@ private bool DidWeSkipSomethingWhereHuggingFaceEmittedUnk(ReadOnlySpan<long> hug
             // Hugging face will be at the end earlier if it emitted [UNK] where we skipped something.
             if (skippedHfUnk > 0 && huggF[iHF] == 102)
             {
-                iOurs++;
                 break;
             }
 
             // [UNK] == 100
             // Skip [UNK]s in Hugging Face's result
-            while (iHF < huggF.Length && huggF[iHF] == 100)
+            if (iHF < huggF.Length && huggF[iHF] == 100)
             {
                 skippedHfUnk++;
                 iHF++;
+                continue;
             }
 
             // The tokens were just different and that Hugging Face emitted [UNK] where we didn't was not the reason for that.