Fix net48 tests/DidWeEmitOneUnkWhereHuggingFaceJustSkippedSomething

georg-jung · Apr 29, 2024 · 642a964 · 642a964
1 parent 8a0a42e
commit 642a964
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/FastBertTokenizer.Tests/AsyncBatchEnumeratorVsHuggingface.cs b/src/FastBertTokenizer.Tests/AsyncBatchEnumeratorVsHuggingface.cs
@@ -343,6 +343,9 @@ private bool DidWeSkipSomethingWhereHuggingFaceEmittedUnk(ReadOnlySpan<long> hug
                 skippedHfUnk++;
                 iHF++;
             }
+
+            // The tokens were just different and that Hugging Face emitted [UNK] where we didn't was not the reason for that.
+            break;
         }
 
         return