adding unk_token removal for the Python testing

OpenNMT · Aug 14, 2024 · 065b240 · 065b240
1 parent e5a2a46
commit 065b240
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/python/tests/test_transformers.py b/python/tests/test_transformers.py
@@ -1020,5 +1020,6 @@ def test_transformers_wav2vec2(
 
         predicted_ids = torch.argmax(logits, dim=-1)
         transcription = processor.decode(predicted_ids, output_word_offsets=True)
+        transcription = transcription[0].replace(processor.tokenizer.unk_token,'')
 
-        assert transcription[0] == expected_transcription[0]
+        assert transcription == expected_transcription[0]