Return values from CompareToSharpToken benchmark

georg-jung · Apr 29, 2024 · 7cbbd26 · 7cbbd26
1 parent 642a964
commit 7cbbd26
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/Benchmarks/CompareToSharpToken.cs b/src/Benchmarks/CompareToSharpToken.cs
@@ -56,7 +56,8 @@ public IReadOnlyCollection<object> SharpTokenFullArticles()
         List<object> res = new(_corpus.Length);
         foreach (var text in _corpus)
         {
-            _encoding.Encode(text);
+            var lst = _encoding.Encode(text);
+            res.Add(lst);
         }
 
         return res;
@@ -70,6 +71,7 @@ public IReadOnlyCollection<object> FastBertTokenizerFullArticles()
         foreach (var text in _corpus)
         {
             var (iid, attM, _) = _tokenizer.Encode(text, _maxSequenceLength);
+            res.Add((iid, attM));
         }
 
         return res;