slightly cleanup tokenizer.rs logic

WorksApplications · Mar 26, 2024 · e850158 · e850158
1 parent 0283406
commit e850158
Showing 1 changed file with 4 additions and 6 deletions.
diff --git a/python/src/tokenizer.rs b/python/src/tokenizer.rs
@@ -150,13 +150,11 @@ impl PyTokenizer {
             default_mode.map(|m| t.set_mode(m));
         });
 
-        // this needs to be in GIL as it references Python memory
-        tokenizer.reset().push_str(text);
         // analysis can be done without GIL
-        let err = {
-            let tokenizer = tokenizer.deref_mut();
-            py.allow_threads(|| tokenizer.do_tokenize())
-        };
+        let err = py.allow_threads(|| {
+            tokenizer.reset().push_str(text);
+            tokenizer.do_tokenize()
+        });
 
         err.map_err(|e| SudachiPyErr::new_err(format!("Tokenization error: {}", e.to_string())))?;