better tokenisation implementation

giellalt · Jan 23, 2025 · f83244d · f83244d
1 parent 54371c5
commit f83244d
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/devtools/cheap-tokeniser.bash b/devtools/cheap-tokeniser.bash
@@ -0,0 +1,9 @@
+#!/bin/bash
+# cheap tokeniser if we can't run full tokeniser on every CI commit
+# or all the other cases when you need all the tokens in a minute instead of an
+# hour.
+cat $@ |\
+    sed -e 's/[.,:!?;)"*]\+ / &/g'\
+        -e 's/[.,:!?:)"*]\+$/ &/' \
+        -e 's/ [(*"[]/& /g' |\
+    tr -s ' ' '\n'