feat: auto detect CPU core numbers

howl-anderson · Mar 9, 2020 · 3db4160 · 3db4160
1 parent 0c2db17
commit 3db4160
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 4 deletions.
diff --git a/compute_brown_cluster.bash b/compute_brown_cluster.bash
@@ -1,3 +1,6 @@
 #!/bin/bash
 
-third-part/brown-cluster/wcluster --text WORDS.txt --c 1000 --threads 39
+cpu_count=`nproc --all`
+process_count=$(expr $cpu_count - 1)
+
+./third-part/brown-cluster/wcluster --text WORDS.txt --c 1000 --threads ${process_count}
diff --git a/compute_plain_word_vec.bash b/compute_plain_word_vec.bash
@@ -1,4 +1,6 @@
 #!/bin/bash
 
-# TODO: process number shoud auto compute
-python ./spacy-dev-resources/training/plain_word_vectors.py -i 200 -n 6 ../chinese-wikipedia-corpus-creator/token_cleaned_plain_files WORDS_VECS.txt
+cpu_count=`nproc --all`
+process_count=$(expr $cpu_count - 1)
+
+python ./spacy-dev-resources/training/plain_word_vectors.py -i 200 -n ${process_count} ../chinese-wikipedia-corpus-creator/token_cleaned_plain_files WORDS_VECS.txt
diff --git a/compute_words_freq.bash b/compute_words_freq.bash
@@ -1,3 +1,6 @@
 #!/bin/bash
 
-python ./spacy-dev-resources/training/plain_word_freqs.py -n 8 token_cleaned_plain_files WORDS_FREQ.txt 
+cpu_count=`nproc --all`
+process_count=$(expr $cpu_count - 1)
+
+python ./spacy-dev-resources/training/plain_word_freqs.py -n ${process_count} token_cleaned_plain_files WORDS_FREQ.txt