-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathrun_fasttext.sh
46 lines (37 loc) · 2.59 KB
/
run_fasttext.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#!/bin/bash
# Train and test models with fastText
# $1: directory of the datasets
# $2: train/test suffix (empty or _scr)
echo "Training the model..." $1 $2
./fastText-0.9.2/fasttext supervised -input $1/train$2.txt -output models/KLID_model$2 -dim 64 -minn 2 -maxn 6 -loss hs -epoch 25 -lr 1.0
echo "Compressing..."
./fastText-0.9.2/fasttext quantize -input $1/train$2.txt -output models/KLID_model$2 -qnorm -cutoff 50000 -retrain
echo "Testing the model..."
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/test$2.txt
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/test$2.txt 2
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/test$2.txt 3
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/test$2.txt 4
if [[ $2!="_scr" ]]
then
echo "Testing with lid.176..."
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/test$2.txt
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/test$2.txt 2
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/test$2.txt 3
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/test$2.txt 4
fi
./fastText-0.9.2/fasttext predict models/KLID_model$2.bin $1/test$2.txt 4 > models/predict$2.txt
./fastText-0.9.2/fasttext predict fastText-0.9.2/lid.176.bin $1/test$2.txt 4 > models/predict_lid.176$2.txt
echo "Prediction done."
echo "Testing and predicting unconventional dataset with the trained model"
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/SouthernKurdish-arab_test_unconventional.txt
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/SouthernKurdish-arab_test_unconventional.txt 2
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/SouthernKurdish-arab_test_unconventional.txt 3
./fastText-0.9.2/fasttext test models/KLID_model$2.bin $1/SouthernKurdish-arab_test_unconventional.txt 4
echo "Testing and predicting unconventional dataset with lid.176"
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/SouthernKurdish-arab_test_unconventional.txt
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/SouthernKurdish-arab_test_unconventional.txt 2
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/SouthernKurdish-arab_test_unconventional.txt 3
./fastText-0.9.2/fasttext test fastText-0.9.2/lid.176.bin $1/SouthernKurdish-arab_test_unconventional.txt 4
./fastText-0.9.2/fasttext predict models/KLID_model$2.bin $1/SouthernKurdish-arab_test_unconventional.txt 4 > models/predict$2_sdh_unconventional.txt
./fastText-0.9.2/fasttext predict fastText-0.9.2/lid.176.bin $1/SouthernKurdish-arab_test_unconventional.txt 4 > models/predict_lid.176_sdh_unconventional.txt
echo "Done."