feat: write a regression test suite that can be sbatch submitted

Starting to address #611

everyvoice/tests/regression/.gitignore

@@ -0,0 +1,2 @@
+EV-regress.*
+regress-*

everyvoice/tests/regression/README.md

@@ -0,0 +1,26 @@
+# EveryVoice regression test suite
+
+## Preparing the regression training data:
+
+ - Download LJ 1.1 from https://keithito.com/LJ-Speech-Dataset/
+ - Download Sinhala TTS from https://openslr.org/30/
+ - Download High quality TTS data for four South African languages (af, st, tn,
+   xh) from https://openslr.org/32
+ - See [`prep-datasets.sh`](prep-datasets.sh) to see where these datasets are expected to be found.
+ - Run this to create the regression testing directory structure:
+
+       mkdir regress-1  # or any suffix you want
+       cd regress-1
+       ../prep-datasets.sh
+
+## Running the regression tests
+
+On a Slurm cluster:
+
+    for dir in regress-*; do
+        pushd $dir
+        sbatch ../../regression-test.sh
+        popd
+    done
+
+Or just use `../../regression-test.sh` directly in the loop if you're not on a cluster.

everyvoice/tests/regression/combine-coverage.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+find . -name .coverage\* | coverage combine --keep
+coverage report --include='*/everyvoice/*' | sed 's/.*EveryVoice\/everyvoice/everyvoice/' > coverage.txt
+coverage html --include='*/everyvoice/*'
+coverage xml --include='*/everyvoice/*'
+sed -i 's/"[^"]*EveryVoice.everyvoice/"everyvoice/g' coverage.xml

everyvoice/tests/regression/go.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+#SBATCH --job-name=EV-r-main
+#SBATCH --partition=standard
+#SBATCH --account=nrc_ict
+#SBATCH --qos=low
+#SBATCH --time=10080
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=8000M
+#SBATCH --output=./%x.o%j
+#SBATCH --error=./%x.e%j
+
+# Automated application of the instructions in README.md
+
+set -o errexit
+
+TOP_LEVEL_DIR=$(mktemp --directory regress-$(date +'%Y%m%d')-XXX)
+cd "$TOP_LEVEL_DIR"
+
+../prep-datasets.sh
+for DIR in regress-*; do
+    pushd "$DIR"
+    sbatch ../../regression-test.sh
+    popd
+done
+
+coverage run -p -m everyvoice test
+
+JOB_COUNT=$(find . -maxdepth 1 -name regress-\* | wc -l)
+while true; do
+    DONE_COUNT=$(find . -maxdepth 2 -name DONE | wc -l)
+    if (( DONE_COUNT + 2 >= JOB_COUNT )); then
+        break
+    fi
+    echo "$DONE_COUNT/$JOB_COUNT regression job(s) done. Still waiting."
+    date
+    sleep $(( 60 * 5 ))
+done
+
+echo "$DONE_COUNT regression jobs done. Calculating coverage now, but some jobs may still be running."
+../combine-coverage.sh
+cat coverage.txt
+
+while true; do
+    DONE_COUNT=$(find . -maxdepth 2 -name DONE | wc -l)
+    if (( DONE_COUNT >= JOB_COUNT )); then
+        break
+    fi
+    echo "$DONE_COUNT/$JOB_COUNT regression job(s) done. Still waiting."
+    date
+    sleep $(( 60 * 5 ))
+done
+
+echo "All $DONE_COUNT regression jobs done. Calculating final coverage."
+rm .coverage
+../combine-coverage.sh
+cat coverage.txt

everyvoice/tests/regression/prep-datasets.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Prepare the datasets and directories for our regression test cases
+
+set -o errexit
+
+# Usage: cat my_file | get_slice lines_to_keep > out
+# Use a number of lines or full to get all lines
+get_slice() {
+    lines=$1
+    if [[ $lines == full ]]; then
+        cat
+    else
+        head -"$lines"
+    fi
+}
+
+EVERYVOICE_REGRESS_ROOT=$(python -c 'import everyvoice; print(everyvoice.__path__[0])')/tests/regression
+
+LJ_SPEECH_DATASET=$HOME/sgile/data/LJSpeech-1.1
+LJSLICES="150 600 1600 full"
+for slice in $LJSLICES; do
+    dir=regress-lj-$slice
+    mkdir "$dir"
+    ln -s "$LJ_SPEECH_DATASET/wavs" "$dir"/
+    get_slice "$slice" < "$LJ_SPEECH_DATASET/metadata.csv" > "$dir"/metadata.csv
+    cp "$EVERYVOICE_REGRESS_ROOT"/wizard-resume-lj "$dir"/wizard-resume
+    cat <<'==EOF==' > "$dir"/test.txt
+This is a test.
+I am an anvil.
+I have no idea what to write here, but it has to be synthesizable text; so here is something!
+Boo!
+==EOF==
+    echo spec > "$dir"/test2.txt
+done
+
+SinhalaTTS=$HOME/sgile/data/SinhalaTTS
+dir=regress-si
+mkdir $dir
+ln -s "$SinhalaTTS/wavs" $dir/
+cp "$SinhalaTTS/si_lk.lines.txt" $dir/
+cp "$EVERYVOICE_REGRESS_ROOT"/wizard-resume-si "$dir"/wizard-resume
+# Source of this sample text: https://en.wikipedia.org/wiki/Sinhala_script CC BY-SA-4.0
+#  - the first line means Sinhala script, found at the top of the page
+#  - the rest is the first verse from the Pali Dhammapada lower on the same page
+cat <<'==EOF==' > "$dir"/test.txt
+සිංහල අක්ෂර මාලාව
+මනොපුබ්‌බඞ්‌ගමා ධම්‌මා, මනොසෙට්‌ඨා මනොමයා;
+මනසා චෙ පදුට්‌ඨෙන, භාසති වා කරොති වා;
+තතො නං දුක්‌ඛමන්‌වෙති, චක්‌කංව වහතො පදං.
+==EOF==
+echo "අක-ෂර" > "$dir"/test2.txt
+
+isiXhosa=$HOME/sgile/data/OpenSLR32-four-South-Afican-languages/xh_za/za/xho
+dir=regress-xh
+mkdir $dir
+ln -s "$isiXhosa/wavs" $dir/
+cp "$isiXhosa/line_index.tsv" $dir/
+cp "$EVERYVOICE_REGRESS_ROOT"/wizard-resume-xh "$dir"/wizard-resume
+# Source of this sample text: individual words copied from
+# https://en.wikipedia.org/wiki/Xhosa_language CC BY-SA-4.0
+cat <<'==EOF==' > "$dir"/test.txt
+ukukrwentshwa
+uqeqesho
+iimpumlo
+==EOF==
+echo isiXhosa > "$dir"/test2.txt

everyvoice/tests/regression/regression-test.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+#SBATCH --job-name=EV-regress
+#SBATCH --partition=gpu_a100
+#SBATCH --account=nrc_ict__gpu_a100
+#SBATCH --qos=low
+#SBATCH --time=180
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=16000M
+#SBATCH --gres=gpu:1
+#SBATCH --output=./%x.o%j
+#SBATCH --error=./%x.e%j
+
+# User env config -- set ACTIVATE_SCRIPT to point to something that will activate the
+# right Python environment, or leave it empty if you don't need it.
+ACTIVATE_SCRIPT=$HOME/start_ev.sh
+
+# Run a command, logging it first
+r() {
+    cmd="$*"
+    printf "\n\n======================================================================\n"
+    printf 'Running "%s"\n' "$cmd"
+    date
+    printf "======================================================================\n"
+    eval "$cmd" 2>&1
+    rc=$?
+    if [[ $rc != 0 ]]; then
+        printf "\n\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
+        echo "Command \"$cmd\" exited with non-zero return code $rc."
+        date
+        printf "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
+    fi
+    return $rc
+}
+
+echo "Start at $(date)"
+date > START
+
+trap 'echo "Failed or killed at $(date)"; date > FAILED' 0
+
+# Regression config
+[[ -e "$ACTIVATE_SCRIPT" ]] && source "$ACTIVATE_SCRIPT"
+export TQDM_MININTERVAL=5
+MAX_STEPS=1000
+# For a production config, use MAX_STEPS=100000 and increase the SBATCH --time above
+
+# Run the new-project wizard
+r "coverage run -p -m everyvoice new-project --resume-from wizard-resume"
+
+# Enter the directory created by the wizard
+cd regress || { echo "Cannot cd into regress directory, aborting."; date > DONE ; exit 1; }
+trap 'echo "Failed or killed at $(date)"; date > ../FAILED' 0
+
+# Preprocess
+r "coverage run -p -m everyvoice preprocess config/everyvoice-text-to-spec.yaml"
+
+# Train the fs2 model
+r "coverage run -p -m everyvoice train text-to-spec config/everyvoice-text-to-spec.yaml --config-args training.max_steps=$MAX_STEPS"
+FS2=logs_and_checkpoints/FeaturePredictionExperiment/base/checkpoints/last.ckpt
+ls $FS2
+
+# Train the vocoder
+r "coverage run -p -m everyvoice train spec-to-wav config/everyvoice-spec-to-wav.yaml --config-args training.max_steps=$MAX_STEPS"
+VOCODER=logs_and_checkpoints/VocoderExperiment/base/checkpoints/last.ckpt
+ls $VOCODER
+
+# Synthesize some text
+r "coverage run -p -m everyvoice synthesize from-text \
+    --output-type wav --output-type spec --output-type textgrid --output-type readalong-xml --output-type readalong-html \
+    --filelist ../test.txt \
+    --vocoder-path '$VOCODER' \
+    '$FS2'"
+# TODO: check the synthesized files, somehow
+
+# Exercise two-step synthesis
+ONE_WORD=$(cat ../test2.txt)
+r "coverage run -p -m everyvoice synthesize from-text --output-type spec --text '$ONE_WORD' '$FS2'"
+r "coverage run -p -m everyvoice synthesize from-spec \
+    --input synthesis_output/synthesized_spec/'$ONE_WORD'-*.pt \
+    --model '$VOCODER'"
+
+# Exercise DeepForceAligner
+# Meh, this appears to be broken...
+#r "coverage run -p -m dfaligner train config/everyvoice-aligner.yaml --config-args training.max_steps=$MAX_STEPS"
+#r "coverage run -p -m dfaligner extract-alignments"
+
+
+# 7: spin up the demo
+# everyvoice demo $FS2 $VOCODER &
+
+
+# 8: use playwright to synthesize something using the demo
+# TODO...
+
+echo "Done at $(date)"
+date > ../DONE
+trap - 0

everyvoice/tests/regression/wizard-resume-lj

@@ -0,0 +1,52 @@
+- - EveryVoice Wizard
+  - 0.2.0a1
+- - Root
+  - null
+- - Name Step
+  - regress
+- - Contact Name Step
+  - EveryVoice Regressor
+- - Contact Email Step
+  - [email protected]
+- - Output Path Step
+  - .
+- - Filelist Step
+  - metadata.csv
+- - Dataset Permission Step
+  - Yes, I do have permission to use this data.
+- - Filelist Format Step
+  - psv
+- - Filelist Has Header Line Step
+  - 'no'
+- - Basename Header Step
+  - 0
+- - Text Header Step
+  - 1
+- - Filelist Text Representation Step
+  - characters
+- - Text Processing Step
+  - - 0
+    - 1
+- - Data Has Speaker Step
+  - 'no'
+- - Know Speaker Step
+  - 'no'
+- - Data Has Language Step
+  - 'no'
+- - Select Language Step
+  - '[eng]: English'
+- - Wavs Dir Step
+  - wavs
+- - Validate Wavs Step
+  - OK
+- - Symbol-Set Step
+  - true
+- - SoX Effects Step
+  - - 0
+    - 1
+- - Dataset Name Step
+  - lj2k
+- - More Datasets Step
+  - 'no'
+- - Config Format Step
+  - 'yaml'

everyvoice/tests/regression/wizard-resume-si

@@ -0,0 +1,48 @@
+- - EveryVoice Wizard
+  - 0.2.0a1
+- - Root
+  - null
+- - Name Step
+  - regress
+- - Contact Name Step
+  - Regression Tester
+- - Contact Email Step
+  - [email protected]
+- - Output Path Step
+  - .
+- - Filelist Step
+  - si_lk.lines.txt
+- - Dataset Permission Step
+  - Yes, I do have permission to use this data.
+- - Filelist Format Step
+  - festival
+- - Filelist Text Representation Step
+  - characters
+- - Text Processing Step
+  - []
+- - Data Has Speaker Step
+  - 'no'
+- - Know Speaker Step
+  - 'yes'
+- - Add Speaker Step
+  - sinhala_speaker
+- - Data Has Language Step
+  - 'no'
+- - Select Language Step
+  - '[und]: my language isn''t here'
+- - Wavs Dir Step
+  - wavs
+- - Validate Wavs Step
+  - OK
+- - Symbol-Set Step
+  - true
+- - SoX Effects Step
+  - - 0
+    - 1
+    - 2
+- - Dataset Name Step
+  - sinhala-regress
+- - More Datasets Step
+  - 'no'
+- - Config Format Step
+  - 'yaml'