Merge pull request #772 from g-maxime/groundtruth-static

Use static version of mkv testing files

.github/workflows/MediaConch_Checks.yml

@@ -81,7 +81,7 @@ jobs:
       - name: Check files
         run: |
           cd Project/GNU/CLI
-          make clone_sample_files clone_checks_files
+          make clone_sample_files
       - name: Check
         run: |
           cd Project/GNU/CLI

Project/GNU/CLI/Makefile.am

@@ -63,14 +63,8 @@ TESTS = \
     test/test_ffv1.sh
 
 SAMPLES_DIR = test/SampleFiles
-CHECKS_DIR = test/ImplementationChecks
 
 clone_sample_files: $(SAMPLES_DIR)
 
-clone_checks_files: $(CHECKS_DIR)
-
 $(SAMPLES_DIR):
 	git clone https://github.com/MediaArea/MediaConch_SampleFiles test/SampleFiles
-
-$(CHECKS_DIR):
-	git clone https://github.com/MediaArea/groundtruth test/ImplementationChecks

Project/GNU/CLI/test/test_mk.sh

@@ -3,104 +3,58 @@
 PATH_SCRIPT=$(dirname "${0}")
 . "${PATH_SCRIPT}/utils.sh"
 
-FILES_DIRECTORY="${PATH_SCRIPT}/ImplementationChecks/matroska"
-
-BUILD_NUMBER="${TRAVIS_BUILD_NUMBER:-0}"
-
-RUN_REPORT="$(date +%Y%m%d)_MediaArea_${BUILD_NUMBER:-0}.txt"
-RUN_META="$(date +%Y%m%d)_MediaArea_${BUILD_NUMBER:-0}.meta.txt"
+FILES_DIRECTORY="${PATH_SCRIPT}/SampleFiles/Groundtruth"
 
 MC="${PWD}/mediaconch"
 
 RCODE=0
 
-pushd "${FILES_DIRECTORY}"
-
-# get and configure ffmpeg sources
-git clone --depth=1 https://github.com/FFmpeg/FFmpeg.git -b n5.1
-mv FFmpeg/* .
-rm -fr FFmpeg
-
-./configure --disable-doc --disable-x86asm --disable-ffplay --disable-ffprobe --enable-static --disable-bzlib --disable-libopenjpeg --disable-iconv --disable-zlib --disable-everything --enable-indev=lavfi --enable-filter=scale,color,testsrc2 --enable-decoder=rawvideo,ffv1 --enable-encoder=rawvideo,ffv1 --enable-demuxer=matroska --enable-muxer=matroska --enable-protocol=file
-
-for FILE in $(ls -v *.md) ; do
-    if ! echo "${FILE}" | grep -Eq '^[0-9]+\.md$' ; then
-        continue
-    fi
-
-    TEST="$(grep -Eom1 '^# [^[:space:]]+$' ${FILE} | grep -Eom1 '[^[:space:]]+$')"
-    SCRIPT="$(perl -0777lne 'print for grep !//,/^```sh\s+(.*?)^```/mgs' ${FILE})"
-
-    POLICY="$(perl -0777lne 'print for grep !//,/^```xml\s+(.*?)^```/mgs' ${FILE})"
-    if [ -n "${POLICY}" ] ; then
-        echo "${POLICY}" > "${TEST}.xml"
-    fi
-
-    echo "${SCRIPT}" > "${TEST}.sh"
-    sed -i'.bak' 's!make!make --assume-new libavcodec/ffv1enc.c --assume-new libavcodec/ffv1enc_template.c!g' "${TEST}.sh"
-    sh "${TEST}.sh"
-    rm "${TEST}.sh"
-
-    # clean intermediate files
-    rm -f "reference.xml" "reference.mkv"
-
-    for TESTFILE in ${TEST}*.mkv ; do
-        if [ ! -e "${TESTFILE}" ] ; then
+pushd "${FILES_DIRECTORY}" >/dev/null
+    for NAME in $(ls -v) ; do
+        if [ ! -d "${NAME}" ] ; then
             continue
         fi
 
-        DATA="$(${MC}${POLICY/$POLICY/ -p $TEST.xml} -fx ${TESTFILE})"
-
-        # test if the check fail as expected
-        if [ "${TEST}" == "Correct" ] ; then
-            # skip this test for correct files
-            T1="true"
-        elif [ -n "${POLICY}" ] ; then
-            # test against a policy
-            T1=`echo "${DATA}" | xmllint --xpath \
-            "boolean(//*/*[@name and @name=\"${TEST}\" and @fail_count and @fail_count!=\"0\"])" -`
-        else
-            T1=`echo "${DATA}" | xmllint --xpath \
-            "boolean(//*/*[@icid and @icid=\"${TEST}\" and @fail_count and @fail_count!=\"0\"])" -`
-        fi
-
-        # test if other check fail
-        if [ -n "${POLICY}" ] ; then
-            # skip this test for policies
-            T2="false"
-        else
-            T2=`echo "${DATA}" | xmllint --xpath \
-            "boolean(//*/*[@icid and @icid!=\"${TEST}\" and @fail_count and @fail_count!=\"0\"])" -`
-        fi
-
-        CLASSID="AVC$(printf '%03d' ${FILE%.*})"
-        COL2="Q0"
-        DOC_UID="${TESTFILE}"
-        COL4="0"
-
-        if [ "${T1}" != "true" ] || [ "${T2}" != "false" ] ; then
-            echo "NOK: ${TESTFILE}" >&9
-            CLASS_BELONG="0"
-            echo -e "${CLASSID}\t${COL2}\t${DOC_UID}\t${COL4}\t${CLASS_BELONG}\t${BUILD_NUMBER}" >> "${RUN_REPORT}"
-            rm -f "${TESTFILE}"
-            RCODE=1
-            continue
-        fi
-
-        rm -f "${TESTFILE}"
-        echo "OK: ${TESTFILE}" >&9
-        CLASS_BELONG="1"
-        echo -e "${CLASSID}\t${COL2}\t${DOC_UID}\t${COL4}\t${CLASS_BELONG}\t${BUILD_NUMBER}" >> "${RUN_REPORT}"
+        pushd "${NAME}" >/dev/null
+            OPTIONS=
+            if [ -f "policy.xml" ] ; then
+                OPTIONS=" -p policy.xml"
+            fi
+
+            for FILE in $(ls -v *.mkv) ; do
+                DATA="$(${MC}${OPTIONS} -fx ${FILE})"
+
+                # test if the check fail as expected
+                if [ "${NAME}" == "Correct" ] ; then
+                    # skip this test for correct files
+                    T1="true"
+                elif [ -f "policy.xml" ] ; then
+                    # test against a policy
+                    T1=`echo "${DATA}" | xmllint --xpath \
+                    "boolean(//*/*[@name and @name=\"${NAME}\" and @fail_count and @fail_count!=\"0\"])" -`
+                else
+                    T1=`echo "${DATA}" | xmllint --xpath \
+                    "boolean(//*/*[@icid and @icid=\"${NAME}\" and @fail_count and @fail_count!=\"0\"])" -`
+                fi
+
+                # test if other check fail
+                if [ -f "policy.xml" ] ; then
+                    # skip this test for policies
+                    T2="false"
+                else
+                    T2=`echo "${DATA}" | xmllint --xpath \
+                    "boolean(//*/*[@icid and @icid!=\"${NAME}\" and @fail_count and @fail_count!=\"0\"])" -`
+                fi
+
+                if [ "${T1}" != "true" ] || [ "${T2}" != "false" ] ; then
+                    echo "NOK: ${FILE}" >&9
+                    RCODE=1
+                    continue
+                fi
+                echo "OK: ${FILE}" >&9
+            done
+        popd >/dev/null
     done
-
-    rm -f "${TEST}.xml"
-
-done
-
-if [ -n "${TRAVIS_COMMIT_MESSAGE}" ] ; then
-    echo -e "${TRAVIS_COMMIT_MESSAGE}" >> "${RUN_META}"
-fi
-
-popd
+popd >/dev/null
 
 exit ${RCODE}