Middleware and Firmware code coverage (#164)

- Added python coverage package
- Added middleware coverage report generation script
- Instrumented firmware unit tests and tcpsigner tests to run with optional coverage
- Added lcov to middleware docker image (used to also run firmware tests)
- Added firmware code coverage generation script
- TCPSigner now catches termination signals so that it can write gcda files when compiled with coverage
- Added code coverage github workflow that uploads coverage output to S3 and generates badges
- Added badges to readme

.github/workflows/coverage.yml

@@ -0,0 +1,59 @@
+name: "Code coverage"
+
+on:
+  push:
+    branches: [ "master" ]
+
+jobs:
+  coverage:
+    name: Run tests and generate coverage reports
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@v3
+
+      - name: Build the middleware docker image
+        run: docker/mware/build
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.CODECOVERAGE_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.CODECOVERAGE_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.CODECOVERAGE_AWS_REGION }}
+
+      - name: Run middleware coverage script
+        run: |
+          middleware/test-all-coverage
+          COVPCT=$(cat middleware/coverage/total)
+          COVCOL=$(utils/coverage-color.sh $COVPCT)
+          echo "{ \"schemaVersion\": 1, \"label\": \"Middleware coverage\", \"message\": \"$COVPCT%\", \"color\": \"$COVCOL\" }" > middleware/coverage/badge.json
+
+      - name: "Upload middleware coverage report"
+        run: |
+          aws s3 sync \
+            middleware/coverage/ \
+            s3://${{ secrets.CODECOVERAGE_S3_BUCKET }}/powhsm_4.1.x/middleware_coverage_report \
+            --sse aws:kms --sse-kms-key-id ${{ secrets.CODECOVERAGE_KMS_KEY_ID }} \
+            --no-progress --follow-symlinks --delete --only-show-errors
+
+      - name: Run firmware coverage script
+        run: |
+          ledger/coverage/gen-coverage
+          COVPCT=$(cat ledger/coverage/output/total)
+          COVCOL=$(utils/coverage-color.sh $COVPCT)
+          echo "{ \"schemaVersion\": 1, \"label\": \"Firmware coverage\", \"message\": \"$COVPCT%\", \"color\": \"$COVCOL\" }" > ledger/coverage/output/badge.json
+
+      - name: "Upload firmware coverage report"
+        run: |
+          aws s3 sync \
+            ledger/coverage/output/ \
+            s3://${{ secrets.CODECOVERAGE_S3_BUCKET }}/powhsm_4.1.x/firmware_coverage_report \
+            --sse aws:kms --sse-kms-key-id ${{ secrets.CODECOVERAGE_KMS_KEY_ID }} \
+            --no-progress --follow-symlinks --delete --only-show-errors
+
+      - name: Invalidate CloudFront cache
+        run: |
+          aws cloudfront create-invalidation \
+            --distribution-id ${{ secrets.CODECOVERAGE_CLOUDFRONT_DIST_ID }} --paths "/*"

README.md

@@ -3,6 +3,8 @@
 ![Tests](https://github.com/rsksmart/rsk-powhsm/actions/workflows/run-tests.yml/badge.svg)
 ![Python linter](https://github.com/rsksmart/rsk-powhsm/actions/workflows/lint-python.yml/badge.svg)
 ![C linter](https://github.com/rsksmart/rsk-powhsm/actions/workflows/lint-c.yml/badge.svg)
+[![Middleware coverage](https://img.shields.io/endpoint?url=https://d16sboe9lzo4ru.cloudfront.net/powhsm_4.1.x/middleware_coverage_report/badge.json)](https://d16sboe9lzo4ru.cloudfront.net/powhsm_4.1.x/middleware_coverage_report/index.html)
+[![Firmware coverage](https://img.shields.io/endpoint?url=https://d16sboe9lzo4ru.cloudfront.net/powhsm_4.1.x/firmware_coverage_report/badge.json)](https://d16sboe9lzo4ru.cloudfront.net/powhsm_4.1.x/firmware_coverage_report/index.html)
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE)
 

docker/mware/Dockerfile

@@ -5,7 +5,8 @@ WORKDIR /hsm2
 RUN apt-get update && \
     apt-get install -y apt-utils vim && \
     apt-get install -y build-essential=12.9 && \
-    apt-get install -y git
+    apt-get install -y git && \
+    apt-get install -y lcov
 
 # Python package prerequisites
 RUN apt-get install -y \

docker/mware/requirements.txt

@@ -353,3 +353,64 @@ yapf==0.31.0 \
 charset-normalizer==2.0.6 \
     --hash=sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6 \
     --hash=sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f
+coverage==7.2.7 \
+    --hash=sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f \
+    --hash=sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2 \
+    --hash=sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a \
+    --hash=sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a \
+    --hash=sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01 \
+    --hash=sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6 \
+    --hash=sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7 \
+    --hash=sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f \
+    --hash=sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02 \
+    --hash=sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c \
+    --hash=sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063 \
+    --hash=sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a \
+    --hash=sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5 \
+    --hash=sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959 \
+    --hash=sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97 \
+    --hash=sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6 \
+    --hash=sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f \
+    --hash=sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9 \
+    --hash=sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5 \
+    --hash=sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f \
+    --hash=sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562 \
+    --hash=sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe \
+    --hash=sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9 \
+    --hash=sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f \
+    --hash=sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb \
+    --hash=sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb \
+    --hash=sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1 \
+    --hash=sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb \
+    --hash=sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250 \
+    --hash=sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e \
+    --hash=sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511 \
+    --hash=sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5 \
+    --hash=sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59 \
+    --hash=sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2 \
+    --hash=sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d \
+    --hash=sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3 \
+    --hash=sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4 \
+    --hash=sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de \
+    --hash=sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9 \
+    --hash=sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833 \
+    --hash=sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0 \
+    --hash=sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9 \
+    --hash=sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d \
+    --hash=sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050 \
+    --hash=sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d \
+    --hash=sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6 \
+    --hash=sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353 \
+    --hash=sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb \
+    --hash=sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e \
+    --hash=sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8 \
+    --hash=sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495 \
+    --hash=sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2 \
+    --hash=sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd \
+    --hash=sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27 \
+    --hash=sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1 \
+    --hash=sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818 \
+    --hash=sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4 \
+    --hash=sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e \
+    --hash=sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850 \
+    --hash=sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3

ledger/coverage/.gitignore

@@ -0,0 +1,2 @@
+coverage.info
+output

ledger/coverage/coverage.mk

@@ -0,0 +1,6 @@
+ifeq ($(COVERAGE),y)
+	COVFLAGS = --coverage
+else
+	COVFLAGS = 
+endif
+COVFILES = *.gcda *.gcno

ledger/coverage/gen-coverage

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+if [[ $1 == "exec" ]]; then
+    BASEDIR=$(realpath $(dirname $0))
+    SRCDIR=$(realpath $BASEDIR/../src)
+    REPOROOT=$(realpath $BASEDIR/../..)
+
+    # Remove any existing coverage data
+    rm -rf $BASEDIR/coverage.info $BASEDIR/output
+    find $REPOROOT/ledger -name "*.gcno" -o -name "*.gcda" | xargs rm -f
+
+    # Run firmware unit tests with coverage generation
+    COVERAGE=y $REPOROOT/ledger/src/signer/test/run-all.sh
+    COVERAGE=y $REPOROOT/ledger/src/ui/test/run-all.sh
+
+    # Run tcpsigner test suite
+    pushd $REPOROOT/ledger/src/tcpsigner > /dev/null
+    COVERAGE=y make clean all
+    ./tcpsigner --checkpoint 0xbdcb3c17c7aee714cec8ad900341bfd987b452280220dcbd6e7191f67ea4209b --difficulty 0x32 --network regtest > /dev/null &
+    popd > /dev/null
+
+    pushd $REPOROOT/ledger/test > /dev/null
+    python run.py
+    err_code=$?
+    popd > /dev/null
+
+    lcov --capture --directory $SRCDIR --list-full-path --output-file $BASEDIR/coverage.info
+    genhtml $BASEDIR/coverage.info --output $BASEDIR/output -p $SRCDIR -t "powHSM firmware"
+    lcov --summary $BASEDIR/coverage.info | grep lines | sed -e "s/.\+lines.\+: \([[:digit:].]\+\).\+/\1/g" > $BASEDIR/output/total
+    mv $BASEDIR/coverage.info $BASEDIR/output
+else
+    # Script directory
+    REPOROOT=$(realpath $(dirname $0)/../..)
+    SCRIPT=$(realpath $0 --relative-to=$REPOROOT)
+
+    # Generate coverage report
+    $REPOROOT/docker/mware/do-notty-nousb /hsm2 "./$SCRIPT exec"
+    err_code=$?
+fi
+
+exit $err_code

ledger/src/.gitignore

@@ -13,3 +13,7 @@ tcpsigner/**/*.json
 # Icons hex
 signer/icon.hex
 ui/icon.hex
+
+# Code coverage artifacts
+*.gcda
+*.gcno

ledger/src/signer/test/btcscript/Makefile

@@ -25,19 +25,21 @@ SRCDIR = ../../src
 TCPSIGNERSRCDIR = ../../../tcpsigner
 CFLAGS = -I $(SRCDIR) -I $(TCPSIGNERSRCDIR) -I $(TESTCOMMONDIR)
 
+include ../../../../coverage/coverage.mk
+
 PROG = test.out
 OBJS = test_fwk.o btcscript.o hex_reader.o test_btcscript.o
 
 all: $(PROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $@ $^
+	$(CC) $(COVFLAGS) -o $@ $^
 
 test_fwk.o: $(TESTCOMMONDIR)/test_fwk.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 btcscript.o: $(SRCDIR)/btcscript.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 hex_reader.o: $(TCPSIGNERSRCDIR)/hex_reader.c
 	$(CC) $(CFLAGS) -c -o $@ $^
@@ -49,7 +51,7 @@ $(TCPSIGNERSRCDIR)/hex_reader.c: $(TCPSIGNERSRCDIR)/hex_reader.h
 
 .PHONY: clean test
 clean:
-	rm -f $(PROG) ./*.o
+	rm -f $(PROG) *.o $(COVFILES)
 
 test: all
 	./$(PROG)

ledger/src/signer/test/btctx/Makefile

@@ -26,22 +26,24 @@ TCPSIGNERSRCDIR = ../../../tcpsigner
 COMMONPATH = ../../../common/src
 CFLAGS = -I $(SRCDIR) -I $(TCPSIGNERSRCDIR) -I $(COMMONPATH) -I $(TESTCOMMONDIR)
 
+include ../../../../coverage/coverage.mk
+
 PROG = test.out
 OBJS = test_fwk.o btctx.o svarint.o hex_reader.o os.o test_btctx.o
 
 all: $(PROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $@ $^
+	$(CC) $(COVFLAGS) -o $@ $^
 
 test_fwk.o: $(TESTCOMMONDIR)/test_fwk.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 btctx.o: $(SRCDIR)/btctx.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 svarint.o: $(SRCDIR)/svarint.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 hex_reader.o: $(TCPSIGNERSRCDIR)/hex_reader.c
 	$(CC) $(CFLAGS) -c -o $@ $^
@@ -57,7 +59,7 @@ $(TCPSIGNERSRCDIR)/hex_reader.c: $(TCPSIGNERSRCDIR)/hex_reader.h
 
 .PHONY: clean test
 clean:
-	rm -f $(PROG) ./*.o
+	rm -f $(PROG) *.o $(COVFILES)
 
 test: all
 	./$(PROG)

ledger/src/signer/test/difficulty/Makefile

@@ -25,6 +25,8 @@ SRCDIR = ../../src
 COMMONPATH = ../../../common/src
 TCPSIGNERPATH = ../../../tcpsigner
 
+include ../../../../coverage/coverage.mk
+
 CFLAGS = -I $(SRCDIR) -I $(COMMONPATH) -I $(TCPSIGNERPATH) -I $(TESTCOMMONDIR) -DFEDHM_EMULATOR -DDEBUG_DIFF
 
 PROG = test.out
@@ -33,19 +35,19 @@ OBJS = test_fwk.o bigdigits.o bc_diff.o os.o test_difficulty.o
 all: $(PROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $@ $^
+	$(CC) $(COVFLAGS) -o $@ $^
 
 test_fwk.o: $(TESTCOMMONDIR)/test_fwk.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 bigdigits.o: $(SRCDIR)/bigdigits.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 os.o: $(TCPSIGNERPATH)/os.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 bc_diff.o: $(SRCDIR)/bc_diff.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 test_difficulty.o: test_difficulty.c test_fwk.o bigdigits.o bc_diff.o os.o
 
@@ -54,7 +56,7 @@ $(SRCDIR)/bc_diff.c: $(SRCDIR)/bc_diff.h $(SRCDIR)/bigdigits.h $(SRCDIR)/bigdtyp
 
 .PHONY: clean test
 clean:
-	rm -f $(PROG) ./*.o
+	rm -f $(PROG) *.o $(COVFILES)
 
 test: all
 	./$(PROG)

ledger/src/signer/test/sha256/Makefile

@@ -24,27 +24,29 @@ TESTCOMMONDIR = ../common
 SRCDIR = ../../src
 CFLAGS = -I $(SRCDIR) -I $(TESTCOMMONDIR)
 
+include ../../../../coverage/coverage.mk
+
 PROG = test.out
 OBJS = test_fwk.o sha256.o test_sha256.o
 
 all: $(PROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $@ $^
+	$(CC) $(COVFLAGS) -o $@ $^
 
 test_fwk.o: $(TESTCOMMONDIR)/test_fwk.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 sha256.o: $(SRCDIR)/sha256.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 test_sha256.o: test_sha256.c test_fwk.o sha256.o
 
 $(SRCDIR)/sha256.c: $(SRCDIR)/sha256.h
 
 .PHONY: clean test
 clean:
-	rm -f $(PROG) ./*.o
+	rm -f $(PROG) *.o $(COVFILES)
 
 test: all
 	./$(PROG)

ledger/src/signer/test/srlp/Makefile

@@ -24,27 +24,29 @@ TESTCOMMONDIR = ../common
 SRCDIR = ../../src
 CFLAGS = -I $(SRCDIR) -I $(TESTCOMMONDIR) -Imocks/ -DMAX_RLP_CTX_DEPTH=10
 
+include ../../../../coverage/coverage.mk
+
 PROG = test.out
 OBJS = test_fwk.o srlp.o test_srlp.o
 
 all: $(PROG)
 
 $(PROG): $(OBJS)
-	$(CC) -o $@ $^
+	$(CC) $(COVFLAGS) -o $@ $^
 
 test_fwk.o: $(TESTCOMMONDIR)/test_fwk.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 srlp.o: $(SRCDIR)/srlp.c
-	$(CC) $(CFLAGS) -c -o $@ $^
+	$(CC) $(CFLAGS) $(COVFLAGS) -c -o $@ $^
 
 test_srlp.o: test_srlp.c test_fwk.o srlp.o
 
 $(SRCDIR)/srlp.c: $(SRCDIR)/srlp.h
 
 .PHONY: clean test
 clean:
-	rm -f $(PROG) ./*.o
+	rm -f $(PROG) *.o $(COVFILES)
 
 test: all
 	./$(PROG)