From 6d806f5766452f3ea1113503895be823e2e111f1 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Wed, 20 Sep 2023 12:12:42 +0200 Subject: [PATCH] WIP: Generate RISC-V DV tests Signed-off-by: Eryk Szpotanski --- .github/scripts/riscv_dv_matrix_include.py | 20 ++++ .github/scripts/riscv_dv_parse_testlist.py | 29 ++++++ .github/workflows/ci.yml | 78 +++++++------- .github/workflows/test-riscv-dv.yml | 113 +++++++++++++++++---- tools/riscv-dv/Makefile | 2 +- 5 files changed, 183 insertions(+), 59 deletions(-) create mode 100644 .github/scripts/riscv_dv_matrix_include.py create mode 100644 .github/scripts/riscv_dv_parse_testlist.py diff --git a/.github/scripts/riscv_dv_matrix_include.py b/.github/scripts/riscv_dv_matrix_include.py new file mode 100644 index 00000000000..03658dc40c7 --- /dev/null +++ b/.github/scripts/riscv_dv_matrix_include.py @@ -0,0 +1,20 @@ +from yaml import load, Loader +from json import dumps +from itertools import product + +RISCV_DV_TESTS = ['riscv_arithmetic_basic_test'] + + +if __name__ == "__main__": + with open('.github/workflows/test-riscv-dv.yml', 'rb') as fd: + run_tests = load(fd, Loader=Loader)['jobs']['run-tests'] + matrix = run_tests['strategy']['matrix'] + isses = matrix['iss'] + coverages = matrix['coverage'] + result = [{ + "test": test, + "iss": iss, + "coverage": coverage, + "version": "pyflow", + } for test, iss, coverage in product(RISCV_DV_TESTS, isses, coverages)] + print(dumps(result)) diff --git a/.github/scripts/riscv_dv_parse_testlist.py b/.github/scripts/riscv_dv_parse_testlist.py new file mode 100644 index 00000000000..3f2ae2ae24f --- /dev/null +++ b/.github/scripts/riscv_dv_parse_testlist.py @@ -0,0 +1,29 @@ +import sys +from json import dumps +from yaml import load, Loader +from typing import Generator + +RISCV_DV_HOME = "third_party/riscv-dv/" + + +def parse_yaml(path: str) -> Generator[str, None, None]: + with open(path, 'rb') as fd: + tests = load(fd, Loader=Loader) + for test in tests: + if 'import' in test: + import_path = test['import'].split('/', 1)[1] + yield from parse_yaml(RISCV_DV_HOME + import_path) + elif 'test' in test: + yield test['test'] + + +if __name__ == "__main__": + if len(sys.argv) == 2: + testlist = parse_yaml( + RISCV_DV_HOME + f'target/{sys.argv[1]}/testlist.yaml') + else: + testlist = parse_yaml(RISCV_DV_HOME + 'yaml/base_testlist.yaml') + testlist = list(testlist) + # remove, will cause incomplete sim, need customized RTL + testlist.remove("riscv_csr_test") + print(dumps(testlist)) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1d8b7c40a7..bc4ee17e688 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,51 +8,51 @@ on: jobs: - Build-Verilator: - name: Build-Verilator - uses: ./.github/workflows/build-verilator.yml + # Build-Verilator: + # name: Build-Verilator + # uses: ./.github/workflows/build-verilator.yml - Build-Spike: - name: Build-Spike - uses: ./.github/workflows/build-spike.yml + # Build-Spike: + # name: Build-Spike + # uses: ./.github/workflows/build-spike.yml - Test-Regression: - name: Test-Regression - needs: [Build-Verilator] - uses: ./.github/workflows/test-regression.yml + # Test-Regression: + # name: Test-Regression + # needs: [Build-Verilator] + # uses: ./.github/workflows/test-regression.yml - Test-Verification: - name: Test-Verification - needs: [Build-Verilator] - uses: ./.github/workflows/test-verification.yml + # Test-Verification: + # name: Test-Verification + # needs: [Build-Verilator] + # uses: ./.github/workflows/test-verification.yml - Test-Microarchitectural: - name: Test-Microarchitectural - needs: [Build-Verilator] - uses: ./.github/workflows/test-uarch.yml + # Test-Microarchitectural: + # name: Test-Microarchitectural + # needs: [Build-Verilator] + # uses: ./.github/workflows/test-uarch.yml Test-RISCV-DV: name: Test-RISCV-DV - needs: [Build-Verilator, Build-Spike] + # needs: [Build-Verilator, Build-Spike] uses: ./.github/workflows/test-riscv-dv.yml - Test-RISCOF: - name: Test-RISCOF - needs: [Build-Verilator, Build-Spike] - uses: ./.github/workflows/test-riscof.yml - - Report-Coverage: - name: Report-Coverage - needs: [Test-Regression, Test-Verification, Test-RISCV-DV, Test-RISCOF] - uses: ./.github/workflows/report-coverage.yml - - Publish-to-GH-Pages: - concurrency: - group: concurrency-group-${{ github.repository }}-publish - cancel-in-progress: false - permissions: - actions: write - contents: write - name: Publish-to-GH-Pages - needs: [Report-Coverage, Test-Verification] - uses: ./.github/workflows/publish-webpage.yml + # Test-RISCOF: + # name: Test-RISCOF + # needs: [Build-Verilator, Build-Spike] + # uses: ./.github/workflows/test-riscof.yml + + # Report-Coverage: + # name: Report-Coverage + # needs: [Test-Regression, Test-Verification, Test-RISCV-DV, Test-RISCOF] + # uses: ./.github/workflows/report-coverage.yml + + # Publish-to-GH-Pages: + # concurrency: + # group: concurrency-group-${{ github.repository }}-publish + # cancel-in-progress: false + # permissions: + # actions: write + # contents: write + # name: Publish-to-GH-Pages + # needs: [Report-Coverage, Test-Verification] + # uses: ./.github/workflows/publish-webpage.yml diff --git a/.github/workflows/test-riscv-dv.yml b/.github/workflows/test-riscv-dv.yml index 947dc31e81c..444dcceb03b 100644 --- a/.github/workflows/test-riscv-dv.yml +++ b/.github/workflows/test-riscv-dv.yml @@ -104,26 +104,103 @@ jobs: #--------------# # Tests #--------------# - tests: + generate-config: + runs-on: ubuntu-latest + outputs: + test-types: ${{ steps.test-types.outputs.tests }} + hash: ${{ steps.hash.outputs.files-hash }} + test-include: ${{ steps.test-types.outputs.include }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - id: test-types + run: | + python3 -m pip install pyyaml + echo "tests=$(python3 .github/scripts/riscv_dv_parse_testlist.py rv32imc)" | tee -a $GITHUB_OUTPUT + echo "include=$(python3 .github/scripts/riscv_dv_matrix_include.py)" | tee -a $GITHUB_OUTPUT + - id: hash + run: | + echo "files-hash=$(sha256sum **/*.sv **/*.py **/*.yml **/*.yaml | cut -d\ -f1 | sha256sum | cut -d\ -f1)" | tee -a $GITHUB_OUTPUT + + generate-code: + runs-on: [ self-hosted, Linux, X64, gcp-custom-runners ] + container: centos:8 + needs: generate-config + strategy: + fail-fast: false + matrix: + test: ${{ fromJSON(needs.generate-config.outputs.test-types) }} + version: [ uvm ] + include: + - test: riscv_arithmetic_basic_test + version: pyflow + env: + GHA_EXTERNAL_DISK: additional-tools + CACHE_HASH: ${{ needs.generate-config.outputs.hash }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Cache Metadata + id: cache_metadata + run: | + cache_code=cache_${{ matrix.test }}_${{ matrix.version }} + echo "cache_code=${cache_code}_${{ env.CACHE_HASH }}" | tee -a "$GITHUB_ENV" + + - name: Cache Code + uses: actions/cache@v3 + id: cache-code + timeout-minutes: 60 + with: + path: tools/riscv-dv/work/test_${{ matrix.test }}/asm_test + key: ${{ env.cache_code }} + + - name: Prepare Environment + if: steps.cache-code.outputs.cache-hit != 'true' + run: _secret_prepare_env + + - name: Generate Tests + if: steps.cache-code.outputs.cache-hit != 'true' && matrix.version == 'uvm' + run: _secret_generate_code + env: + RISCV_DV_TEST: ${{ matrix.test }} + + - name: Generate code + if: steps.cache-code.outputs.cache-hit != 'true' && matrix.version == 'pyflow' + run: | + export RV_ROOT=`realpath .` + pushd tools/riscv-dv + make -j`nproc` \ + RISCV_DV_TEST=${{ matrix.test }} \ + RISCV_DV_ITER=3 \ + RISCV_DV_SEED=999 \ + generate + popd + + + run-tests: name: Run RISC-V DV tests runs-on: ubuntu-latest - needs: [veer-iss, renode] + needs: [ veer-iss, renode, generate-config, generate-code ] strategy: fail-fast: false matrix: - test: - - riscv_arithmetic_basic_test + test: ${{ fromJSON(needs.generate-config.outputs.test-types) }} iss: - spike - whisper - renode coverage: ["branch", "toggle"] #TODO: add functional coverage - + version: [ uvm ] + include: ${{ fromJSON(needs.generate-config.outputs.test-include) }} env: DEBIAN_FRONTEND: "noninteractive" CCACHE_DIR: "/opt/riscv-dv/.cache/" VERILATOR_VERSION: v5.010 SPIKE_VERSION: d70ea67d + CACHE_HASH: ${{ needs.generate-config.outputs.hash }} steps: - name: Install utils @@ -144,6 +221,7 @@ jobs: cache_spike_key=${cache_spike_restore_key}${{ env.SPIKE_VERSION }} cache_test_restore_key=${{ matrix.test }}_${{ matrix.coverage }}_ cache_test_key=${cache_test_restore_key}${time} + cache_code=cache_${{ matrix.test }}_${{ matrix.version }} echo "date=$date" | tee -a "$GITHUB_ENV" echo "time=$time" | tee -a "$GITHUB_ENV" @@ -153,6 +231,7 @@ jobs: echo "cache_spike_key=$cache_spike_key" | tee -a "$GITHUB_ENV" echo "cache_test_restore_key=$cache_test_restore_key" | tee -a "$GITHUB_ENV" echo "cache_test_key=$cache_test_key" | tee -a "$GITHUB_ENV" + echo "cache_code=${cache_code}_${{ env.CACHE_HASH }}" | tee -a "$GITHUB_ENV" - name: Restore verilator cache id: cache-verilator-restore @@ -217,23 +296,19 @@ jobs: key: cache_tests_${{ steps.cache_timestamp.outputs.time }} restore-keys: cache_tests_ - - name: Generate code - run: | - export RV_ROOT=`realpath .` - - pushd tools/riscv-dv - make -j`nproc` \ - RISCV_DV_TEST=${{ matrix.test }} \ - RISCV_DV_ISS=${{ matrix.iss }} \ - RISCV_DV_ITER=3 \ - RISCV_DV_SEED=999 \ - generate - popd + - name: Cache Code Restore + uses: actions/cache/restore@v3 + id: cache-code-restore + timeout-minutes: 60 + with: + path: tools/riscv-dv/work/test_${{ matrix.test }}/asm_test + key: ${{ env.cache_code }} - name: Run test run: | export PATH=/opt/verilator/bin:$PATH export PATH=/opt/veer-iss:$PATH + export RV_ROOT=`realpath .` export RISCV_GCC=riscv64-unknown-elf-gcc export RISCV_OBJCOPY=riscv64-unknown-elf-objcopy export RISCV_NM=riscv64-unknown-elf-nm @@ -241,8 +316,8 @@ jobs: export WHISPER_ISS=/opt/veer-iss/whisper export RENODE_PATH=/opt/renode - echo "PATH=${PATH}" >> ${GITHUB_ENV} echo "RV_ROOT=${RV_ROOT}" >> ${GITHUB_ENV} + echo "PATH=${PATH}" >> ${GITHUB_ENV} ${RISCV_GCC} --version @@ -275,5 +350,5 @@ jobs: if: always() uses: actions/upload-artifact@v3 with: - name: artifacts-${{ matrix.test }}-${{ matrix.iss }} + name: artifacts-${{ matrix.test }}-${{ matrix.iss }}-${{ matrix.version }} path: tools/riscv-dv/work/test_* diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile index fa516d7472a..51352fceec6 100644 --- a/tools/riscv-dv/Makefile +++ b/tools/riscv-dv/Makefile @@ -149,7 +149,7 @@ run: # Run RISC-V DV compilation and simulation $(MAKE) -f $(MAKEFILE) $(TEST_DIR)/iss_sim.log # Run HDL simulation(s) and trace comparison - find $(TEST_DIR)/ -name "sim_*.log" | sed 's/sim_/comp_/g' | xargs $(MAKE) -f $(MAKEFILE) + find $(TEST_DIR)/$(RISCV_DV_ISS)_sim -name "*.log" | sed 's|sim/|sim/../comp_|g' | xargs realpath --relative-to=$(PWD) | xargs $(MAKE) -f $(MAKEFILE) # Check for errors for F in $(TEST_DIR)/comp_*.log; do grep "\[PASSED\]" $$F; if [ $$? -ne 0 ]; then exit 255; fi; done # Aggregate coverage data