WIP: Generate RISC-V DV tests

Signed-off-by: Eryk Szpotanski <[email protected]>

.github/scripts/riscv_dv_matrix_include.py

@@ -0,0 +1,20 @@
+from yaml import load, Loader
+from json import dumps
+from itertools import product
+
+RISCV_DV_TESTS = ['riscv_arithmetic_basic_test']
+
+
+if __name__ == "__main__":
+    with open('.github/workflows/test-riscv-dv.yml', 'rb') as fd:
+        run_tests = load(fd, Loader=Loader)['jobs']['run-tests']
+    matrix = run_tests['strategy']['matrix']
+    isses = matrix['iss']
+    coverages = matrix['coverage']
+    result = [{
+        "test": test,
+        "iss": iss,
+        "coverage": coverage,
+        "version": "pyflow",
+    } for test, iss, coverage in product(RISCV_DV_TESTS, isses, coverages)]
+    print(dumps(result))

.github/scripts/riscv_dv_parse_testlist.py

@@ -0,0 +1,29 @@
+import sys
+from json import dumps
+from yaml import load, Loader
+from typing import Generator
+
+RISCV_DV_HOME = "third_party/riscv-dv/"
+
+
+def parse_yaml(path: str) -> Generator[str, None, None]:
+    with open(path, 'rb') as fd:
+        tests = load(fd, Loader=Loader)
+    for test in tests:
+        if 'import' in test:
+            import_path = test['import'].split('/', 1)[1]
+            yield from parse_yaml(RISCV_DV_HOME + import_path)
+        elif 'test' in test:
+            yield test['test']
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 2:
+        testlist = parse_yaml(
+            RISCV_DV_HOME + f'target/{sys.argv[1]}/testlist.yaml')
+    else:
+        testlist = parse_yaml(RISCV_DV_HOME + 'yaml/base_testlist.yaml')
+    testlist = list(testlist)
+    # remove, will cause incomplete sim, need customized RTL
+    testlist.remove("riscv_csr_test")
+    print(dumps(testlist))

.github/workflows/ci.yml

@@ -8,51 +8,51 @@ on:
 
 jobs:
 
-  Build-Verilator:
-    name: Build-Verilator
-    uses: ./.github/workflows/build-verilator.yml
+  # Build-Verilator:
+  #   name: Build-Verilator
+  #   uses: ./.github/workflows/build-verilator.yml
 
-  Build-Spike:
-    name: Build-Spike
-    uses: ./.github/workflows/build-spike.yml
+  # Build-Spike:
+  #   name: Build-Spike
+  #   uses: ./.github/workflows/build-spike.yml
 
-  Test-Regression:
-    name: Test-Regression
-    needs: [Build-Verilator]
-    uses: ./.github/workflows/test-regression.yml
+  # Test-Regression:
+  #   name: Test-Regression
+  #   needs: [Build-Verilator]
+  #   uses: ./.github/workflows/test-regression.yml
 
-  Test-Verification:
-    name: Test-Verification
-    needs: [Build-Verilator]
-    uses: ./.github/workflows/test-verification.yml
+  # Test-Verification:
+  #   name: Test-Verification
+  #   needs: [Build-Verilator]
+  #   uses: ./.github/workflows/test-verification.yml
 
-  Test-Microarchitectural:
-    name: Test-Microarchitectural
-    needs: [Build-Verilator]
-    uses: ./.github/workflows/test-uarch.yml
+  # Test-Microarchitectural:
+  #   name: Test-Microarchitectural
+  #   needs: [Build-Verilator]
+  #   uses: ./.github/workflows/test-uarch.yml
 
   Test-RISCV-DV:
     name: Test-RISCV-DV
-    needs: [Build-Verilator, Build-Spike]
+    # needs: [Build-Verilator, Build-Spike]
     uses: ./.github/workflows/test-riscv-dv.yml
 
-  Test-RISCOF:
-    name: Test-RISCOF
-    needs: [Build-Verilator, Build-Spike]
-    uses: ./.github/workflows/test-riscof.yml
-
-  Report-Coverage:
-    name: Report-Coverage
-    needs: [Test-Regression, Test-Verification, Test-RISCV-DV, Test-RISCOF]
-    uses: ./.github/workflows/report-coverage.yml
-
-  Publish-to-GH-Pages:
-    concurrency:
-      group: concurrency-group-${{ github.repository }}-publish
-      cancel-in-progress: false
-    permissions:
-      actions: write
-      contents: write
-    name: Publish-to-GH-Pages
-    needs: [Report-Coverage, Test-Verification]
-    uses: ./.github/workflows/publish-webpage.yml
+  # Test-RISCOF:
+  #   name: Test-RISCOF
+  #   needs: [Build-Verilator, Build-Spike]
+  #   uses: ./.github/workflows/test-riscof.yml
+
+  # Report-Coverage:
+  #   name: Report-Coverage
+  #   needs: [Test-Regression, Test-Verification, Test-RISCV-DV, Test-RISCOF]
+  #   uses: ./.github/workflows/report-coverage.yml
+
+  # Publish-to-GH-Pages:
+  #   concurrency:
+  #     group: concurrency-group-${{ github.repository }}-publish
+  #     cancel-in-progress: false
+  #   permissions:
+  #     actions: write
+  #     contents: write
+  #   name: Publish-to-GH-Pages
+  #   needs: [Report-Coverage, Test-Verification]
+  #   uses: ./.github/workflows/publish-webpage.yml

.github/workflows/test-riscv-dv.yml

@@ -104,26 +104,103 @@ jobs:
 #--------------#
 #    Tests
 #--------------#
-  tests:
+  generate-config:
+    runs-on: ubuntu-latest
+    outputs:
+      test-types: ${{ steps.test-types.outputs.tests }}
+      hash: ${{ steps.hash.outputs.files-hash }}
+      test-include: ${{ steps.test-types.outputs.include }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - id: test-types
+        run: |
+          python3 -m pip install pyyaml
+          echo "tests=$(python3 .github/scripts/riscv_dv_parse_testlist.py rv32imc)" | tee -a $GITHUB_OUTPUT
+          echo "include=$(python3 .github/scripts/riscv_dv_matrix_include.py)" | tee -a $GITHUB_OUTPUT
+      - id: hash
+        run: |
+          echo "files-hash=$(sha256sum **/*.sv **/*.py **/*.yml **/*.yaml | cut -d\  -f1 | sha256sum | cut -d\  -f1)" | tee -a $GITHUB_OUTPUT
+
+  generate-code:
+    runs-on: [ self-hosted, Linux, X64, gcp-custom-runners ]
+    container: centos:8
+    needs: generate-config
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJSON(needs.generate-config.outputs.test-types) }}
+        version: [ uvm ]
+        include: 
+          - test: riscv_arithmetic_basic_test
+            version: pyflow
+    env:
+      GHA_EXTERNAL_DISK: additional-tools
+      CACHE_HASH: ${{ needs.generate-config.outputs.hash }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Cache Metadata
+        id: cache_metadata
+        run: |
+          cache_code=cache_${{ matrix.test }}_${{ matrix.version }}
+          echo "cache_code=${cache_code}_${{ env.CACHE_HASH }}" | tee -a "$GITHUB_ENV"
+
+      - name: Cache Code
+        uses: actions/cache@v3
+        id: cache-code
+        timeout-minutes: 60
+        with:
+          path: tools/riscv-dv/work/test_${{ matrix.test }}/asm_test
+          key: ${{ env.cache_code }}
+
+      - name: Prepare Environment
+        if: steps.cache-code.outputs.cache-hit != 'true'
+        run: _secret_prepare_env
+
+      - name: Generate Tests
+        if: steps.cache-code.outputs.cache-hit != 'true' && matrix.version == 'uvm'
+        run: _secret_generate_code
+        env:
+          RISCV_DV_TEST: ${{ matrix.test }}
+
+      - name: Generate code
+        if: steps.cache-code.outputs.cache-hit != 'true' && matrix.version == 'pyflow'
+        run: |
+          export RV_ROOT=`realpath .`
+          pushd tools/riscv-dv
+            make -j`nproc` \
+              RISCV_DV_TEST=${{ matrix.test }} \
+              RISCV_DV_ITER=3 \
+              RISCV_DV_SEED=999 \
+              generate
+          popd
+
+
+  run-tests:
     name: Run RISC-V DV tests
     runs-on: ubuntu-latest
-    needs: [veer-iss, renode]
+    needs: [ veer-iss, renode, generate-config, generate-code ]
     strategy:
       fail-fast: false
       matrix:
-        test:
-          - riscv_arithmetic_basic_test
+        test: ${{ fromJSON(needs.generate-config.outputs.test-types) }}
         iss:
           - spike
           - whisper
           - renode
         coverage: ["branch", "toggle"] #TODO: add functional coverage
-
+        version: [ uvm ]
+        include: ${{ fromJSON(needs.generate-config.outputs.test-include) }}
     env:
       DEBIAN_FRONTEND: "noninteractive"
       CCACHE_DIR: "/opt/riscv-dv/.cache/"
       VERILATOR_VERSION: v5.010
       SPIKE_VERSION: d70ea67d
+      CACHE_HASH: ${{ needs.generate-config.outputs.hash }}
 
     steps:
       - name: Install utils
@@ -144,6 +221,7 @@ jobs:
           cache_spike_key=${cache_spike_restore_key}${{ env.SPIKE_VERSION }}
           cache_test_restore_key=${{ matrix.test }}_${{ matrix.coverage }}_
           cache_test_key=${cache_test_restore_key}${time}
+          cache_code=cache_${{ matrix.test }}_${{ matrix.version }}
 
           echo "date=$date" | tee -a "$GITHUB_ENV"
           echo "time=$time" | tee -a "$GITHUB_ENV"
@@ -153,6 +231,7 @@ jobs:
           echo "cache_spike_key=$cache_spike_key" | tee -a "$GITHUB_ENV"
           echo "cache_test_restore_key=$cache_test_restore_key" | tee -a "$GITHUB_ENV"
           echo "cache_test_key=$cache_test_key" | tee -a "$GITHUB_ENV"
+          echo "cache_code=${cache_code}_${{ env.CACHE_HASH }}" | tee -a "$GITHUB_ENV"
 
       - name: Restore verilator cache
         id: cache-verilator-restore
@@ -217,32 +296,28 @@ jobs:
           key: cache_tests_${{ steps.cache_timestamp.outputs.time }}
           restore-keys: cache_tests_
 
-      - name: Generate code
-        run: |
-          export RV_ROOT=`realpath .`
-
-          pushd tools/riscv-dv
-            make -j`nproc` \
-              RISCV_DV_TEST=${{ matrix.test }} \
-              RISCV_DV_ISS=${{ matrix.iss }} \
-              RISCV_DV_ITER=3 \
-              RISCV_DV_SEED=999 \
-              generate
-          popd
+      - name: Cache Code Restore
+        uses: actions/cache/restore@v3
+        id: cache-code-restore
+        timeout-minutes: 60
+        with:
+          path: tools/riscv-dv/work/test_${{ matrix.test }}/asm_test
+          key: ${{ env.cache_code }}
 
       - name: Run test
         run: |
           export PATH=/opt/verilator/bin:$PATH
           export PATH=/opt/veer-iss:$PATH
+          export RV_ROOT=`realpath .`
           export RISCV_GCC=riscv64-unknown-elf-gcc
           export RISCV_OBJCOPY=riscv64-unknown-elf-objcopy
           export RISCV_NM=riscv64-unknown-elf-nm
           export SPIKE_PATH=/opt/spike/bin
           export WHISPER_ISS=/opt/veer-iss/whisper
           export RENODE_PATH=/opt/renode
 
-          echo "PATH=${PATH}"       >> ${GITHUB_ENV}
           echo "RV_ROOT=${RV_ROOT}" >> ${GITHUB_ENV}
+          echo "PATH=${PATH}"       >> ${GITHUB_ENV}
 
           ${RISCV_GCC} --version
 
@@ -275,5 +350,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: artifacts-${{ matrix.test }}-${{ matrix.iss }}
+          name: artifacts-${{ matrix.test }}-${{ matrix.iss }}-${{ matrix.version }}
           path: tools/riscv-dv/work/test_*

tools/riscv-dv/Makefile

@@ -149,7 +149,7 @@ run:
 	# Run RISC-V DV compilation and simulation
 	$(MAKE) -f $(MAKEFILE) $(TEST_DIR)/iss_sim.log
 	# Run HDL simulation(s) and trace comparison
-	find $(TEST_DIR)/ -name "sim_*.log" | sed 's/sim_/comp_/g' | xargs $(MAKE) -f $(MAKEFILE)
+	find $(TEST_DIR)/$(RISCV_DV_ISS)_sim -name "*.log" | sed 's|sim/|sim/../comp_|g' | xargs realpath --relative-to=$(PWD) | xargs $(MAKE) -f $(MAKEFILE)
 	# Check for errors
 	for F in $(TEST_DIR)/comp_*.log; do grep "\[PASSED\]" $$F; if [ $$? -ne 0 ]; then exit 255; fi; done
 	# Aggregate coverage data