diff --git a/.github/Dockerfile.base b/.github/Dockerfile.base index 3bcf41cb5..a4712a5ee 100644 --- a/.github/Dockerfile.base +++ b/.github/Dockerfile.base @@ -33,7 +33,8 @@ RUN apt-get update && apt-get install -y \ jq \ curl \ gh \ - expect + expect \ + lcov # Install clang 17 RUN wget https://apt.llvm.org/llvm.sh && \ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6cbe1c92..d449fd2d7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -106,16 +106,110 @@ jobs: cd ../../.. cp third_party/tvm/python/dist/*.whl dist/ - - name: Run Unit Tests - shell: bash - run: | - source env/activate - pip install dist/forge*.whl --force-reinstall - cmake --build ${{ steps.strings.outputs.build-output-dir }} -- run_unit_tests - - name: Upload Build uses: actions/upload-artifact@v4 if: success() || failure() with: name: forge-wheel path: dist/*.whl + + + test-build: + strategy: + fail-fast: false + matrix: + build: + - runs-on: [runner] + + runs-on: + - in-service + - ${{ matrix.build.runs-on }} + + container: + image: ${{ inputs.docker-image }} + + steps: + + - name: Fetch job id + id: fetch-job-id + uses: tenstorrent/tt-github-actions/.github/actions/job_id@main + with: + job_name: "${{ github.job }}" + + - name: Set reusable strings + id: strings + shell: bash + env: + JOB_ID: ${{ steps.fetch-job-id.outputs.job_id }} + run: | + echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" + echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" + echo "install-output-dir=$(pwd)/install" >> "$GITHUB_OUTPUT" + echo "test_report_path=report_$JOB_ID.xml" >> "$GITHUB_OUTPUT" + + - name: Git safe dir + run: | + git config --system --add safe.directory ${{ steps.strings.outputs.work-dir }} + chown -R root:root ${{ steps.strings.outputs.work-dir }} + + - uses: actions/checkout@v4 + with: + submodules: recursive + clean: true + fetch-depth: 0 # Fetch all history and tags + + # Clean everything from submodules (needed to avoid issues + # with cmake generated files leftover from previous builds) + - name: Cleanup submodules + run: | + git submodule foreach --recursive git clean -ffdx + git submodule foreach --recursive git reset --hard + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: ${{ matrix.build.runs-on }}-runtime-${{ matrix.build.enable_runtime }} + + - name: Debug Build + shell: bash + run: | + source env/activate + cmake -G Ninja \ + -B ${{ steps.strings.outputs.build-output-dir }} \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCODE_COVERAGE=ON + cmake --build ${{ steps.strings.outputs.build-output-dir }} + + - name: Run Unit Tests + shell: bash + run: | + source env/activate + cmake --build ${{ steps.strings.outputs.build-output-dir }} -- run_unit_tests + + - name: Prepare code coverage report + run: | + lcov --directory build --capture --output-file coverage.info --gcov-tool ${{ steps.strings.outputs.work-dir }}/.github/workflows/gcov_for_clang.sh + lcov --extract coverage.info '**/tt-forge-fe/forge/csrc/*' --output-file coverage.info + sed -i 's|SF:/__w/tt-forge-fe/tt-forge-fe/forge/csrc/|SF:csrc/|' coverage.info + lcov --list coverage.info + + - name: Upload coverage reports to Codecov + if: success() || failure() + uses: codecov/codecov-action@v5 + with: + files: coverage.info + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload test results to Codecov + if: success() || failure() + uses: codecov/test-results-action@v1 + with: + files: ${{ steps.strings.outputs.test_report_path }} + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/gcov_for_clang.sh b/.github/workflows/gcov_for_clang.sh new file mode 100755 index 000000000..63f59bd5e --- /dev/null +++ b/.github/workflows/gcov_for_clang.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 + +exec llvm-cov-17 gcov "$@" diff --git a/.gitignore b/.gitignore index 9505ab379..142bc6ca0 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ generated infra/.coverage infra/pytest.xml pipeline_*.json + +coverage.info +coverage.xml diff --git a/CMakeLists.txt b/CMakeLists.txt index e2a355e06..405368350 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,22 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebIn add_compile_options(-DDEBUG) endif() +add_library(coverage_config INTERFACE) +option(CODE_COVERAGE "Enable coverage reporting" OFF) +if(CODE_COVERAGE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + # Add required flags (GCC & LLVM/Clang) + target_compile_options(coverage_config INTERFACE + -O0 # no optimization + -g # generate debug info + --coverage # sets all required flags + ) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) + target_link_options(coverage_config INTERFACE --coverage) + else() + target_link_libraries(coverage_config INTERFACE --coverage) + endif() +endif(CODE_COVERAGE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(TTFORGE_CSRC_WARNINGS -Wall -Wextra -Wno-pragmas -Wno-unused-parameter) set(CFLAGS_NO_WARN -DFMT_HEADER_ONLY) set(TTFORGE_CSRC_CFLAGS ${CFLAGS_NO_WARN} ${TTFORGE_CSRC_WARNINGS} -DUTILS_LOGGER_PYTHON_OSTREAM_REDIRECT=1) diff --git a/forge/csrc/CMakeLists.txt b/forge/csrc/CMakeLists.txt index 4d3f940a3..c47f697d2 100644 --- a/forge/csrc/CMakeLists.txt +++ b/forge/csrc/CMakeLists.txt @@ -94,6 +94,8 @@ target_link_libraries(ttforge_csrc PRIVATE ${TORCH_LIBRARIES} ) +target_link_libraries(ttforge_csrc PUBLIC coverage_config) + target_compile_options(ttforge_csrc PRIVATE ${TTFORGE_CSRC_CFLAGS} ${CXXFLAGS} diff --git a/forge/csrc/backend_api/CMakeLists.txt b/forge/csrc/backend_api/CMakeLists.txt index fde194e3e..bc52142a2 100644 --- a/forge/csrc/backend_api/CMakeLists.txt +++ b/forge/csrc/backend_api/CMakeLists.txt @@ -4,5 +4,6 @@ add_library(backend_api backend_api.cpp) target_link_libraries(backend_api shared_utils) +target_link_libraries(backend_api coverage_config) add_dependencies(backend_api shared_utils) target_compile_options(backend_api PRIVATE ${STATIC_LIB_FLAGS} ${TTFORGE_CSRC_CFLAGS}) diff --git a/forge/csrc/graph_lib/CMakeLists.txt b/forge/csrc/graph_lib/CMakeLists.txt index 596b484bd..8d47b7ae1 100644 --- a/forge/csrc/graph_lib/CMakeLists.txt +++ b/forge/csrc/graph_lib/CMakeLists.txt @@ -9,4 +9,6 @@ add_library(graph utils.cpp python_bindings.cpp) +target_link_libraries(graph PUBLIC coverage_config) + target_compile_options(graph PRIVATE ${STATIC_LIB_FLAGS} ${TTFORGE_CSRC_CFLAGS}) diff --git a/forge/csrc/reportify/CMakeLists.txt b/forge/csrc/reportify/CMakeLists.txt index 04b5fe87c..315c4d4aa 100644 --- a/forge/csrc/reportify/CMakeLists.txt +++ b/forge/csrc/reportify/CMakeLists.txt @@ -4,4 +4,6 @@ add_library(reportify paths.cpp to_json.cpp) +target_link_libraries(reportify PUBLIC coverage_config) + target_compile_options(reportify PRIVATE ${STATIC_LIB_FLAGS} ${TTFORGE_CSRC_CFLAGS}) diff --git a/forge/csrc/runtime/CMakeLists.txt b/forge/csrc/runtime/CMakeLists.txt index fb4e875a8..1472f6378 100644 --- a/forge/csrc/runtime/CMakeLists.txt +++ b/forge/csrc/runtime/CMakeLists.txt @@ -1,4 +1,6 @@ add_library(runtime STATIC runtime.cpp tt_device.cpp python_bindings.cpp) add_dependencies(runtime tt-mlir) +target_link_libraries(runtime PUBLIC coverage_config) + target_compile_options(runtime PRIVATE ${STATIC_LIB_FLAGS} ${TTFORGE_CSRC_CFLAGS}) diff --git a/forge/csrc/shared_utils/CMakeLists.txt b/forge/csrc/shared_utils/CMakeLists.txt index 1ebdf0528..cec13ceae 100644 --- a/forge/csrc/shared_utils/CMakeLists.txt +++ b/forge/csrc/shared_utils/CMakeLists.txt @@ -5,4 +5,6 @@ add_library(shared_utils sparse_matmul_utils.cpp string_extension.cpp) +target_link_libraries(shared_utils PUBLIC coverage_config) + target_compile_options(shared_utils PRIVATE ${STATIC_LIB_FLAGS} ${TTFORGE_CSRC_CFLAGS}) diff --git a/forge/csrc/tt_torch_device/CMakeLists.txt b/forge/csrc/tt_torch_device/CMakeLists.txt index 122fccf18..61dcf88fe 100644 --- a/forge/csrc/tt_torch_device/CMakeLists.txt +++ b/forge/csrc/tt_torch_device/CMakeLists.txt @@ -4,5 +4,7 @@ add_library(tt_torch_device torch_device_impl.cpp python_bindings.cpp) +target_link_libraries(tt_torch_device PUBLIC coverage_config) + target_compile_options(tt_torch_device PRIVATE ${STATIC_LIB_FLAGS} ${TTFORGE_CSRC_CFLAGS}) add_dependencies(tt_torch_device tt-mlir)