Merge branch 'main' into akannan/llava_model_bringup

.github/pull_request_template.md

@@ -0,0 +1,12 @@
+### Ticket
+Link to Github Issue
+
+### Problem description
+Provide context for the problem.
+
+### What's changed
+Describe the approach used to solve the problem.
+Summarize the changes made and its impact.
+
+### Checklist
+- [ ] New/Existing tests provide coverage for changes

.github/workflows/build.yml

@@ -0,0 +1,121 @@
+name: Build
+
+on:
+  workflow_dispatch:
+    inputs:
+      mlir_override:
+        description: 'Git SHA of commit in tenstorrent/tt-mlir or branch name'
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      mlir_override:
+        description: 'Git SHA of commit in tenstorrent/tt-mlir or branch name'
+        required: false
+        type: string
+      docker-image:
+        description: 'Docker image to use for build'
+        required: true
+        type: string
+
+permissions:
+  packages: write
+  checks: write
+  pull-requests: write # only required if `comment: true` was enabled
+
+jobs:
+
+  forge-build:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+          - runs-on: [runner]
+
+    runs-on:
+      - in-service
+      - ${{ matrix.build.runs-on }}
+
+    container:
+      image: ${{ inputs.docker-image }}
+
+    steps:
+
+    - name: Fetch job id
+      id: fetch-job-id
+      uses: tenstorrent/tt-github-actions/.github/actions/job_id@main
+      with:
+        job_name: "${{ github.job }}"
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      env:
+        JOB_ID: ${{ steps.fetch-job-id.outputs.job_id }}
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+    - name: Git safe dir
+      run: |
+        git config --system --add safe.directory ${{ steps.strings.outputs.work-dir }}
+        chown -R root:root ${{ steps.strings.outputs.work-dir }}
+
+    - uses: actions/checkout@v4
+      with:
+          submodules: recursive
+          clean: true
+          fetch-depth: 0 # Fetch all history and tags
+
+    # Clean everything from submodules (needed to avoid issues
+    # with cmake generated files leftover from previous builds)
+    - name: Cleanup submodules
+      run: |
+          git submodule foreach --recursive git clean -ffdx
+          git submodule foreach --recursive git reset --hard
+
+    - name: Update submodule if mlir_override is set
+      if: ${{ inputs.mlir_override }}
+      run: |
+        cd third_party/tt-mlir
+        git fetch
+        git checkout ${{ inputs.mlir_override }}
+        branch_name=$(git rev-parse --abbrev-ref HEAD)
+        commit_sha=$(git rev-parse HEAD)
+        commit_title=$(git log -1 --pretty=%s)
+        echo "Branch name: $branch_name"
+        echo "Commit SHA: $commit_sha"
+        echo "Commit title: $commit_title"
+        echo "::notice::Using tt-mlir branch: $branch_name, commit: $commit_sha, title: $commit_title"
+        cd ../..
+
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        create-symlink: true
+        key: ${{ matrix.build.runs-on }}-runtime-${{ matrix.build.enable_runtime }}
+
+    - name: Build (creates tvm and tt-forge-fe wheels)
+      shell: bash
+      run: |
+        source env/activate
+        python3 setup.py bdist_wheel
+        cd third_party/tvm/python
+        python3 setup.py bdist_wheel
+        cd ../../..
+        cp third_party/tvm/python/dist/*.whl dist/
+
+    - name: Run Unit Tests
+      shell: bash
+      run: |
+        source env/activate
+        pip install dist/forge*.whl --force-reinstall
+        cmake --build ${{ steps.strings.outputs.build-output-dir }} -- run_unit_tests
+
+    - name: Upload Build
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: forge-wheel
+        path: dist/*.whl

.github/workflows/model-analysis.yml

@@ -36,10 +36,12 @@ jobs:
         - /etc/udev/rules.d:/etc/udev/rules.d
         - /lib/modules:/lib/modules
         - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+        - /mnt/dockercache:/mnt/dockercache
 
     env:
       GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      HF_HOME: /mnt/dockercache/huggingface
       HF_HUB_DISABLE_PROGRESS_BARS: 1
 
     steps:

.github/workflows/on-nightly.yml

@@ -7,9 +7,30 @@ on:
 
 jobs:
   docker-build:
-    uses: ./.github/workflows/build-and-test.yml
+      uses: ./.github/workflows/build-image.yml
+      secrets: inherit
+  build:
+    needs: docker-build
+    uses: ./.github/workflows/build.yml
+    secrets: inherit
+    with:
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}
+  test:
+    needs:
+      - docker-build
+      - build
+    uses: ./.github/workflows/test.yml
     secrets: inherit
     with:
       test_mark: 'nightly'
       test_group_cnt: 4
       test_group_ids: '[1,2,3,4]'
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}
+  perf-benchmark:
+    needs:
+      - docker-build
+      - build
+    uses: ./.github/workflows/perf-benchmark.yml
+    secrets: inherit
+    with:
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}

.github/workflows/on-pr.yml

@@ -2,11 +2,16 @@ name: On PR
 
 on:
   workflow_dispatch:
+    inputs:
+      mlir_override:
+        description: 'Git SHA of commit in tenstorrent/tt-mlir'
+        required: false
+        type: string
   pull_request:
     branches: [ "main" ]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.mlir_override }}
   cancel-in-progress: true
 
 jobs:
@@ -17,13 +22,38 @@ jobs:
     uses: ./.github/workflows/pre-commit.yml
     secrets: inherit
   docker-build:
-    uses: ./.github/workflows/build-and-test.yml
+      uses: ./.github/workflows/build-image.yml
+      secrets: inherit
+  build:
+    needs: docker-build
+    uses: ./.github/workflows/build.yml
+    secrets: inherit
+    with:
+      mlir_override: ${{ inputs.mlir_override }}
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}
+  test:
+    needs:
+      - docker-build
+      - build
+    uses: ./.github/workflows/test.yml
     secrets: inherit
     with:
       test_mark: 'push'
       test_group_cnt: 2
       test_group_ids: '[1,2]'
-  perf-benchmark:
-    needs: docker-build
-    uses: ./.github/workflows/perf-benchmark.yml
-    secrets: inherit
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}
+
+  check-all-green:
+    if: always()
+    needs:
+      - pre-commit
+      - spdx
+      - docker-build
+      - build
+      - test
+    runs-on: Ubuntu-latest
+    steps:
+    - name: Check if the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}

.github/workflows/on-push.yml

@@ -7,9 +7,23 @@ on:
 
 jobs:
   docker-build:
-    uses: ./.github/workflows/build-and-test.yml
+      uses: ./.github/workflows/build-image.yml
+      secrets: inherit
+  build:
+    needs: docker-build
+    uses: ./.github/workflows/build.yml
+    secrets: inherit
+    with:
+      mlir_override: ${{ inputs.mlir_override }}
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}
+  test:
+    needs:
+      - docker-build
+      - build
+    uses: ./.github/workflows/test.yml
     secrets: inherit
     with:
       test_mark: 'push'
       test_group_cnt: 2
       test_group_ids: '[1,2]'
+      docker-image: ${{ needs.docker-build.outputs.docker-image }}

.github/workflows/perf-benchmark.yml

@@ -1,18 +1,17 @@
-name: Build and Test
+name: Perf benchmark
 
 on:
-  workflow_dispatch:
   workflow_call:
+    inputs:
+      docker-image:
+        description: 'Docker image to use for build'
+        required: true
+        type: string
 
 jobs:
 
-  docker-build:
-    uses: ./.github/workflows/build-image.yml
-    secrets: inherit
+  run-perf-benchmarks:
 
-  build-and-run-perf:
-
-    needs: docker-build
     strategy:
       fail-fast: false
     runs-on:
@@ -21,7 +20,7 @@ jobs:
       - performance
 
     container:
-      image: ${{ needs.docker-build.outputs.docker-image }}
+      image: ${{ inputs.docker-image }}
       options: --device /dev/tenstorrent/0
       volumes:
         - /dev/hugepages:/dev/hugepages
@@ -31,31 +30,33 @@ jobs:
         - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
     steps:
 
+    - name: Fetch job id
+      id: fetch-job-id
+      uses: tenstorrent/tt-github-actions/.github/actions/job_id@main
+      with:
+        job_name: "${{ github.job }}"
+
     - name: Set reusable strings
       id: strings
       shell: bash
       env:
-        job-name: "${{ github.job }}"
+        JOB_ID: ${{ steps.fetch-job-id.outputs.job_id }}
       run: |
         echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
         echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
-
-        # Github job context unfortunately doesn't contain job_id, this is the workaround how to fetch it using GH API
-        echo "Expected job name: ${{ env.job-name }}"
-        JOB_ID=$(curl -s -H "Authorization: token ${{ secrets.GH_TOKEN }}" \
-          "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs" | \
-          jq -r '.jobs[] | select(.name | contains("${{ env.job-name }}")) | .id ')
-        echo "Current job id: $JOB_ID"
-
-        echo "job-id=$JOB_ID" >> "$GITHUB_OUTPUT"
         echo "perf_report_path=forge-benchmark-e2e-mnist_$JOB_ID.json" >> "$GITHUB_OUTPUT"
 
     - name: Git safe dir
       run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
 
     - uses: actions/checkout@v4
       with:
-          submodules: recursive
+          sparse-checkout: |
+            env/
+            forge/test
+            pytest.ini
+            conftest.py
+            .test_durations
           fetch-depth: 0 # Fetch all history and tags
 
     # Clean everything from submodules (needed to avoid issues
@@ -65,30 +66,23 @@ jobs:
           git submodule foreach --recursive git clean -ffdx
           git submodule foreach --recursive git reset --hard
 
-    - name: ccache
-      uses: hendrikmuhs/[email protected]
+    - name: Download build artifacts
+      uses: actions/download-artifact@v4
       with:
-        create-symlink: true
-        key: runner-runtime--
+        name: forge-wheel
 
-    - name: Build
+    - name: Install wheel
       shell: bash
       run: |
         source env/activate
-        cmake -G Ninja \
-        -B ${{ steps.strings.outputs.build-output-dir }} \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_C_COMPILER=clang \
-        -DCMAKE_CXX_COMPILER=clang++ \
-        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-        cmake --build ${{ steps.strings.outputs.build-output-dir }}
+        pip install tvm*.whl --force-reinstall
+        pip install forge*.whl --force-reinstall
 
     - name: Run Perf Benchmark
       shell: bash
       run: |
         source env/activate
-        python forge/test/benchmark/benchmark.py -m mnist_linear -bs 1 -o ${{ steps.strings.outputs.perf_report_path }}
+        python forge/test/benchmark/benchmark.py -m mnist_linear -bs 1 -lp 32 -o ${{ steps.strings.outputs.perf_report_path }}
 
     - name: Upload Perf Report
       uses: actions/upload-artifact@v4

.github/workflows/produce_data.yml

@@ -1,4 +1,5 @@
 name: "[internal] Collect workflow data"
+run-name: "Collect data for run_id ${{ github.event.workflow_run.id }} attempt ${{ github.event.workflow_run.run_attempt }}"
 
 on:
   workflow_run: