Skip to content

Commit

Permalink
Merge branch 'main' into adjordjevic/dump_configuration_registers
Browse files Browse the repository at this point in the history
  • Loading branch information
adjordjevic-TT authored Mar 1, 2025
2 parents f9a483d + b309929 commit 2b3e0c9
Show file tree
Hide file tree
Showing 136 changed files with 3,174 additions and 891 deletions.
4 changes: 4 additions & 0 deletions .github/scripts/data_analysis/print_gtest_annotations.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ def _escape(s: str) -> str:
tree = ET.parse(xml_file)
root = tree.getroot()
for testsuite in root.findall("testsuite"):
# Skip printing out pytest information
# It's already handled by pytest-github-actions-annotate-failures plugin
if testsuite.attrib.get("name") == "pytest":
continue
for testcase in testsuite.findall("testcase"):
failure = testcase.find("failure")
# If failure exists, print the failure message
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/_produce-data.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ on:
- "(Single-card) Nightly model and ttnn tests"
- "(Single-card) Tests for new models"
- "(T3K) T3000 demo tests"
- "(T3K) T3000 frequent tests"
- "(T3K) T3000 model perf tests"
- "(T3K) T3000 nightly tests"
- "(T3K) T3000 perplexity tests"
- "(T3K) T3000 model perf tests"
- "(T3K) T3000 profiler tests"
- "(T3K) T3000 unit tests"
- "(TG) TG unit tests"
Expand Down
13 changes: 13 additions & 0 deletions .github/workflows/pipeline-select-galaxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ on:
required: false
type: boolean
default: false
tg-nightly:
required: false
type: boolean
default: false

run-name: ${{ inputs.description }}
jobs:
Expand Down Expand Up @@ -92,3 +96,12 @@ jobs:
secrets: inherit
uses: ./.github/workflows/tg-model-perf-tests-impl.yaml
if: ${{ inputs.tg-model-perf }}
tg-nightly-tests:
needs: build-artifact
if: ${{ inputs.tg-nightly }}
secrets: inherit
uses: ./.github/workflows/tg-nightly-tests-impl.yaml
with:
docker-image: ${{ needs.build-artifact.outputs.ci-build-docker-image }}
wheel-artifact-name: ${{ needs.build-artifact.outputs.wheel-artifact-name }}
build-artifact-name: ${{ needs.build-artifact.outputs.build-artifact-name }}
6 changes: 6 additions & 0 deletions .github/workflows/t3000-demo-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ jobs:
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
- name: Disable performance mode
if: always()
run: |
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/t3000-frequent-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,9 @@ jobs:
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
6 changes: 6 additions & 0 deletions .github/workflows/t3000-model-perf-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,12 @@ jobs:
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
- name: Disable performance mode
if: always()
run: |
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/t3000-nightly-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,9 @@ jobs:
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
6 changes: 6 additions & 0 deletions .github/workflows/t3000-perplexity-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,9 @@ jobs:
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
6 changes: 6 additions & 0 deletions .github/workflows/t3000-profiler-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,9 @@ jobs:
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: U03BJ1L3LUQ # Mo Memarian
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
14 changes: 14 additions & 0 deletions .github/workflows/t3000-unit-tests-impl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,24 @@ jobs:
source ${{ github.workspace }}/python_env/bin/activate
cd $TT_METAL_HOME
export PYTHONPATH=$TT_METAL_HOME
export GTEST_OUTPUT=xml:generated/test_reports/
mkdir -p generated/test_reports
source ${{ github.workspace }}/tests/scripts/t3000/run_t3000_unit_tests.sh
${{ matrix.test-group.cmd }}
- uses: ./.github/actions/slack-report
if: ${{ failure() }}
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- uses: ./.github/actions/upload-artifact-with-job-uuid
if: ${{ !cancelled() }}
with:
path: |
generated/test_reports/
prefix: "test_reports_"
- name: Generate gtest annotations on failure
uses: ./.github/actions/generate-gtest-failure-message
if: ${{ failure() }}
with:
path: |
generated/test_reports/
80 changes: 80 additions & 0 deletions .github/workflows/tg-nightly-tests-impl.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
name: "[internal] TG nightly tests impl"

on:
workflow_call:
inputs:
docker-image:
required: true
type: string
wheel-artifact-name:
required: true
type: string
build-artifact-name:
required: true
type: string

jobs:
tg-nightly-tests:
strategy:
fail-fast: false
matrix:
test-group: [
{ name: "TG CCL tests", arch: wormhole_b0, cmd: run_tg_nightly_ccl_tests, timeout: 40, owner_id: ULMEPM2MA}, # Sean Nijjar
]
name: ${{ matrix.test-group.name }}
runs-on: ["arch-wormhole_b0", "config-tg", "in-service", "pipeline-functional"]
container:
image: ${{ inputs.docker-image }}
env:
TT_METAL_HOME: /work
PYTHONPATH: /work
LD_LIBRARY_PATH: /work/build/lib
LOGURU_LEVEL: INFO
ARCH_NAME: ${{ matrix.test-group.arch }}
volumes:
- ${{ github.workspace }}/docker-job:/work # Subdir to workaround https://github.com/actions/runner/issues/691
- /dev/hugepages-1G:/dev/hugepages-1G
- /mnt/MLPerf:/mnt/MLPerf
options: "--device /dev/tenstorrent"
defaults:
run:
shell: bash
working-directory: /work # https://github.com/actions/runner/issues/878
steps:
- name: ⬇️ Checkout
uses: actions/checkout@v4
with:
submodules: recursive
path: docker-job # Here be dragons; keep it scoped to our desired volume, yet must be under github.workspace and be sure to clean up at the end
- uses: actions/download-artifact@v4
with:
name: ${{ inputs.build-artifact-name }}
path: /work
- name: Extract files
run: tar -xvf ttm_any.tar
- name: ⬇️ Download Wheel
uses: actions/download-artifact@v4
with:
name: ${{ inputs.wheel-artifact-name }}
path: /work
- name: Install Wheel
run: |
WHEEL_FILENAME=$(ls -1 *.whl)
pip3 install $WHEEL_FILENAME
- name: Run tests
shell: bash {0}
timeout-minutes: ${{ matrix.test-group.timeout }}
run: |
source tests/scripts/tg/run_tg_nightly_tests.sh
${{ matrix.test-group.cmd }}
- uses: ./.github/actions/slack-report
if: ${{ failure() }}
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- name: Cleanup
if: always()
run: |
# We are forced to checkout the repo into a subdir of the host's workdir; this pollutes the host
# with root-owned files. Be sure to clean up after ourselves in case we're on a non-ephemeral runner.
rm -rf /__w/tt-metal/tt-metal/docker-job
69 changes: 6 additions & 63 deletions .github/workflows/tg-nightly-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,66 +13,9 @@ jobs:
build-wheel: true
tg-nightly-tests:
needs: build-artifact
strategy:
fail-fast: false
matrix:
test-group: [
{ name: "placeholder", arch: wormhole_b0, cmd: "echo 'Placeholder'", timeout: 90, owner_id: U03FJB5TM5Y}, # Colman Glagovich
]
name: ${{ matrix.test-group.name }}
runs-on: ["arch-wormhole_b0", "config-tg", "in-service", "pipeline-functional"]
container:
image: ${{ needs.build-artifact.outputs.ci-build-docker-image }}
env:
TT_METAL_HOME: /work
PYTHONPATH: /work
LD_LIBRARY_PATH: /work/build/lib
LOGURU_LEVEL: INFO
ARCH_NAME: ${{ matrix.test-group.arch }}
volumes:
- ${{ github.workspace }}/docker-job:/work # Subdir to workaround https://github.com/actions/runner/issues/691
- /dev/hugepages-1G:/dev/hugepages-1G
- /mnt/MLPerf:/mnt/MLPerf
options: "--device /dev/tenstorrent"
defaults:
run:
shell: bash
working-directory: /work # https://github.com/actions/runner/issues/878
steps:
- name: ⬇️ Checkout
uses: actions/checkout@v4
with:
submodules: recursive
path: docker-job # Here be dragons; keep it scoped to our desired volume, yet must be under github.workspace and be sure to clean up at the end
- uses: actions/download-artifact@v4
with:
name: ${{ needs.build-artifact.outputs.build-artifact-name }}
path: /work
- name: Extract files
run: tar -xvf ttm_any.tar
- name: ⬇️ Download Wheel
uses: actions/download-artifact@v4
with:
name: ${{ needs.build-artifact.outputs.wheel-artifact-name }}
path: /work
- name: Install Wheel
run: |
WHEEL_FILENAME=$(ls -1 *.whl)
pip3 install $WHEEL_FILENAME
- name: Run demo regression tests
shell: bash {0}
timeout-minutes: ${{ matrix.test-group.timeout }}
run: |
source ${{ github.workspace }}/tests/scripts/tg/run_tg_nightly_tests.sh
${{ matrix.test-group.cmd }}
- uses: ./.github/actions/slack-report
if: ${{ failure() }}
with:
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
owner: ${{ matrix.test-group.owner_id }}
- name: Cleanup
if: always()
run: |
# We are forced to checkout the repo into a subdir of the host's workdir; this pollutes the host
# with root-owned files. Be sure to clean up after ourselves in case we're on a non-ephemeral runner.
rm -rf /__w/tt-metal/tt-metal/docker-job
uses: ./.github/workflows/tg-nightly-tests-impl.yaml
secrets: inherit
with:
docker-image: ${{ needs.build-artifact.outputs.ci-build-docker-image }}
wheel-artifact-name: ${{ needs.build-artifact.outputs.wheel-artifact-name }}
build-artifact-name: ${{ needs.build-artifact.outputs.build-artifact-name }}
Loading

0 comments on commit 2b3e0c9

Please sign in to comment.