#13363: Surface job errors where Set up runner does not complete successfully

infra/data_collection/github/utils.py

@@ -7,9 +7,12 @@
 import pathlib
 import os
 from datetime import datetime
+from typing import Optional, Union
 
 from loguru import logger
 
+from infra.data_collection.models import InfraErrorV1
+
 BENCHMARK_ENVIRONMENT_CSV_FIELDS = (
     "git_repo_name",
     "git_commit_hash",
@@ -117,6 +120,22 @@ def return_first_string_starts_with(starting_string, strings):
     raise Exception(f"{strings} do not have any that match {starting_string}")
 
 
+def get_job_failure_signature_(github_job) -> Optional[Union[InfraErrorV1]]:
+    if github_job["conclusion"] == "success":
+        return None
+    for step in github_job["steps"]:
+        is_generic_setup_failure = (
+            step["name"] == "Set up runner"
+            and step["status"] in ("completed", "cancelled")
+            and step["conclusion"] != "success"
+            and step["started_at"] is not None
+            and step["completed_at"] is None
+        )
+        if is_generic_setup_failure:
+            return str(InfraErrorV1.GENERIC_SET_UP_FAILURE)
+    return None
+
+
 def get_job_row_from_github_job(github_job):
     github_job_id = github_job["id"]
 
@@ -220,6 +239,8 @@ def get_job_row_from_github_job(github_job):
 
     github_job_link = github_job["html_url"]
 
+    failure_signature = get_job_failure_signature_(github_job)
+
     return {
         "github_job_id": github_job_id,
         "host_name": host_name,
@@ -235,6 +256,7 @@ def get_job_row_from_github_job(github_job):
         "job_matrix_config": job_matrix_config,
         "docker_image": docker_image,
         "github_job_link": github_job_link,
+        "failure_signature": failure_signature,
     }
 
 

infra/data_collection/models.py

@@ -0,0 +1,5 @@
+import enum
+
+
+class InfraErrorV1(enum.Enum):
+    GENERIC_SET_UP_FAILURE = enum.auto()

infra/pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-addopts = --import-mode=importlib -vvs -rA --durations=0
+addopts = --import-mode=importlib -s -rA --durations=0

infra/tests/_data/data_collection/cicd/all_post_commit_runner_died_12626_10996802864/workflow.json

@@ -0,0 +1 @@
+{"id":10996802864,"name":"All post-commit tests","node_id":"WFR_kwLOI9Wqc88AAAACj3XlMA","head_branch":"virdhatchani/5_clean","head_sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","path":".github/workflows/all-post-commit-workflows.yaml","display_title":"All post-commit tests","run_number":16317,"event":"workflow_dispatch","status":"completed","conclusion":"failure","workflow_id":67993574,"check_suite_id":28731146125,"check_suite_node_id":"CS_kwDOI9Wqc88AAAAGsIJ_jQ","url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/runs/10996802864","html_url":"https://github.com/tenstorrent/tt-metal/actions/runs/10996802864","pull_requests":[],"created_at":"2024-09-23T14:49:36Z","updated_at":"2024-09-23T21:11:38Z","actor":{"login":"VirdhatchaniKN","id":138196495,"node_id":"U_kgDOCDy2Dw","avatar_url":"https://avatars.githubusercontent.com/u/138196495?v=4","gravatar_id":"","url":"https://api.github.com/users/VirdhatchaniKN","html_url":"https://github.com/VirdhatchaniKN","followers_url":"https://api.github.com/users/VirdhatchaniKN/followers","following_url":"https://api.github.com/users/VirdhatchaniKN/following{/other_user}","gists_url":"https://api.github.com/users/VirdhatchaniKN/gists{/gist_id}","starred_url":"https://api.github.com/users/VirdhatchaniKN/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VirdhatchaniKN/subscriptions","organizations_url":"https://api.github.com/users/VirdhatchaniKN/orgs","repos_url":"https://api.github.com/users/VirdhatchaniKN/repos","events_url":"https://api.github.com/users/VirdhatchaniKN/events{/privacy}","received_events_url":"https://api.github.com/users/VirdhatchaniKN/received_events","type":"User","site_admin":false},"run_attempt":1,"referenced_workflows":[{"path":"tenstorrent/tt-metal/.github/workflows/all-static-checks.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/_build-wheels-impl.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/umd-unit-tests.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/models-post-commit.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/build-artifact.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/ttnn-post-commit.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/docs-latest-public.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/run-profiler-regression.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/build-and-unit-tests.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/_test-wheels-impl.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/build.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/fast-dispatch-build-and-unit-tests.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"},{"path":"tenstorrent/tt-metal/.github/workflows/cpp-post-commit.yaml@64249512152ed52c6df7d6adf6bb1a59b96db81c","sha":"64249512152ed52c6df7d6adf6bb1a59b96db81c","ref":"refs/heads/virdhatchani/5_clean"}],"run_started_at":"2024-09-23T14:49:36Z","triggering_actor":{"login":"VirdhatchaniKN","id":138196495,"node_id":"U_kgDOCDy2Dw","avatar_url":"https://avatars.githubusercontent.com/u/138196495?v=4","gravatar_id":"","url":"https://api.github.com/users/VirdhatchaniKN","html_url":"https://github.com/VirdhatchaniKN","followers_url":"https://api.github.com/users/VirdhatchaniKN/followers","following_url":"https://api.github.com/users/VirdhatchaniKN/following{/other_user}","gists_url":"https://api.github.com/users/VirdhatchaniKN/gists{/gist_id}","starred_url":"https://api.github.com/users/VirdhatchaniKN/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VirdhatchaniKN/subscriptions","organizations_url":"https://api.github.com/users/VirdhatchaniKN/orgs","repos_url":"https://api.github.com/users/VirdhatchaniKN/repos","events_url":"https://api.github.com/users/VirdhatchaniKN/events{/privacy}","received_events_url":"https://api.github.com/users/VirdhatchaniKN/received_events","type":"User","site_admin":false},"jobs_url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/runs/10996802864/attempts/1/jobs","logs_url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/runs/10996802864/attempts/1/logs","check_suite_url":"https://api.github.com/repos/tenstorrent/tt-metal/check-suites/28731146125","artifacts_url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/runs/10996802864/artifacts","cancel_url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/runs/10996802864/cancel","rerun_url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/runs/10996802864/rerun","previous_attempt_url":null,"workflow_url":"https://api.github.com/repos/tenstorrent/tt-metal/actions/workflows/67993574","head_commit":{"id":"64249512152ed52c6df7d6adf6bb1a59b96db81c","tree_id":"88785582b5db01a4bafb25ba3016bd16c205a04a","message":"#12990: Update Files","timestamp":"2024-09-23T14:49:13Z","author":{"name":"VirdhatchaniKN","email":"[email protected]"},"committer":{"name":"VirdhatchaniKN","email":"[email protected]"}},"repository":{"id":601205363,"node_id":"R_kgDOI9Wqcw","name":"tt-metal","full_name":"tenstorrent/tt-metal","private":false,"owner":{"login":"tenstorrent","id":64161552,"node_id":"MDEyOk9yZ2FuaXphdGlvbjY0MTYxNTUy","avatar_url":"https://avatars.githubusercontent.com/u/64161552?v=4","gravatar_id":"","url":"https://api.github.com/users/tenstorrent","html_url":"https://github.com/tenstorrent","followers_url":"https://api.github.com/users/tenstorrent/followers","following_url":"https://api.github.com/users/tenstorrent/following{/other_user}","gists_url":"https://api.github.com/users/tenstorrent/gists{/gist_id}","starred_url":"https://api.github.com/users/tenstorrent/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tenstorrent/subscriptions","organizations_url":"https://api.github.com/users/tenstorrent/orgs","repos_url":"https://api.github.com/users/tenstorrent/repos","events_url":"https://api.github.com/users/tenstorrent/events{/privacy}","received_events_url":"https://api.github.com/users/tenstorrent/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/tenstorrent/tt-metal","description":":metal: TT-NN operator library, and TT-Metalium low level kernel programming model.","fork":false,"url":"https://api.github.com/repos/tenstorrent/tt-metal","forks_url":"https://api.github.com/repos/tenstorrent/tt-metal/forks","keys_url":"https://api.github.com/repos/tenstorrent/tt-metal/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tenstorrent/tt-metal/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tenstorrent/tt-metal/teams","hooks_url":"https://api.github.com/repos/tenstorrent/tt-metal/hooks","issue_events_url":"https://api.github.com/repos/tenstorrent/tt-metal/issues/events{/number}","events_url":"https://api.github.com/repos/tenstorrent/tt-metal/events","assignees_url":"https://api.github.com/repos/tenstorrent/tt-metal/assignees{/user}","branches_url":"https://api.github.com/repos/tenstorrent/tt-metal/branches{/branch}","tags_url":"https://api.github.com/repos/tenstorrent/tt-metal/tags","blobs_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/refs{/sha}","trees_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tenstorrent/tt-metal/statuses/{sha}","languages_url":"https://api.github.com/repos/tenstorrent/tt-metal/languages","stargazers_url":"https://api.github.com/repos/tenstorrent/tt-metal/stargazers","contributors_url":"https://api.github.com/repos/tenstorrent/tt-metal/contributors","subscribers_url":"https://api.github.com/repos/tenstorrent/tt-metal/subscribers","subscription_url":"https://api.github.com/repos/tenstorrent/tt-metal/subscription","commits_url":"https://api.github.com/repos/tenstorrent/tt-metal/commits{/sha}","git_commits_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/commits{/sha}","comments_url":"https://api.github.com/repos/tenstorrent/tt-metal/comments{/number}","issue_comment_url":"https://api.github.com/repos/tenstorrent/tt-metal/issues/comments{/number}","contents_url":"https://api.github.com/repos/tenstorrent/tt-metal/contents/{+path}","compare_url":"https://api.github.com/repos/tenstorrent/tt-metal/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tenstorrent/tt-metal/merges","archive_url":"https://api.github.com/repos/tenstorrent/tt-metal/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tenstorrent/tt-metal/downloads","issues_url":"https://api.github.com/repos/tenstorrent/tt-metal/issues{/number}","pulls_url":"https://api.github.com/repos/tenstorrent/tt-metal/pulls{/number}","milestones_url":"https://api.github.com/repos/tenstorrent/tt-metal/milestones{/number}","notifications_url":"https://api.github.com/repos/tenstorrent/tt-metal/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tenstorrent/tt-metal/labels{/name}","releases_url":"https://api.github.com/repos/tenstorrent/tt-metal/releases{/id}","deployments_url":"https://api.github.com/repos/tenstorrent/tt-metal/deployments"},"head_repository":{"id":601205363,"node_id":"R_kgDOI9Wqcw","name":"tt-metal","full_name":"tenstorrent/tt-metal","private":false,"owner":{"login":"tenstorrent","id":64161552,"node_id":"MDEyOk9yZ2FuaXphdGlvbjY0MTYxNTUy","avatar_url":"https://avatars.githubusercontent.com/u/64161552?v=4","gravatar_id":"","url":"https://api.github.com/users/tenstorrent","html_url":"https://github.com/tenstorrent","followers_url":"https://api.github.com/users/tenstorrent/followers","following_url":"https://api.github.com/users/tenstorrent/following{/other_user}","gists_url":"https://api.github.com/users/tenstorrent/gists{/gist_id}","starred_url":"https://api.github.com/users/tenstorrent/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tenstorrent/subscriptions","organizations_url":"https://api.github.com/users/tenstorrent/orgs","repos_url":"https://api.github.com/users/tenstorrent/repos","events_url":"https://api.github.com/users/tenstorrent/events{/privacy}","received_events_url":"https://api.github.com/users/tenstorrent/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/tenstorrent/tt-metal","description":":metal: TT-NN operator library, and TT-Metalium low level kernel programming model.","fork":false,"url":"https://api.github.com/repos/tenstorrent/tt-metal","forks_url":"https://api.github.com/repos/tenstorrent/tt-metal/forks","keys_url":"https://api.github.com/repos/tenstorrent/tt-metal/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tenstorrent/tt-metal/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tenstorrent/tt-metal/teams","hooks_url":"https://api.github.com/repos/tenstorrent/tt-metal/hooks","issue_events_url":"https://api.github.com/repos/tenstorrent/tt-metal/issues/events{/number}","events_url":"https://api.github.com/repos/tenstorrent/tt-metal/events","assignees_url":"https://api.github.com/repos/tenstorrent/tt-metal/assignees{/user}","branches_url":"https://api.github.com/repos/tenstorrent/tt-metal/branches{/branch}","tags_url":"https://api.github.com/repos/tenstorrent/tt-metal/tags","blobs_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/refs{/sha}","trees_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tenstorrent/tt-metal/statuses/{sha}","languages_url":"https://api.github.com/repos/tenstorrent/tt-metal/languages","stargazers_url":"https://api.github.com/repos/tenstorrent/tt-metal/stargazers","contributors_url":"https://api.github.com/repos/tenstorrent/tt-metal/contributors","subscribers_url":"https://api.github.com/repos/tenstorrent/tt-metal/subscribers","subscription_url":"https://api.github.com/repos/tenstorrent/tt-metal/subscription","commits_url":"https://api.github.com/repos/tenstorrent/tt-metal/commits{/sha}","git_commits_url":"https://api.github.com/repos/tenstorrent/tt-metal/git/commits{/sha}","comments_url":"https://api.github.com/repos/tenstorrent/tt-metal/comments{/number}","issue_comment_url":"https://api.github.com/repos/tenstorrent/tt-metal/issues/comments{/number}","contents_url":"https://api.github.com/repos/tenstorrent/tt-metal/contents/{+path}","compare_url":"https://api.github.com/repos/tenstorrent/tt-metal/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tenstorrent/tt-metal/merges","archive_url":"https://api.github.com/repos/tenstorrent/tt-metal/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tenstorrent/tt-metal/downloads","issues_url":"https://api.github.com/repos/tenstorrent/tt-metal/issues{/number}","pulls_url":"https://api.github.com/repos/tenstorrent/tt-metal/pulls{/number}","milestones_url":"https://api.github.com/repos/tenstorrent/tt-metal/milestones{/number}","notifications_url":"https://api.github.com/repos/tenstorrent/tt-metal/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tenstorrent/tt-metal/labels{/name}","releases_url":"https://api.github.com/repos/tenstorrent/tt-metal/releases{/id}","deployments_url":"https://api.github.com/repos/tenstorrent/tt-metal/deployments"}}

infra/tests/_data/data_collection/cicd/all_post_commit_runner_died_12626_11110261767/11110261767/logs/30868260202.log

@@ -0,0 +1,7 @@
+2024-09-30T16:42:38.6994568Z Requested labels: N300, cloud-virtual-machine, in-service
+2024-09-30T16:42:38.6994806Z Job defined at: tenstorrent/tt-metal/.github/workflows/build-and-unit-tests.yaml@refs/heads/asaigal/dispatch_s
+2024-09-30T16:42:38.6995054Z Reusable workflow chain:
+2024-09-30T16:42:38.6995163Z tenstorrent/tt-metal/.github/workflows/all-post-commit-workflows.yaml@refs/heads/asaigal/dispatch_s (a6fea2adef8786a0615456df19fd0582e925bc61)
+2024-09-30T16:42:38.6995240Z -> tenstorrent/tt-metal/.github/workflows/build-and-unit-tests.yaml@refs/heads/asaigal/dispatch_s (a6fea2adef8786a0615456df19fd0582e925bc61)
+2024-09-30T16:42:38.6995328Z Waiting for a runner to pick up this job...
+2024-09-30T16:51:40.6203230Z Job is about to start running on the runner: tt-metal-ci-vm-46 (repository)