diff --git a/torchci/components/GranularityPicker.tsx b/torchci/components/GranularityPicker.tsx index f5f54e0ca7..0e92bb46e7 100644 --- a/torchci/components/GranularityPicker.tsx +++ b/torchci/components/GranularityPicker.tsx @@ -25,10 +25,10 @@ export default function GranularityPicker({ labelId="granularity-select-label" onChange={handleChange} > - month - week - day hour + day + week + month ); diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts index 9bae26d883..9df9d824a1 100644 --- a/torchci/lib/types.ts +++ b/torchci/lib/types.ts @@ -149,7 +149,7 @@ export interface JobsPerCommitData { } export interface CompilerPerformanceData { - abs_latency: number, + abs_latency: number; accuracy: string; compilation_latency: number; compiler: string; diff --git a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx index 3b9b00942f..2f4d85a889 100644 --- a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx +++ b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx @@ -52,6 +52,7 @@ import { LOG_PREFIX, COMMIT_TO_WORKFLOW_ID, WORKFLOW_ID_TO_COMMIT, + SHA_DISPLAY_LENGTH, } from "../../compilers"; import { CompilerPerformanceData } from "lib/types"; import styles from "components/metrics.module.css"; @@ -73,16 +74,22 @@ const SCALE = 4; function CommitPanel({ suite, - branch, - commit, + lBranch, + lCommit, + lDate, + rBranch, + rCommit, + rDate, workflowId, - date, }: { suite: string; - branch: string; - commit: string; + lBranch: string; + lCommit: string; + lDate: string; + rBranch: string; + rCommit: string; + rDate: string; workflowId: number; - date: string; }) { const queryCollection = "commons"; const queryName = "get_workflow_jobs"; @@ -141,11 +148,16 @@ function CommitPanel({ return ( - *This report was generated by CI running on PyTorch {branch} branch at{" "} - - {commit.substring(0, 7)} + *This report was generated by CI running on PyTorch {lBranch} branch at{" "} + + {lCommit.substring(0, SHA_DISPLAY_LENGTH)} {" "} - on {dayjs(date).format("YYYY/MM/DD")}. The running logs per shard are:{" "} + on {dayjs(lDate).format("YYYY/MM/DD")} comparing with {rBranch} branch + at commit{" "} + + {rCommit.substring(0, SHA_DISPLAY_LENGTH)} + + . The running logs per shard are:{" "} . @@ -155,6 +167,7 @@ function CommitPanel({ function ModelPanel({ startTime, stopTime, + granularity, suite, mode, dtype, @@ -169,6 +182,7 @@ function ModelPanel({ }: { startTime: dayjs.Dayjs; stopTime: dayjs.Dayjs; + granularity: Granularity; suite: string; mode: string; dtype: string; @@ -287,7 +301,7 @@ function ModelPanel({ : undefined; const encodedName = encodeURIComponent(name); - const url = `/benchmark/${suite}/${compiler}?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&model=${encodedName}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; + const url = `/benchmark/${suite}/${compiler}?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&model=${encodedName}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; if (lLog === undefined) { return ( @@ -303,7 +317,7 @@ function ModelPanel({  ( - {lCommit.substr(0, 7)} + {lCommit.substr(0, SHA_DISPLAY_LENGTH)} ) @@ -317,11 +331,11 @@ function ModelPanel({  ( - {rCommit.substr(0, 7)} + {rCommit.substr(0, SHA_DISPLAY_LENGTH)} {" "} →{" "} - {lCommit.substr(0, 7)} + {lCommit.substr(0, SHA_DISPLAY_LENGTH)} ) @@ -691,9 +705,7 @@ function GraphPanel({ record.compression_ratio = Number( record.compression_ratio.toFixed(SCALE) ); - record.abs_latency = Number( - record.abs_latency.toFixed(SCALE) - ); + record.abs_latency = Number(record.abs_latency.toFixed(SCALE)); // Truncate the data to make it consistent with the display value return record; }); @@ -984,10 +996,17 @@ function Report({
@@ -1166,6 +1196,7 @@ export default function Page() { setStopTime={setStopTime} timeRange={timeRange} setTimeRange={setTimeRange} + setGranularity={setGranularity} /> {branches[branch].map((r: any) => ( - {r.head_sha.substring(0, 7)} ( + {r.head_sha.substring(0, SHA_DISPLAY_LENGTH)} ( {dayjs(r.event_time).format("YYYY/MM/DD")}) ))} @@ -716,15 +717,21 @@ export function LogLinks({ } function CommitPanel({ - branch, - commit, + lBranch, + lCommit, + lDate, + rBranch, + rCommit, + rDate, workflowId, - date, }: { - branch: string; - commit: string; + lBranch: string; + lCommit: string; + lDate: string; + rBranch: string; + rCommit: string; + rDate: string; workflowId: number; - date: string; }) { const queryCollection = "commons"; const queryName = "get_workflow_jobs"; @@ -780,12 +787,17 @@ function CommitPanel({ return ( - *This report was generated by CI running on PyTorch {branch} branch at + *This report was generated by CI running on PyTorch {lBranch} branch at commit{" "} - - {commit.substring(0, 7)} + + {lCommit.substring(0, SHA_DISPLAY_LENGTH)} {" "} - on {dayjs(date).format("YYYY/MM/DD")}. The running logs per shard are:{" "} + on {dayjs(lDate).format("YYYY/MM/DD")} comparing with {rBranch} branch + at commit{" "} + + {rCommit.substring(0, SHA_DISPLAY_LENGTH)} + + . The running logs per shard are:{" "} {Object.keys(SUITES).map((suite: string) => { // Hack alert: The test configuration uses timm instead of timm_model as its output const name = suite.includes("timm") ? "timm" : suite; @@ -900,6 +912,7 @@ function extractPercentage(value: string) { function SummaryPanel({ startTime, stopTime, + granularity, mode, dtype, lBranch, @@ -911,6 +924,7 @@ function SummaryPanel({ }: { startTime: dayjs.Dayjs; stopTime: dayjs.Dayjs; + granularity: Granularity; mode: string; dtype: string; lBranch: string; @@ -1009,7 +1023,7 @@ function SummaryPanel({ const url = `/benchmark/${suite}/${ DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ?? params.row.compiler - }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; + }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; const l = extractPercentage(v.l); const r = extractPercentage(v.r); @@ -1097,7 +1111,7 @@ function SummaryPanel({ const url = `/benchmark/${suite}/${ DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ?? params.row.compiler - }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; + }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; const l = Number(v.l).toFixed(SCALE); const r = Number(v.r).toFixed(SCALE); @@ -1188,7 +1202,7 @@ function SummaryPanel({ const url = `/benchmark/${suite}/${ DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ?? params.row.compiler - }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; + }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; const l = Number(v.l).toFixed(0); const r = Number(v.r).toFixed(0); @@ -1292,7 +1306,7 @@ function SummaryPanel({ const url = `/benchmark/${suite}/${ DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ?? params.row.compiler - }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; + }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`; const l = Number(v.l).toFixed(SCALE); const r = Number(v.r).toFixed(SCALE); @@ -1753,14 +1767,18 @@ function Report({ return (
@@ -1914,6 +1942,7 @@ export default function Page() { setStopTime={setStopTime} timeRange={timeRange} setTimeRange={setTimeRange} + setGranularity={setGranularity} /> = PARSE_DATETIME_ISO8601(:startTime) - AND torch_dynamo_perf_stats._event_time < PARSE_DATETIME_ISO8601(:stopTime) + torch_dynamo_perf_stats._event_time >= PARSE_DATETIME_ISO8601(: startTime) + AND torch_dynamo_perf_stats._event_time < PARSE_DATETIME_ISO8601(: stopTime) + AND torch_dynamo_perf_stats.filename LIKE '%_performance' + AND torch_dynamo_perf_stats.filename LIKE CONCAT( + '%_', : dtypes, '_', : mode, '_', : device, + '_%' + ) ORDER BY w.head_branch, - event_time DESC \ No newline at end of file + event_time DESC diff --git a/torchci/rockset/inductor/compilers_benchmark_performance.lambda.json b/torchci/rockset/inductor/compilers_benchmark_performance.lambda.json index 2eca213252..b97cf9bdda 100644 --- a/torchci/rockset/inductor/compilers_benchmark_performance.lambda.json +++ b/torchci/rockset/inductor/compilers_benchmark_performance.lambda.json @@ -68,4 +68,4 @@ } ], "description": "Gather the benchmark performance of different PyTorch compilers" -} \ No newline at end of file +} diff --git a/torchci/rockset/inductor/compilers_benchmark_performance_branches.lambda.json b/torchci/rockset/inductor/compilers_benchmark_performance_branches.lambda.json index 828efeb3d6..e9ac17c6db 100644 --- a/torchci/rockset/inductor/compilers_benchmark_performance_branches.lambda.json +++ b/torchci/rockset/inductor/compilers_benchmark_performance_branches.lambda.json @@ -1,11 +1,26 @@ { "sql_path": "__sql/compilers_benchmark_performance_branches.sql", "default_parameters": [ + { + "name": "device", + "type": "string", + "value": "cuda" + }, + { + "name": "dtypes", + "type": "string", + "value": "amp" + }, { "name": "granularity", "type": "string", "value": "day" }, + { + "name": "mode", + "type": "string", + "value": "training" + }, { "name": "startTime", "type": "string", diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json index c890564121..40dff80cab 100644 --- a/torchci/rockset/prodVersions.json +++ b/torchci/rockset/prodVersions.json @@ -77,11 +77,11 @@ }, "inductor": { "compilers_benchmark_performance": "d43601277905c4a0", - "compilers_benchmark_performance_branches": "f259d06862b41888" + "compilers_benchmark_performance_branches": "8896fe6bbd61e7dc" }, "utilization": { "runner_utilization": "2dfc4f9c16e51da0", "runner_utilization_by_repo": "2ca7468c6f158924", "runner_utilization_by_activity": "343929e0ebeee379" } -} \ No newline at end of file +}