Various fixes for PT2 inductor dashboard (#4235)

This PR includes several fixes for PT2 inductor dashboard including: * #4186: Switching between training and inference modes now shows the correct commits associated with each mode. Previously, all commits were shown and inference mode, running less frequently, wasn't available in some. Thus, a blank page was displayed on the dashboard as there was no data * #4123: It turns out that the fix for #4186 also fixes this issue as there wasn't any PT2 benchmark data 3 months ago * I also update the granularity picker so that when choosing a quarterly period or longer time range (half, yearly), the weekly granularity is used by default initially. This is to avoid cluttering the graph with too many data points. * #4137: The link now includes HTTP protocol * #4121: The display length of 7 was used by GitHub. I now increase the display length to 10 * #4122. I tweak the summary line so that it prints and links both new and old commits instead of just the new commit ### Testing * [Inference mode](https://torchci-git-fork-huydhn-indutor-dashboard-20230523-fbopensource.vercel.app/benchmark/compilers?startTime=Tue%2C%2023%20May%202023%2000%3A12%3A32%20GMT&stopTime=Tue%2C%2030%20May%202023%2000%3A12%3A32%20GMT&granularity=hour&suite=torchbench&mode=inference&dtype=amp&lBranch=main&lCommit=af70fe9f3e4018531722ab2d9d3ede9150adf2e7&rBranch=main&rCommit=bf059e3925a2565cb05a407f541cc11d762eef05) * [Last month view](https://torchci-git-fork-huydhn-indutor-dashboard-20230523-fbopensource.vercel.app/benchmark/compilers?startTime=Sun%2C%2030%20Apr%202023%2000%3A15%3A50%20GMT&stopTime=Tue%2C%2030%20May%202023%2000%3A15%3A50%20GMT&granularity=day&suite=torchbench&mode=training&dtype=amp&lBranch=main&lCommit=af1d437654aa7fd5aa4a5d30ae2f2e9af34c7765&rBranch=main&rCommit=9cda7b9e4725f37093ed383691d5713ddd168af4) * [Last quarter view](https://torchci-git-fork-huydhn-indutor-dashboard-20230523-fbopensource.vercel.app/benchmark/compilers?startTime=Wed%2C%2001%20Mar%202023%2001%3A12%3A55%20GMT&stopTime=Tue%2C%2030%20May%202023%2000%3A12%3A55%20GMT&granularity=week&suite=torchbench&mode=inference&dtype=amp&lBranch=main&lCommit=af70fe9f3e4018531722ab2d9d3ede9150adf2e7&rBranch=main&rCommit=d255c8e1ad332a2ded0ff1595b63d50034c473ee). I will need to follow up with a way to reduce the amount of data transfer in lager time range.
pytorch · May 30, 2023 · 39dc73b · 39dc73b · vercel · May 30, 2023
1 parent f45e640
commit 39dc73b
Show file tree

Hide file tree

Showing 10 changed files with 167 additions and 58 deletions.
diff --git a/torchci/components/GranularityPicker.tsx b/torchci/components/GranularityPicker.tsx
@@ -25,10 +25,10 @@ export default function GranularityPicker({
         labelId="granularity-select-label"
         onChange={handleChange}
       >
-        <MenuItem value={"month"}>month</MenuItem>
-        <MenuItem value={"week"}>week</MenuItem>
-        <MenuItem value={"day"}>day</MenuItem>
         <MenuItem value={"hour"}>hour</MenuItem>
+        <MenuItem value={"day"}>day</MenuItem>
+        <MenuItem value={"week"}>week</MenuItem>
+        <MenuItem value={"month"}>month</MenuItem>
       </Select>
     </FormControl>
   );

diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts
@@ -149,7 +149,7 @@ export interface JobsPerCommitData {
 }
 
 export interface CompilerPerformanceData {
-  abs_latency: number,
+  abs_latency: number;
   accuracy: string;
   compilation_latency: number;
   compiler: string;

diff --git a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx
@@ -52,6 +52,7 @@ import {
   LOG_PREFIX,
   COMMIT_TO_WORKFLOW_ID,
   WORKFLOW_ID_TO_COMMIT,
+  SHA_DISPLAY_LENGTH,
 } from "../../compilers";
 import { CompilerPerformanceData } from "lib/types";
 import styles from "components/metrics.module.css";
@@ -73,16 +74,22 @@ const SCALE = 4;
 
 function CommitPanel({
   suite,
-  branch,
-  commit,
+  lBranch,
+  lCommit,
+  lDate,
+  rBranch,
+  rCommit,
+  rDate,
   workflowId,
-  date,
 }: {
   suite: string;
-  branch: string;
-  commit: string;
+  lBranch: string;
+  lCommit: string;
+  lDate: string;
+  rBranch: string;
+  rCommit: string;
+  rDate: string;
   workflowId: number;
-  date: string;
 }) {
   const queryCollection = "commons";
   const queryName = "get_workflow_jobs";
@@ -141,11 +148,16 @@ function CommitPanel({
   return (
     <Stack direction="row" spacing={2} sx={{ mb: 2 }}>
       <Typography fontSize={"1rem"} fontStyle={"italic"}>
-        *This report was generated by CI running on PyTorch {branch} branch at{" "}
-        <a href={`${HUD_PREFIX}/${commit}#inductor-a100-perf-nightly`}>
-          {commit.substring(0, 7)}
+        *This report was generated by CI running on PyTorch {lBranch} branch at{" "}
+        <a href={`${HUD_PREFIX}/${lCommit}#inductor-a100-perf-nightly`}>
+          {lCommit.substring(0, SHA_DISPLAY_LENGTH)}
         </a>{" "}
-        on {dayjs(date).format("YYYY/MM/DD")}. The running logs per shard are:{" "}
+        on {dayjs(lDate).format("YYYY/MM/DD")} comparing with {rBranch} branch
+        at commit{" "}
+        <a href={`${HUD_PREFIX}/${rCommit}#inductor-a100-perf-nightly`}>
+          {rCommit.substring(0, SHA_DISPLAY_LENGTH)}
+        </a>
+        . The running logs per shard are:{" "}
         <LogLinks key={`log-${name}`} suite={name} logs={logs} />.
       </Typography>
     </Stack>
@@ -155,6 +167,7 @@ function CommitPanel({
 function ModelPanel({
   startTime,
   stopTime,
+  granularity,
   suite,
   mode,
   dtype,
@@ -169,6 +182,7 @@ function ModelPanel({
 }: {
   startTime: dayjs.Dayjs;
   stopTime: dayjs.Dayjs;
+  granularity: Granularity;
   suite: string;
   mode: string;
   dtype: string;
@@ -287,7 +301,7 @@ function ModelPanel({
                     : undefined;
 
                 const encodedName = encodeURIComponent(name);
-                const url = `/benchmark/${suite}/${compiler}?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&model=${encodedName}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
+                const url = `/benchmark/${suite}/${compiler}?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&model=${encodedName}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
 
                 if (lLog === undefined) {
                   return (
@@ -303,7 +317,7 @@ function ModelPanel({
                       </a>
                       &nbsp;(
                       <a target="_blank" rel="noreferrer" href={lLog}>
-                        <u>{lCommit.substr(0, 7)}</u>
+                        <u>{lCommit.substr(0, SHA_DISPLAY_LENGTH)}</u>
                       </a>
                       )
                     </>
@@ -317,11 +331,11 @@ function ModelPanel({
                     </a>
                     &nbsp;(
                     <a target="_blank" rel="noreferrer" href={rLog}>
-                      <u>{rCommit.substr(0, 7)}</u>
+                      <u>{rCommit.substr(0, SHA_DISPLAY_LENGTH)}</u>
                     </a>{" "}
                     →{" "}
                     <a target="_blank" rel="noreferrer" href={lLog}>
-                      <u>{lCommit.substr(0, 7)}</u>
+                      <u>{lCommit.substr(0, SHA_DISPLAY_LENGTH)}</u>
                     </a>
                     )
                   </>
@@ -691,9 +705,7 @@ function GraphPanel({
       record.compression_ratio = Number(
         record.compression_ratio.toFixed(SCALE)
       );
-      record.abs_latency = Number(
-        record.abs_latency.toFixed(SCALE)
-      );
+      record.abs_latency = Number(record.abs_latency.toFixed(SCALE));
       // Truncate the data to make it consistent with the display value
       return record;
     });
@@ -984,10 +996,17 @@ function Report({
     <div>
       <CommitPanel
         suite={suite}
-        branch={lBranch}
-        commit={lCommit}
+        lBranch={lBranch}
+        lCommit={lCommit}
+        lDate={lData[0].granularity_bucket}
+        rBranch={rBranch}
+        rCommit={rCommit}
+        rDate={
+          rData !== undefined && rData.length !== 0
+            ? rData[0].granularity_bucket
+            : undefined
+        }
         workflowId={lData[0].workflow_id}
-        date={lData[0].granularity_bucket}
       />
       <GraphPanel
         queryParams={queryParams}
@@ -1001,6 +1020,7 @@ function Report({
       <ModelPanel
         startTime={startTime}
         stopTime={stopTime}
+        granularity={granularity}
         suite={suite}
         mode={mode}
         dtype={dtype}
@@ -1060,6 +1080,12 @@ export default function Page() {
       }
     }
 
+    const granularity: Granularity =
+      (router.query.granularity as Granularity) ?? undefined;
+    if (granularity !== undefined) {
+      setGranularity(granularity);
+    }
+
     const mode: string = (router.query.mode as string) ?? undefined;
     if (mode !== undefined) {
       setMode(mode);
@@ -1090,7 +1116,11 @@ export default function Page() {
       setRCommit(rCommit);
     }
 
-    setBaseUrl(`${window.location.host}${router.asPath.replace(/\?.+/, "")}`);
+    setBaseUrl(
+      `${window.location.protocol}//${
+        window.location.host
+      }${router.asPath.replace(/\?.+/, "")}`
+    );
   }, [router.query]);
 
   if (suite === undefined || compiler === undefined) {
@@ -1153,7 +1183,7 @@ export default function Page() {
               startTime.toString()
             )}&stopTime=${encodeURIComponent(
               stopTime.toString()
-            )}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}` +
+            )}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}` +
             (model === undefined ? "" : `&model=${model}`)
           }
         />
@@ -1166,6 +1196,7 @@ export default function Page() {
           setStopTime={setStopTime}
           timeRange={timeRange}
           setTimeRange={setTimeRange}
+          setGranularity={setGranularity}
         />
         <GranularityPicker
           granularity={granularity}

diff --git a/torchci/pages/benchmark/compilers.tsx b/torchci/pages/benchmark/compilers.tsx
@@ -43,6 +43,7 @@ const ROW_GAP = 100;
 const ROW_HEIGHT = 38;
 const PASSRATE_DISPLAY_NAME_REGEX = new RegExp("^([0-9]+)%,\\s.+$");
 
+export const SHA_DISPLAY_LENGTH = 10;
 export const LAST_N_DAYS = 7;
 export const HUD_PREFIX = "/pytorch/pytorch/commit";
 export const TIME_FIELD_NAME = "granularity_bucket";
@@ -681,7 +682,7 @@ export function BranchAndCommitPicker({
         >
           {branches[branch].map((r: any) => (
             <MenuItem key={r.head_sha} value={r.head_sha}>
-              {r.head_sha.substring(0, 7)} (
+              {r.head_sha.substring(0, SHA_DISPLAY_LENGTH)} (
               {dayjs(r.event_time).format("YYYY/MM/DD")})
             </MenuItem>
           ))}
@@ -716,15 +717,21 @@ export function LogLinks({
 }
 
 function CommitPanel({
-  branch,
-  commit,
+  lBranch,
+  lCommit,
+  lDate,
+  rBranch,
+  rCommit,
+  rDate,
   workflowId,
-  date,
 }: {
-  branch: string;
-  commit: string;
+  lBranch: string;
+  lCommit: string;
+  lDate: string;
+  rBranch: string;
+  rCommit: string;
+  rDate: string;
   workflowId: number;
-  date: string;
 }) {
   const queryCollection = "commons";
   const queryName = "get_workflow_jobs";
@@ -780,12 +787,17 @@ function CommitPanel({
   return (
     <Stack direction="row" spacing={2} sx={{ mb: 2 }}>
       <Typography fontSize={"1rem"} fontStyle={"italic"}>
-        *This report was generated by CI running on PyTorch {branch} branch at
+        *This report was generated by CI running on PyTorch {lBranch} branch at
         commit{" "}
-        <a href={`${HUD_PREFIX}/${commit}#inductor-a100-perf-nightly`}>
-          {commit.substring(0, 7)}
+        <a href={`${HUD_PREFIX}/${lCommit}#inductor-a100-perf-nightly`}>
+          {lCommit.substring(0, SHA_DISPLAY_LENGTH)}
         </a>{" "}
-        on {dayjs(date).format("YYYY/MM/DD")}. The running logs per shard are:{" "}
+        on {dayjs(lDate).format("YYYY/MM/DD")} comparing with {rBranch} branch
+        at commit{" "}
+        <a href={`${HUD_PREFIX}/${rCommit}#inductor-a100-perf-nightly`}>
+          {rCommit.substring(0, SHA_DISPLAY_LENGTH)}
+        </a>
+        . The running logs per shard are:{" "}
         {Object.keys(SUITES).map((suite: string) => {
           // Hack alert: The test configuration uses timm instead of timm_model as its output
           const name = suite.includes("timm") ? "timm" : suite;
@@ -900,6 +912,7 @@ function extractPercentage(value: string) {
 function SummaryPanel({
   startTime,
   stopTime,
+  granularity,
   mode,
   dtype,
   lBranch,
@@ -911,6 +924,7 @@ function SummaryPanel({
 }: {
   startTime: dayjs.Dayjs;
   stopTime: dayjs.Dayjs;
+  granularity: Granularity;
   mode: string;
   dtype: string;
   lBranch: string;
@@ -1009,7 +1023,7 @@ function SummaryPanel({
                     const url = `/benchmark/${suite}/${
                       DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ??
                       params.row.compiler
-                    }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
+                    }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
 
                     const l = extractPercentage(v.l);
                     const r = extractPercentage(v.r);
@@ -1097,7 +1111,7 @@ function SummaryPanel({
                     const url = `/benchmark/${suite}/${
                       DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ??
                       params.row.compiler
-                    }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
+                    }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
 
                     const l = Number(v.l).toFixed(SCALE);
                     const r = Number(v.r).toFixed(SCALE);
@@ -1188,7 +1202,7 @@ function SummaryPanel({
                     const url = `/benchmark/${suite}/${
                       DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ??
                       params.row.compiler
-                    }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
+                    }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
 
                     const l = Number(v.l).toFixed(0);
                     const r = Number(v.r).toFixed(0);
@@ -1292,7 +1306,7 @@ function SummaryPanel({
                     const url = `/benchmark/${suite}/${
                       DISPLAY_NAMES_TO_COMPILER_NAMES[params.row.compiler] ??
                       params.row.compiler
-                    }?startTime=${startTime}&stopTime=${stopTime}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
+                    }?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`;
 
                     const l = Number(v.l).toFixed(SCALE);
                     const r = Number(v.r).toFixed(SCALE);
@@ -1753,14 +1767,18 @@ function Report({
   return (
     <div>
       <CommitPanel
-        branch={lBranch}
-        commit={lCommit}
+        lBranch={lBranch}
+        lCommit={lCommit}
+        lDate={lData[0].granularity_bucket}
+        rBranch={rBranch}
+        rCommit={rCommit}
+        rDate={rData[0].granularity_bucket}
         workflowId={lData[0].workflow_id}
-        date={lData[0].granularity_bucket}
       />
       <SummaryPanel
         startTime={startTime}
         stopTime={stopTime}
+        granularity={granularity}
         mode={mode}
         dtype={dtype}
         lBranch={lBranch}
@@ -1821,6 +1839,12 @@ export default function Page() {
       }
     }
 
+    const granularity: Granularity =
+      (router.query.granularity as Granularity) ?? undefined;
+    if (granularity !== undefined) {
+      setGranularity(granularity);
+    }
+
     const suite: string = (router.query.suite as string) ?? undefined;
     if (suite !== undefined) {
       setSuite(suite);
@@ -1856,7 +1880,11 @@ export default function Page() {
       setRCommit(rCommit);
     }
 
-    setBaseUrl(`${window.location.host}${router.asPath.replace(/\?.+/, "")}`);
+    setBaseUrl(
+      `${window.location.protocol}//${
+        window.location.host
+      }${router.asPath.replace(/\?.+/, "")}`
+    );
   }, [router.query]);
 
   const queryParams: RocksetParam[] = [
@@ -1903,7 +1931,7 @@ export default function Page() {
             startTime.toString()
           )}&stopTime=${encodeURIComponent(
             stopTime.toString()
-          )}&suite=${suite}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`}
+          )}&granularity=${granularity}&suite=${suite}&mode=${mode}&dtype=${dtype}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`}
         />
       </Stack>
       <Stack direction="row" spacing={2} sx={{ mb: 2 }}>
@@ -1914,6 +1942,7 @@ export default function Page() {
           setStopTime={setStopTime}
           timeRange={timeRange}
           setTimeRange={setTimeRange}
+          setGranularity={setGranularity}
         />
         <GranularityPicker
           granularity={granularity}