Run profile queries after fetching columns (#2677)

* Run profile queries after fetching columns * Fixing prettier * Moving query matching to a new file * Fix lint * Updated E2E * Fix source profiling * TopLevelProfiling => TableProfiling
rilldata · Jul 17, 2023 · 8719932 · 8719932 · github-actions · Jul 17, 2023
1 parent 59a91f4
commit 8719932
Show file tree

Hide file tree

Showing 12 changed files with 243 additions and 77 deletions.
diff --git a/web-common/src/components/column-profile/ColumnProfile.svelte b/web-common/src/components/column-profile/ColumnProfile.svelte
@@ -52,7 +52,7 @@
     nestedColumnProfileQuery = getSummaries(
       objectName,
       $runtime?.instanceId,
-      $profileColumns?.data?.profileColumns
+      $profileColumns
     );
   }
 
@@ -83,11 +83,11 @@
     <option value={sortByName}>sort by name</option>
   </select>
   <select
-    style:transform="translateX(4px)"
     bind:value={mode}
     class={NATIVE_SELECT}
     class:hidden={containerWidth < 325}
     style:font-size="11px"
+    style:transform="translateX(4px)"
   >
     <option value="summaries">show summary&nbsp;</option>
     <option value="example">show example</option>
@@ -113,6 +113,7 @@
         {hideRight}
         {hideNullPercentage}
         {compact}
+        enableProfiling={!$profileColumns?.isFetching}
       />
     {/each}
   {/if}

diff --git a/web-common/src/components/column-profile/column-types/NestedProfile.svelte b/web-common/src/components/column-profile/column-types/NestedProfile.svelte
@@ -28,19 +28,32 @@
   export let compact = false;
   export let hideNullPercentage = false;
 
+  export let enableProfiling = true;
+
   let topKLimit = 15;
 
   let active = false;
 
-  $: nulls = getNullPercentage($runtime?.instanceId, objectName, columnName);
+  $: nulls = getNullPercentage(
+    $runtime?.instanceId,
+    objectName,
+    columnName,
+    enableProfiling
+  );
 
   $: columnCardinality = getCountDistinct(
     $runtime?.instanceId,
     objectName,
-    columnName
+    columnName,
+    enableProfiling
   );
 
-  $: topK = getTopK($runtime?.instanceId, objectName, columnName);
+  $: topK = getTopK(
+    $runtime?.instanceId,
+    objectName,
+    columnName,
+    enableProfiling
+  );
 
   function toggleColumnProfile() {
     active = !active;
@@ -51,20 +64,20 @@
 </script>
 
 <ProfileContainer
-  isFetching={fetchingSummaries}
   {active}
   {compact}
   emphasize={active}
   {example}
   {hideNullPercentage}
   {hideRight}
+  isFetching={fetchingSummaries}
   {mode}
   on:select={toggleColumnProfile}
   on:shift-click={() =>
     copyToClipboard(columnName, `copied ${columnName} to clipboard`)}
   {type}
 >
-  <ColumnProfileIcon slot="icon" isFetching={fetchingSummaries} {type} />
+  <ColumnProfileIcon isFetching={fetchingSummaries} slot="icon" {type} />
 
   <svelte:fragment slot="left">{columnName}</svelte:fragment>
 
@@ -83,15 +96,15 @@
   />
   <div
     class="pl-10 pr-4 py-4"
-    slot="details"
     class:hidden={INTERVALS.has(type)}
+    slot="details"
   >
     <TopK
-      {type}
-      topK={$topK}
+      colorClass={DATA_TYPE_COLORS["STRUCT"].bgClass}
       k={topKLimit}
+      topK={$topK}
       totalRows={$columnCardinality?.totalRows}
-      colorClass={DATA_TYPE_COLORS["STRUCT"].bgClass}
+      {type}
     />
   </div>
 </ProfileContainer>
diff --git a/web-common/src/components/column-profile/column-types/NumericProfile.svelte b/web-common/src/components/column-profile/column-types/NumericProfile.svelte
@@ -37,15 +37,23 @@
   export let compact = false;
   export let hideNullPercentage = false;
 
+  export let enableProfiling = true;
+
   let active = false;
 
-  $: nulls = getNullPercentage($runtime?.instanceId, objectName, columnName);
+  $: nulls = getNullPercentage(
+    $runtime?.instanceId,
+    objectName,
+    columnName,
+    enableProfiling
+  );
 
   $: diagnosticHistogram = getNumericHistogram(
     $runtime?.instanceId,
     objectName,
     columnName,
     QueryServiceColumnNumericHistogramHistogramMethod.HISTOGRAM_METHOD_DIAGNOSTIC,
+    enableProfiling,
     active
   );
   let fdHistogram;
@@ -55,6 +63,7 @@
       objectName,
       columnName,
       QueryServiceColumnNumericHistogramHistogramMethod.HISTOGRAM_METHOD_FD,
+      enableProfiling,
       active
     );
   }
@@ -81,10 +90,16 @@
         select($query) {
           return $query?.numericSummary?.numericOutliers?.outliers;
         },
+        enabled: enableProfiling,
       },
     }
   );
-  $: topK = getTopK($runtime?.instanceId, objectName, columnName);
+  $: topK = getTopK(
+    $runtime?.instanceId,
+    objectName,
+    columnName,
+    enableProfiling
+  );
 
   $: summary = derived(
     createQueryServiceColumnDescriptiveStatistics(
@@ -93,6 +108,11 @@
       {
         columnName: columnName,
         priority: getPriorityForColumn("descriptive-statistics", active),
+      },
+      {
+        query: {
+          enabled: enableProfiling,
+        },
       }
     ),
     ($query) => {
@@ -146,23 +166,23 @@
 </script>
 
 <ProfileContainer
-  isFetching={fetchingSummaries}
   {active}
   {compact}
   emphasize={active}
   {example}
   {hideNullPercentage}
   {hideRight}
+  isFetching={fetchingSummaries}
   {mode}
   on:select={toggleColumnProfile}
   on:shift-click={() =>
     copyToClipboard(columnName, `copied ${columnName} to clipboard`)}
   {type}
 >
-  <ColumnProfileIcon slot="icon" isFetching={fetchingSummaries} {type} />
+  <ColumnProfileIcon isFetching={fetchingSummaries} slot="icon" {type} />
 
   <svelte:fragment slot="left">{columnName}</svelte:fragment>
-  <NumericSpark {type} {compact} data={histogramData} slot="summary" />
+  <NumericSpark {compact} data={histogramData} slot="summary" {type} />
   <NullPercentageSpark
     nullCount={$nulls?.nullCount}
     slot="nullity"
@@ -171,8 +191,8 @@
   />
   <div
     class="pl-10 pr-4 py-4"
-    slot="details"
     class:hidden={INTERVALS.has(type)}
+    slot="details"
   >
     <NumericPlot
       data={histogramData}

diff --git a/web-common/src/components/column-profile/column-types/TimestampProfile.svelte b/web-common/src/components/column-profile/column-types/TimestampProfile.svelte
@@ -25,6 +25,8 @@
   export let compact = false;
   export let hideNullPercentage = false;
 
+  export let enableProfiling = true;
+
   let timestampDetailHeight = 160;
 
   let active = false;
@@ -33,13 +35,15 @@
   $: nullPercentage = getNullPercentage(
     $runtime?.instanceId,
     objectName,
-    columnName
+    columnName,
+    enableProfiling
   );
 
   $: timeSeries = getTimeSeriesAndSpark(
     $runtime?.instanceId,
     objectName,
     columnName,
+    enableProfiling,
     active
   );
 
@@ -52,20 +56,20 @@
 </script>
 
 <ProfileContainer
-  isFetching={fetchingSummaries}
   {active}
   {compact}
   emphasize={active}
   {example}
   {hideNullPercentage}
   {hideRight}
+  isFetching={fetchingSummaries}
   {mode}
   on:select={toggleColumnProfile}
   on:shift-click={() =>
     copyToClipboard(columnName, `copied ${columnName} to clipboard`)}
   {type}
 >
-  <ColumnProfileIcon slot="icon" {type} isFetching={fetchingSummaries} />
+  <ColumnProfileIcon isFetching={fetchingSummaries} slot="icon" {type} />
   <div slot="left">{columnName}</div>
 
   <!-- wrap in div to get size of grid item -->

diff --git a/web-common/src/components/column-profile/column-types/VarcharProfile.svelte b/web-common/src/components/column-profile/column-types/VarcharProfile.svelte
@@ -24,19 +24,33 @@
   export let hideNullPercentage = false;
   export let mode: "example" | "summaries" = "summaries";
 
+  export let enableProfiling = true;
+
   let active = false;
 
   let topKLimit = 15;
 
-  $: nulls = getNullPercentage($runtime?.instanceId, objectName, columnName);
+  $: nulls = getNullPercentage(
+    $runtime?.instanceId,
+    objectName,
+    columnName,
+    enableProfiling
+  );
 
   $: columnCardinality = getCountDistinct(
     $runtime?.instanceId,
     objectName,
-    columnName
+    columnName,
+    enableProfiling
   );
 
-  $: topK = getTopK($runtime?.instanceId, objectName, columnName, active);
+  $: topK = getTopK(
+    $runtime?.instanceId,
+    objectName,
+    columnName,
+    enableProfiling,
+    active
+  );
 
   function toggleColumnProfile() {
     active = !active;
@@ -47,19 +61,19 @@
 </script>
 
 <ProfileContainer
-  isFetching={fetchingSummaries}
   {active}
   emphasize={active}
   {example}
   {hideNullPercentage}
   {hideRight}
+  isFetching={fetchingSummaries}
   {mode}
   on:select={toggleColumnProfile}
   on:shift-click={() =>
     copyToClipboard(columnName, `copied ${columnName} to clipboard`)}
   {type}
 >
-  <ColumnProfileIcon slot="icon" {type} isFetching={fetchingSummaries} />
+  <ColumnProfileIcon isFetching={fetchingSummaries} slot="icon" {type} />
   <svelte:fragment slot="left">{columnName}</svelte:fragment>
 
   <ColumnCardinalitySpark
@@ -84,10 +98,10 @@
   >
     <div>
       <TopK
-        {type}
-        topK={$topK}
         k={topKLimit}
+        topK={$topK}
         totalRows={$columnCardinality?.totalRows}
+        {type}
       />
     </div>
   </div>

diff --git a/web-common/src/components/column-profile/column-types/details/TopK.svelte b/web-common/src/components/column-profile/column-types/details/TopK.svelte
@@ -44,6 +44,11 @@
       ? format("0.1%")
       : () => "";
 
+  // We need this to get transition working properly.
+  // Since the topk query is in a reactive statement with `enable`, `topK` can be undefined.
+  // This leads to unexpected issues when paired with transition
+  $: topKCopy = topK ?? topKCopy;
+
   function ensureSpaces(str: string, n = 6) {
     return `${Array.from({ length: n - str.length })
       .fill("&nbsp;")
@@ -67,9 +72,9 @@
   /** handle LISTs and STRUCTs */
 </script>
 
-{#if topK && totalRows}
+{#if topKCopy && totalRows}
   <div transition:slide|local={{ duration: LIST_SLIDE_DURATION }}>
-    {#each topK.slice(0, k) as item (item.value)}
+    {#each topKCopy.slice(0, k) as item (item.value)}
       {@const negligiblePercentage = item.count / totalRows < 0.0002}
       {@const percentage = negligiblePercentage
         ? "<.01%"