From 3046c56871e6b28a450092eed770354e6a79cbd5 Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Tue, 7 Nov 2023 15:04:55 +0530 Subject: [PATCH 1/4] Fix Project deployment status to show loading while dashboard list is loading. (#3392) * Fix Project deployment status to show loading * Showing a spinner when loading initially --- .../projects/ProjectDeploymentStatusChip.svelte | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte b/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte index d0e9623dc81..efc991a05d0 100644 --- a/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte +++ b/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte @@ -97,17 +97,6 @@ textClass: "text-purple-600", wrapperClass: "bg-purple-50 border-purple-300", }, - // [V1DeploymentStatus.DEPLOYMENT_STATUS_RECONCILING]: { - // icon: Spinner, - // iconProps: { - // bg: "linear-gradient(90deg, #22D3EE -0.5%, #6366F1 98.5%)", - // className: "text-purple-600 hover:text-purple-500", - // status: EntityStatus.Running, - // }, - // text: "syncing", - // textClass: "text-purple-600", - // wrapperClass: "bg-purple-50 border-purple-300", - // }, [V1DeploymentStatus.DEPLOYMENT_STATUS_ERROR]: { icon: CancelCircle, iconProps: { className: "text-red-600 hover:text-red-500" }, @@ -142,7 +131,11 @@ } -{#if deploymentStatus} +{#if $deploymentStatusFromDashboards.isFetching && !$deploymentStatusFromDashboards?.data} +
+ +
+{:else if deploymentStatus} {#if iconOnly}
Date: Tue, 7 Nov 2023 15:55:22 +0530 Subject: [PATCH 2/4] Fixing dashbaord refresh due to source refresh (#3391) * Fixing dashbaord refresh due to source refresh * Fix long refresh blocking dashboard usage --- .../dashboards/workspace/DashboardCTAs.svelte | 2 +- .../entity-management/resource-status-utils.ts | 16 +++++++++++----- .../(application)/dashboard/[name]/+page.svelte | 3 ++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte b/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte index 8f05f99c4f1..20a869d4d59 100644 --- a/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte +++ b/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte @@ -30,7 +30,7 @@ $: dashboardQuery = useDashboard($runtime.instanceId, metricViewName); $: dashboardIsIdle = - $dashboardQuery.data.meta.reconcileStatus === + $dashboardQuery.data?.meta?.reconcileStatus === V1ReconcileStatus.RECONCILE_STATUS_IDLE; function viewMetrics(metricViewName: string) { diff --git a/web-common/src/features/entity-management/resource-status-utils.ts b/web-common/src/features/entity-management/resource-status-utils.ts index af119d71b13..6fc1f3f8fd5 100644 --- a/web-common/src/features/entity-management/resource-status-utils.ts +++ b/web-common/src/features/entity-management/resource-status-utils.ts @@ -162,7 +162,8 @@ export function waitForResourceUpdate( export function getResourceStatusStore( queryClient: QueryClient, instanceId: string, - filePath: string + filePath: string, + validator?: (res: V1Resource) => boolean ): Readable { return derived( [ @@ -189,10 +190,15 @@ export function getResourceStatusStore( }; } - const isBusy = - resourceResp.isFetching || - resourceResp.data?.meta?.reconcileStatus !== - V1ReconcileStatus.RECONCILE_STATUS_IDLE; + let isBusy: boolean; + if (validator && resourceResp.data) { + isBusy = !validator(resourceResp.data); + } else { + isBusy = + resourceResp.isFetching || + resourceResp.data?.meta?.reconcileStatus !== + V1ReconcileStatus.RECONCILE_STATUS_IDLE; + } return { status: isBusy ? ResourceStatus.Busy : ResourceStatus.Idle, diff --git a/web-local/src/routes/(application)/dashboard/[name]/+page.svelte b/web-local/src/routes/(application)/dashboard/[name]/+page.svelte index 829acb0fd2a..58c5346a72d 100644 --- a/web-local/src/routes/(application)/dashboard/[name]/+page.svelte +++ b/web-local/src/routes/(application)/dashboard/[name]/+page.svelte @@ -45,7 +45,8 @@ $: resourceStatusStore = getResourceStatusStore( queryClient, $runtime.instanceId, - filePath + filePath, + (res) => !!res?.metricsView?.state?.validSpec ); let showErrorPage = false; $: if (metricViewName) { From 5bc2efb6a9505464444ddd5ad1b4a6f6439d723d Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Tue, 7 Nov 2023 16:05:41 +0530 Subject: [PATCH 3/4] Adding support for round_to_grain for APIs (#3362) * Adding support for round_to_grain for APIs * Adding DST tests * Adding the time controls to report * Fix lint * Fixing tests * PR comments * Fix tests --- runtime/queries/metricsview_aggregation.go | 2 +- .../queries/metricsview_comparison_toplist.go | 61 +------ runtime/queries/metricsview_timeseries.go | 61 ------- .../queries/metricsview_timeseries_test.go | 70 +------- runtime/queries/timeutil.go | 153 ++++++++++++++++++ runtime/queries/timeutil_test.go | 134 +++++++++++++++ runtime/reconcilers/report.go | 12 ++ 7 files changed, 310 insertions(+), 183 deletions(-) create mode 100644 runtime/queries/timeutil.go create mode 100644 runtime/queries/timeutil_test.go diff --git a/runtime/queries/metricsview_aggregation.go b/runtime/queries/metricsview_aggregation.go index bb7c768231d..5185ca561d0 100644 --- a/runtime/queries/metricsview_aggregation.go +++ b/runtime/queries/metricsview_aggregation.go @@ -204,7 +204,7 @@ func (q *MetricsViewAggregation) buildMetricsAggregationSQL(mv *runtimev1.Metric whereClause := "" if mv.TimeDimension != "" { timeCol := safeName(mv.TimeDimension) - clause, err := timeRangeClause(q.TimeRange, dialect, timeCol, &args) + clause, err := timeRangeClause(q.TimeRange, mv, dialect, timeCol, &args) if err != nil { return "", nil, err } diff --git a/runtime/queries/metricsview_comparison_toplist.go b/runtime/queries/metricsview_comparison_toplist.go index e63f362d6e3..9dffbad8d10 100644 --- a/runtime/queries/metricsview_comparison_toplist.go +++ b/runtime/queries/metricsview_comparison_toplist.go @@ -6,12 +6,10 @@ import ( "fmt" "io" "strings" - "time" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" "github.com/rilldata/rill/runtime/drivers" - "github.com/rilldata/rill/runtime/pkg/duration" "github.com/rilldata/rill/runtime/pkg/pbutil" "google.golang.org/protobuf/types/known/structpb" @@ -270,7 +268,7 @@ func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsView args := []any{} td := safeName(mv.TimeDimension) - trc, err := timeRangeClause(q.TimeRange, dialect, td, &args) + trc, err := timeRangeClause(q.TimeRange, mv, dialect, td, &args) if err != nil { return "", nil, err } @@ -414,7 +412,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M td := safeName(mv.TimeDimension) - trc, err := timeRangeClause(q.TimeRange, dialect, td, &args) + trc, err := timeRangeClause(q.TimeRange, mv, dialect, td, &args) if err != nil { return "", nil, err } @@ -430,7 +428,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M args = append(args, clauseArgs...) } - trc, err = timeRangeClause(q.ComparisonTimeRange, dialect, td, &args) + trc, err = timeRangeClause(q.ComparisonTimeRange, mv, dialect, td, &args) if err != nil { return "", nil, err } @@ -835,60 +833,15 @@ func (q *MetricsViewComparison) generateFilename() string { // TODO: a) Ensure correct time zone handling, b) Implement support for tr.RoundToGrain // (Maybe consider pushing all this logic into the SQL instead?) -func timeRangeClause(tr *runtimev1.TimeRange, dialect drivers.Dialect, timeCol string, args *[]any) (string, error) { +func timeRangeClause(tr *runtimev1.TimeRange, mv *runtimev1.MetricsViewSpec, dialect drivers.Dialect, timeCol string, args *[]any) (string, error) { var clause string if isTimeRangeNil(tr) { return clause, nil } - tz := time.UTC - if tr.TimeZone != "" { - var err error - tz, err = time.LoadLocation(tr.TimeZone) - if err != nil { - return "", fmt.Errorf("invalid time_range.time_zone %q: %w", tr.TimeZone, err) - } - } - - var start, end time.Time - if tr.Start != nil { - start = tr.Start.AsTime().In(tz) - } - if tr.End != nil { - end = tr.End.AsTime().In(tz) - } - - if tr.IsoDuration != "" { - if !start.IsZero() && !end.IsZero() { - return "", fmt.Errorf("only two of time_range.{start,end,iso_duration} can be specified") - } - - d, err := duration.ParseISO8601(tr.IsoDuration) - if err != nil { - return "", fmt.Errorf("invalid iso_duration %q: %w", tr.IsoDuration, err) - } - - if !start.IsZero() { - end = d.Add(start) - } else if !end.IsZero() { - start = d.Sub(end) - } else { - return "", fmt.Errorf("one of time_range.{start,end} must be specified with time_range.iso_duration") - } - } - - if tr.IsoOffset != "" { - d, err := duration.ParseISO8601(tr.IsoOffset) - if err != nil { - return "", fmt.Errorf("invalid iso_offset %q: %w", tr.IsoOffset, err) - } - - if !start.IsZero() { - start = d.Add(start) - } - if !end.IsZero() { - end = d.Add(end) - } + start, end, err := ResolveTimeRange(tr, mv) + if err != nil { + return "", err } if !start.IsZero() { diff --git a/runtime/queries/metricsview_timeseries.go b/runtime/queries/metricsview_timeseries.go index e6ed93f30e1..78ff5e07881 100644 --- a/runtime/queries/metricsview_timeseries.go +++ b/runtime/queries/metricsview_timeseries.go @@ -347,67 +347,6 @@ func (q *MetricsViewTimeSeries) buildDuckDBSQL(args []any, mv *runtimev1.Metrics return sql } -func TruncateTime(start time.Time, tg runtimev1.TimeGrain, tz *time.Location, firstDay, firstMonth int) time.Time { - switch tg { - case runtimev1.TimeGrain_TIME_GRAIN_MILLISECOND: - return start.Truncate(time.Millisecond) - case runtimev1.TimeGrain_TIME_GRAIN_SECOND: - return start.Truncate(time.Second) - case runtimev1.TimeGrain_TIME_GRAIN_MINUTE: - return start.Truncate(time.Minute) - case runtimev1.TimeGrain_TIME_GRAIN_HOUR: - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), start.Day(), start.Hour(), 0, 0, 0, tz) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_DAY: - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_WEEK: - start = start.In(tz) - weekday := int(start.Weekday()) - if weekday == 0 { - weekday = 7 - } - if firstDay < 1 { - firstDay = 1 - } - if firstDay > 7 { - firstDay = 7 - } - - daysToSubtract := -(weekday - firstDay) - if weekday < firstDay { - daysToSubtract = -7 + daysToSubtract - } - start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) - start = start.AddDate(0, 0, daysToSubtract) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_MONTH: - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) - start = start.In(time.UTC) - return start - case runtimev1.TimeGrain_TIME_GRAIN_QUARTER: - monthsToSubtract := 1 - int(start.Month())%3 // todo first month of year - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) - start = start.AddDate(0, monthsToSubtract, 0) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_YEAR: - start = start.In(tz) - year := start.Year() - if int(start.Month()) < firstMonth { - year = start.Year() - 1 - } - - start = time.Date(year, time.Month(firstMonth), 1, 0, 0, 0, 0, tz) - return start.In(time.UTC) - } - - return start -} - func generateNullRecords(schema *runtimev1.StructType) *structpb.Struct { nullStruct := structpb.Struct{Fields: make(map[string]*structpb.Value, len(schema.Fields))} for _, f := range schema.Fields { diff --git a/runtime/queries/metricsview_timeseries_test.go b/runtime/queries/metricsview_timeseries_test.go index 29eb0a2eda5..1a4927e61da 100644 --- a/runtime/queries/metricsview_timeseries_test.go +++ b/runtime/queries/metricsview_timeseries_test.go @@ -4,7 +4,6 @@ import ( "context" // "fmt" "testing" - "time" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" @@ -138,7 +137,10 @@ func TestMetricsViewsTimeseries_quarter_grain_IST(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data + require.Len(t, rows, 6) i := 0 + require.Equal(t, parseTime(t, "2022-10-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ require.Equal(t, parseTime(t, "2022-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-03-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) @@ -179,69 +181,3 @@ func TestMetricsViewsTimeseries_year_grain_IST(t *testing.T) { i++ require.Equal(t, parseTime(t, "2023-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) } - -func TestTruncateTime(t *testing.T) { - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:00:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-04-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, time.UTC, 1, 1)) -} - -func TestTruncateTime_Kathmandu(t *testing.T) { - tz, err := time.LoadLocation("Asia/Kathmandu") - require.NoError(t, err) - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:15:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-06T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2023-10-08T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-31T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-03-31T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2018-12-31T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 1, 1)) -} - -func TestTruncateTime_UTC_first_day(t *testing.T) { - tz := time.UTC - require.Equal(t, parseTestTime(t, "2023-10-08T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) - require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T00:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) -} - -func TestTruncateTime_Kathmandu_first_day(t *testing.T) { - tz, err := time.LoadLocation("Asia/Kathmandu") - require.NoError(t, err) - require.Equal(t, parseTestTime(t, "2023-10-07T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-09T18:16:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) -} - -func TestTruncateTime_UTC_first_month(t *testing.T) { - tz := time.UTC - require.Equal(t, parseTestTime(t, "2023-02-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) - require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-03-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) - require.Equal(t, parseTestTime(t, "2023-01-01T00:00:00Z"), queries.TruncateTime(parseTestTime(t, "2023-01-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) -} - -func TestTruncateTime_Kathmandu_first_month(t *testing.T) { - tz, err := time.LoadLocation("Asia/Kathmandu") - require.NoError(t, err) - require.Equal(t, parseTestTime(t, "2023-01-31T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) - require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-03-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) - require.Equal(t, parseTestTime(t, "2022-12-31T18:15:00Z"), queries.TruncateTime(parseTestTime(t, "2023-01-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) -} - -func parseTestTime(tst *testing.T, t string) time.Time { - ts, err := time.Parse(time.RFC3339, t) - require.NoError(tst, err) - return ts -} diff --git a/runtime/queries/timeutil.go b/runtime/queries/timeutil.go new file mode 100644 index 00000000000..a0e2aca36b6 --- /dev/null +++ b/runtime/queries/timeutil.go @@ -0,0 +1,153 @@ +package queries + +import ( + "fmt" + "time" + + runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/pkg/duration" + + // Load IANA time zone data + _ "time/tzdata" +) + +func TruncateTime(start time.Time, tg runtimev1.TimeGrain, tz *time.Location, firstDay, firstMonth int) time.Time { + switch tg { + case runtimev1.TimeGrain_TIME_GRAIN_MILLISECOND: + return start.Truncate(time.Millisecond) + case runtimev1.TimeGrain_TIME_GRAIN_SECOND: + return start.Truncate(time.Second) + case runtimev1.TimeGrain_TIME_GRAIN_MINUTE: + return start.Truncate(time.Minute) + case runtimev1.TimeGrain_TIME_GRAIN_HOUR: + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), start.Day(), start.Hour(), 0, 0, 0, tz) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_DAY: + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_WEEK: + start = start.In(tz) + weekday := int(start.Weekday()) + if weekday == 0 { + weekday = 7 + } + if firstDay < 1 { + firstDay = 1 + } + if firstDay > 7 { + firstDay = 7 + } + + daysToSubtract := -(weekday - firstDay) + if weekday < firstDay { + daysToSubtract = -7 + daysToSubtract + } + start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) + start = start.AddDate(0, 0, daysToSubtract) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_MONTH: + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) + start = start.In(time.UTC) + return start + case runtimev1.TimeGrain_TIME_GRAIN_QUARTER: + monthsToSubtract := (3 + int(start.Month()) - firstMonth%3) % 3 + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) + start = start.AddDate(0, -monthsToSubtract, 0) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_YEAR: + start = start.In(tz) + year := start.Year() + if int(start.Month()) < firstMonth { + year = start.Year() - 1 + } + + start = time.Date(year, time.Month(firstMonth), 1, 0, 0, 0, 0, tz) + return start.In(time.UTC) + } + + return start +} + +func ResolveTimeRange(tr *runtimev1.TimeRange, mv *runtimev1.MetricsViewSpec) (time.Time, time.Time, error) { + tz := time.UTC + + if tr.TimeZone != "" { + var err error + tz, err = time.LoadLocation(tr.TimeZone) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid time_range.time_zone %q: %w", tr.TimeZone, err) + } + } + + var start, end time.Time + if tr.Start != nil { + start = tr.Start.AsTime().In(tz) + } + if tr.End != nil { + end = tr.End.AsTime().In(tz) + } + + isISO := false + + if tr.IsoDuration != "" { + if !start.IsZero() && !end.IsZero() { + return time.Time{}, time.Time{}, fmt.Errorf("only two of time_range.{start,end,iso_duration} can be specified") + } + + d, err := duration.ParseISO8601(tr.IsoDuration) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid iso_duration %q: %w", tr.IsoDuration, err) + } + + if !start.IsZero() { + end = d.Add(start) + } else if !end.IsZero() { + start = d.Sub(end) + } else { + return time.Time{}, time.Time{}, fmt.Errorf("one of time_range.{start,end} must be specified with time_range.iso_duration") + } + + isISO = true + } + + if tr.IsoOffset != "" { + d, err := duration.ParseISO8601(tr.IsoOffset) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid iso_offset %q: %w", tr.IsoOffset, err) + } + + if !start.IsZero() { + start = d.Add(start) + } + if !end.IsZero() { + end = d.Add(end) + } + + isISO = true + } + + // Only modify the start and end if ISO duration or offset was sent. + // This is to maintain backwards compatibility for calls from the UI. + if isISO { + fdow := int(mv.FirstDayOfWeek) + if mv.FirstDayOfWeek > 7 || mv.FirstDayOfWeek <= 0 { + fdow = 1 + } + fmoy := int(mv.FirstMonthOfYear) + if mv.FirstMonthOfYear > 12 || mv.FirstMonthOfYear <= 0 { + fmoy = 1 + } + if !start.IsZero() { + start = TruncateTime(start, tr.RoundToGrain, tz, fdow, fmoy) + } + if !end.IsZero() { + end = TruncateTime(end, tr.RoundToGrain, tz, fdow, fmoy) + } + } + + return start, end, nil +} diff --git a/runtime/queries/timeutil_test.go b/runtime/queries/timeutil_test.go new file mode 100644 index 00000000000..2b4dc407382 --- /dev/null +++ b/runtime/queries/timeutil_test.go @@ -0,0 +1,134 @@ +package queries + +import ( + "testing" + "time" + + runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestTruncateTime(t *testing.T) { + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-04-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, time.UTC, 1, 1)) +} + +func TestTruncateTime_Kathmandu(t *testing.T) { + tz, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:15:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-06T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2023-10-08T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-03-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2018-12-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 1, 1)) +} + +func TestTruncateTime_UTC_first_day(t *testing.T) { + tz := time.UTC + require.Equal(t, parseTestTime(t, "2023-10-08T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) + require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T00:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) +} + +func TestTruncateTime_Kathmandu_first_day(t *testing.T) { + tz, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.Equal(t, parseTestTime(t, "2023-10-07T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-09T18:16:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) +} + +func TestTruncateTime_UTC_first_month(t *testing.T) { + tz := time.UTC + require.Equal(t, parseTestTime(t, "2023-08-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-11-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 5)) + require.Equal(t, parseTestTime(t, "2023-09-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-09-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + + require.Equal(t, parseTestTime(t, "2023-02-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-03-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) + require.Equal(t, parseTestTime(t, "2023-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-01-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) +} + +func TestTruncateTime_Kathmandu_first_month(t *testing.T) { + tz, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.Equal(t, parseTestTime(t, "2023-07-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-10-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 5)) + require.Equal(t, parseTestTime(t, "2023-08-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-08-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + + require.Equal(t, parseTestTime(t, "2023-01-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-03-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) + require.Equal(t, parseTestTime(t, "2022-12-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-01-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) +} + +func TestResolveTimeRange(t *testing.T) { + cases := []struct { + title string + tr *runtimev1.TimeRange + start, end string + }{ + { + "day light savings start US/Canada", + &runtimev1.TimeRange{End: timeToPB("2023-03-12T12:00:00Z"), IsoDuration: "PT4H", TimeZone: "America/Los_Angeles"}, + "2023-03-12T08:00:00Z", + "2023-03-12T12:00:00Z", + }, + { + "day light savings end US/Canada", + &runtimev1.TimeRange{Start: timeToPB("2023-11-05T08:00:00.000Z"), IsoDuration: "PT4H", TimeZone: "America/Los_Angeles"}, + "2023-11-05T08:00:00Z", + "2023-11-05T12:00:00Z", + }, + { + "going through feb", + &runtimev1.TimeRange{Start: timeToPB("2023-01-05T00:00:00Z"), IsoDuration: "P1M"}, + "2023-01-05T00:00:00Z", + "2023-02-05T00:00:00Z", + }, + } + + for _, tc := range cases { + t.Run(tc.title, func(t *testing.T) { + start, end, err := ResolveTimeRange(tc.tr, &runtimev1.MetricsViewSpec{ + FirstDayOfWeek: 1, + FirstMonthOfYear: 1, + }) + require.NoError(t, err) + require.Equal(t, parseTestTime(t, tc.start), start.UTC()) + require.Equal(t, parseTestTime(t, tc.end), end.UTC()) + }) + } +} + +func timeToPB(t string) *timestamppb.Timestamp { + ts, _ := time.Parse(time.RFC3339, t) + return timestamppb.New(ts) +} + +func parseTestTime(tst *testing.T, t string) time.Time { + ts, err := time.Parse(time.RFC3339, t) + require.NoError(tst, err) + return ts +} diff --git a/runtime/reconcilers/report.go b/runtime/reconcilers/report.go index 9397f5323a1..59f1d9fe689 100644 --- a/runtime/reconcilers/report.go +++ b/runtime/reconcilers/report.go @@ -320,6 +320,7 @@ func buildQuery(rep *runtimev1.Report, t time.Time) (*runtimev1.Query, error) { if err != nil { return nil, fmt.Errorf("invalid properties for query %q: %w", rep.Spec.QueryName, err) } + req.TimeRange = overrideTimeRange(req.TimeRange, t) case "MetricsViewToplist": req := &runtimev1.MetricsViewToplistRequest{} qry.Query = &runtimev1.Query_MetricsViewToplistRequest{MetricsViewToplistRequest: req} @@ -348,6 +349,7 @@ func buildQuery(rep *runtimev1.Report, t time.Time) (*runtimev1.Query, error) { if err != nil { return nil, fmt.Errorf("invalid properties for query %q: %w", rep.Spec.QueryName, err) } + req.TimeRange = overrideTimeRange(req.TimeRange, t) default: return nil, fmt.Errorf("query %q not supported for reports", rep.Spec.QueryName) } @@ -367,3 +369,13 @@ func formatExportFormat(f runtimev1.ExportFormat) string { return f.String() } } + +func overrideTimeRange(tr *runtimev1.TimeRange, t time.Time) *runtimev1.TimeRange { + if tr == nil { + tr = &runtimev1.TimeRange{} + } + + tr.End = timestamppb.New(t) + + return tr +} From cc61866038040761064551887fcef5ea8203332f Mon Sep 17 00:00:00 2001 From: bcolloran Date: Tue, 7 Nov 2023 12:13:16 -0800 Subject: [PATCH 4/4] Update percent of total calculations (#3331) * some cleanups around the dimesion table data builder * starting to work on percent of total calculations * cleaning up leaderboard in preparation for new totals query * move leaderboard query out to a selector * lint fix * move memoizeMetricsStore to own file (fixes vitest problem) * fix type check in memoizeMetricsStore * fix types * enable updated percentage calculation in leaderboard, and lots of cleanup * allow cancelling queries within actions + related cleanup * cleanups * clean up leaderboard display component * cleanup * cleanup in dimension table * implement correct percent of total calculations for dimension table * cleanup * cleanup * lint cleanup * fix search in dimension table * updating names * cleanup * additional post-merge cleanups * fix comparisons --- .../features/dashboards/dashboard-utils.ts | 5 +- .../dimension-table/DimensionDisplay.svelte | 170 +++++------------- .../DimensionFilterGutter.svelte | 9 +- .../dimension-table/DimensionHeader.svelte | 62 ++++--- .../dimension-table/DimensionTable.svelte | 23 ++- .../dimension-table/dimension-table-utils.ts | 143 +++++++++------ .../leaderboard/DimensionCompareMenu.svelte | 22 ++- .../dashboards/leaderboard/Leaderboard.svelte | 162 ++++------------- .../leaderboard/LeaderboardDisplay.svelte | 127 ++++--------- .../leaderboard/LeaderboardHeader.svelte | 33 ++-- .../leaderboard/LeaderboardListItem.svelte | 30 ++-- .../leaderboard/leaderboard-utils.ts | 72 +++++--- .../src/features/dashboards/selectors.ts | 37 ++-- .../state-managers/actions/comparison.ts | 21 +++ .../state-managers/actions/context-columns.ts | 18 +- .../state-managers/actions/core-actions.ts | 6 +- .../actions/dimension-filters.ts | 51 ++++++ .../state-managers/actions/dimension-table.ts | 53 ++++-- .../state-managers/actions/dimensions.ts | 18 ++ .../state-managers/actions/index.ts | 77 ++++++-- .../state-managers/actions/sorting.ts | 45 +++-- .../state-managers/actions/types.ts | 18 +- .../state-managers/memoize-metrics-store.ts | 21 +++ .../selectors/active-measure.ts | 64 +++++-- .../state-managers/selectors/comparisons.ts | 22 +++ .../selectors/context-column.ts | 16 +- .../selectors/dashboard-queries.ts | 153 ++++++++++++++++ .../selectors/data-formatting.ts | 6 +- .../selectors/dimension-filters.ts | 101 +++++++++++ .../selectors/dimension-table.ts | 132 ++++++++++++++ .../state-managers/selectors/dimensions.ts | 87 +++++++++ .../state-managers/selectors/index.ts | 112 ++++++++++-- .../state-managers/selectors/measures.ts | 26 +++ .../state-managers/selectors/sorting.ts | 11 +- .../state-managers/selectors/time-range.ts | 33 ++++ .../state-managers/selectors/types.ts | 21 ++- .../state-managers/state-managers.ts | 66 ++++--- .../dashboards/state-managers/types.ts | 8 + .../dashboards/stores/dashboard-stores.ts | 49 +++-- .../stores/metrics-explorer-entity.ts | 9 +- .../time-controls/time-control-store.ts | 141 ++++++++------- .../time-dimension-data-store.ts | 6 +- .../time-series/timeseries-data-store.ts | 6 +- .../dashboards/workspace/Dashboard.svelte | 7 +- .../number-formatting/format-measure-value.ts | 4 + web-local/src/lib/types.ts | 1 + 46 files changed, 1550 insertions(+), 754 deletions(-) create mode 100644 web-common/src/features/dashboards/state-managers/actions/comparison.ts create mode 100644 web-common/src/features/dashboards/state-managers/actions/dimension-filters.ts create mode 100644 web-common/src/features/dashboards/state-managers/actions/dimensions.ts create mode 100644 web-common/src/features/dashboards/state-managers/memoize-metrics-store.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/comparisons.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/dashboard-queries.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/dimension-filters.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/dimension-table.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/dimensions.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/measures.ts create mode 100644 web-common/src/features/dashboards/state-managers/selectors/time-range.ts create mode 100644 web-common/src/features/dashboards/state-managers/types.ts diff --git a/web-common/src/features/dashboards/dashboard-utils.ts b/web-common/src/features/dashboards/dashboard-utils.ts index abd4035c4b9..34854ca64a9 100644 --- a/web-common/src/features/dashboards/dashboard-utils.ts +++ b/web-common/src/features/dashboards/dashboard-utils.ts @@ -31,7 +31,8 @@ export function prepareSortedQueryBody( dimensionName: string, measureNames: string[], timeControls: TimeControlState, - sortMeasureName: string, + // Note: sortMeasureName may be null if we are sorting by dimension values + sortMeasureName: string | null, sortType: SortType, sortAscending: boolean, filterForDimension: V1MetricsViewFilter @@ -45,7 +46,7 @@ export function prepareSortedQueryBody( // Benjamin and Egor put in a patch that will allow us to use the // dimension name as the measure name. This will need to be updated // once they have stabilized the API. - if (sortType === SortType.DIMENSION) { + if (sortType === SortType.DIMENSION || sortMeasureName === null) { sortMeasureName = dimensionName; // note also that we need to remove the comparison time range // when sorting by dimension values, or the query errors diff --git a/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte b/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte index e44900271e6..26b28ccf418 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte @@ -6,81 +6,51 @@ * to be displayed in explore */ import { cancelDashboardQueries } from "@rilldata/web-common/features/dashboards/dashboard-queries"; - import { - useMetaDimension, - useMetaMeasure, - useMetaQuery, - } from "@rilldata/web-common/features/dashboards/selectors"; + import { getStateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; import { useTimeControlStore } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; import { createQueryServiceMetricsViewComparison, createQueryServiceMetricsViewTotals, - MetricsViewDimension, - MetricsViewSpecMeasureV2, } from "@rilldata/web-common/runtime-client"; import { useQueryClient } from "@tanstack/svelte-query"; - import { runtime } from "../../../runtime-client/runtime-store"; - import { - getDimensionFilterWithSearch, - prepareDimensionTableRows, - prepareVirtualizedDimTableColumns, - } from "./dimension-table-utils"; + import { getDimensionFilterWithSearch } from "./dimension-table-utils"; import DimensionHeader from "./DimensionHeader.svelte"; import DimensionTable from "./DimensionTable.svelte"; - import { - getDimensionColumn, - isSummableMeasure, - prepareSortedQueryBody, - } from "../dashboard-utils"; import { metricsExplorerStore } from "../stores/dashboard-stores"; - export let metricViewName: string; - export let dimensionName: string; - - let searchText = ""; - - const queryClient = useQueryClient(); - - $: instanceId = $runtime.instanceId; - - $: metaQuery = useMetaQuery(instanceId, metricViewName); - - $: dimensionQuery = useMetaDimension( - instanceId, - metricViewName, - dimensionName - ); - - let dimension: MetricsViewDimension; - $: dimension = $dimensionQuery?.data as MetricsViewDimension; - $: dimensionColumn = getDimensionColumn(dimension); const stateManagers = getStateManagers(); - const timeControlsStore = useTimeControlStore(stateManagers); - const { dashboardStore, selectors: { - sorting: { sortedAscending }, + dashboardQueries: { + dimensionTableSortedQueryBody, + dimensionTableTotalQueryBody, + }, + comparison: { isBeingCompared }, + dimensions: { dimensionTableDimName }, + dimensionTable: { + virtualizedTableColumns, + selectedDimensionValueNames, + prepareDimTableRows, + }, + activeMeasure: { activeMeasureName }, }, + metricsViewName, + runtime, } = stateManagers; - $: leaderboardMeasureName = $dashboardStore?.leaderboardMeasureName; - $: isBeingCompared = - $dashboardStore?.selectedComparisonDimension === dimensionName; + // cast is safe because dimensionTableDimName must be defined + // for the dimension table to be open + $: dimensionName = $dimensionTableDimName as string; - $: leaderboardMeasureQuery = useMetaMeasure( - instanceId, - metricViewName, - leaderboardMeasureName - ); + let searchText = ""; - $: validPercentOfTotal = ( - $leaderboardMeasureQuery?.data as MetricsViewSpecMeasureV2 - )?.validPercentOfTotal; + const queryClient = useQueryClient(); - $: excludeMode = - $dashboardStore?.dimensionFilterExcludeMode.get(dimensionName) ?? false; + $: instanceId = $runtime.instanceId; + + const timeControlsStore = useTimeControlStore(stateManagers); $: filterSet = getDimensionFilterWithSearch( $dashboardStore?.filters, @@ -88,26 +58,10 @@ dimensionName ); - $: selectedValues = - (excludeMode - ? $dashboardStore?.filters?.exclude?.find((d) => d.name === dimensionName) - ?.in - : $dashboardStore?.filters?.include?.find((d) => d.name === dimensionName) - ?.in) ?? []; - - $: visibleMeasures = - $metaQuery.data?.measures?.filter((m) => - $dashboardStore?.visibleMeasureKeys.has(m.name ?? "") - ) ?? []; - $: totalsQuery = createQueryServiceMetricsViewTotals( instanceId, - metricViewName, - { - measureNames: $dashboardStore?.selectedMeasureNames, - timeStart: $timeControlsStore.timeStart, - timeEnd: $timeControlsStore.timeEnd, - }, + $metricsViewName, + $dimensionTableTotalQueryBody, { query: { enabled: $timeControlsStore.ready, @@ -115,40 +69,14 @@ } ); - $: unfilteredTotal = $totalsQuery?.data?.data?.[leaderboardMeasureName] ?? 0; + $: unfilteredTotal = $totalsQuery?.data?.data?.[$activeMeasureName] ?? 0; - let referenceValues: { [key: string]: number } = {}; - $: if ($totalsQuery?.data?.data) { - visibleMeasures.map((m) => { - if (m.name && isSummableMeasure(m)) { - referenceValues[m.name] = $totalsQuery.data?.data?.[m.name]; - } - }); - } - - $: columns = prepareVirtualizedDimTableColumns( - $dashboardStore, - visibleMeasures, - referenceValues, - dimension, - $timeControlsStore?.showComparison ?? false, - validPercentOfTotal ?? false - ); - - $: sortedQueryBody = prepareSortedQueryBody( - dimensionName, - $dashboardStore?.selectedMeasureNames, - $timeControlsStore, - leaderboardMeasureName, - $dashboardStore.dashboardSortType, - $sortedAscending, - filterSet - ); + $: columns = $virtualizedTableColumns($totalsQuery); $: sortedQuery = createQueryServiceMetricsViewComparison( $runtime.instanceId, - metricViewName, - sortedQueryBody, + $metricsViewName, + $dimensionTableSortedQueryBody, { query: { enabled: $timeControlsStore.ready && !!filterSet, @@ -156,47 +84,39 @@ } ); - $: tableRows = prepareDimensionTableRows( - $sortedQuery?.data?.rows ?? [], - $metaQuery.data?.measures ?? [], - leaderboardMeasureName, - dimensionColumn, - $timeControlsStore.showComparison ?? false, - validPercentOfTotal ?? false, - unfilteredTotal - ); + $: tableRows = $prepareDimTableRows($sortedQuery, unfilteredTotal); $: areAllTableRowsSelected = tableRows.every((row) => - selectedValues.includes(row[dimensionColumn]) + $selectedDimensionValueNames.includes(row[dimensionName] as string) ); function onSelectItem(event) { - const label = tableRows[event.detail][dimensionColumn] as string; - cancelDashboardQueries(queryClient, metricViewName); - metricsExplorerStore.toggleFilter(metricViewName, dimensionName, label); + const label = tableRows[event.detail][dimensionName] as string; + cancelDashboardQueries(queryClient, $metricsViewName); + metricsExplorerStore.toggleFilter($metricsViewName, dimensionName, label); } function toggleComparisonDimension(dimensionName, isBeingCompared) { metricsExplorerStore.setComparisonDimension( - metricViewName, + $metricsViewName, isBeingCompared ? undefined : dimensionName ); } function toggleAllSearchItems() { - const labels = tableRows.map((row) => row[dimensionColumn] as string); - cancelDashboardQueries(queryClient, metricViewName); + const labels = tableRows.map((row) => row[dimensionName] as string); + cancelDashboardQueries(queryClient, $metricsViewName); if (areAllTableRowsSelected) { metricsExplorerStore.deselectItemsInFilter( - metricViewName, + $metricsViewName, dimensionName, labels ); return; } else { metricsExplorerStore.selectItemsInFilter( - metricViewName, + $metricsViewName, dimensionName, labels ); @@ -208,9 +128,7 @@
- {#if tableRows && columns.length && dimensionColumn} + {#if tableRows && columns.length && dimensionName}
onSelectItem(event)} on:toggle-dimension-comparison={() => toggleComparisonDimension(dimensionName, isBeingCompared)} isFetching={$sortedQuery?.isFetching} - dimensionName={dimensionColumn} - {isBeingCompared} + {dimensionName} {columns} - {selectedValues} + selectedValues={$selectedDimensionValueNames} rows={tableRows} - {excludeMode} />
{/if} diff --git a/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte b/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte index 03a98d2988f..95041978c91 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte @@ -10,6 +10,7 @@ import { getContext } from "svelte"; import type { VirtualizedTableConfig } from "../../../components/virtualized-table/types"; import DimensionCompareMenu from "@rilldata/web-common/features/dashboards/leaderboard/DimensionCompareMenu.svelte"; + import { getStateManagers } from "../state-managers/state-managers"; export let totalHeight: number; export let virtualRowItems; @@ -18,6 +19,12 @@ export let isBeingCompared = false; export let atLeastOneActive = false; + const { + selectors: { + dimensions: { dimensionTableDimName }, + }, + } = getStateManagers(); + function getInsertIndex(arr, num) { return arr .concat(num) @@ -63,7 +70,7 @@ style:height="{config.columnHeaderHeight}px" class="sticky left-0 top-0 surface z-40 flex items-center" > - +
{#each virtualRowItems as row (`row-${row.key}`)} {@const isSelected = selectedIndex.includes(row.index)} diff --git a/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte b/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte index 6fae70df440..77d9c6ef3f1 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte @@ -13,7 +13,6 @@ import { EntityStatus } from "@rilldata/web-common/features/entity-management/types"; import { slideRight } from "@rilldata/web-common/lib/transitions"; import { useQueryClient } from "@tanstack/svelte-query"; - import { createEventDispatcher } from "svelte"; import { fly } from "svelte/transition"; import Spinner from "../../entity-management/Spinner.svelte"; import { metricsExplorerStore } from "web-common/src/features/dashboards/stores/dashboard-stores"; @@ -21,53 +20,66 @@ import { getStateManagers } from "../state-managers/state-managers"; import { SortType } from "../proto-state/derived-types"; import Button from "@rilldata/web-common/components/button/Button.svelte"; + import { createEventDispatcher } from "svelte"; - export let metricViewName: string; export let dimensionName: string; export let isFetching: boolean; - export let excludeMode = false; export let areAllTableRowsSelected = false; export let isRowsEmpty = true; + const dispatch = createEventDispatcher(); + const stateManagers = getStateManagers(); const { selectors: { sorting: { sortedByDimensionValue }, + dimensionTable: { dimensionTableSearchString }, + dimensionFilters: { isFilterExcludeMode }, }, actions: { sorting: { toggleSort }, + dimensionTable: { + setDimensionTableSearchString, + clearDimensionTableSearchString, + }, + dimensions: { setPrimaryDimension }, }, + metricsViewName, } = stateManagers; + $: excludeMode = $isFilterExcludeMode(dimensionName); + const queryClient = useQueryClient(); $: filterKey = excludeMode ? "exclude" : "include"; $: otherFilterKey = excludeMode ? "include" : "exclude"; - let searchToggle = false; - - const dispatch = createEventDispatcher(); + let searchBarOpen = false; - let searchText = ""; + // FIXME: this extra `searchText` variable should be eliminated, + // but there is no way to make the component a fully + // "controlled" component for now, so we have to go through the + // `value` binding it exposes. + let searchText: string | undefined = undefined; + $: searchText = $dimensionTableSearchString; function onSearch() { - dispatch("search", searchText); + setDimensionTableSearchString(searchText); } function closeSearchBar() { - searchText = ""; - searchToggle = !searchToggle; - onSearch(); + clearDimensionTableSearchString(); + searchBarOpen = false; } const goBackToLeaderboard = () => { - metricsExplorerStore.setMetricDimensionName(metricViewName, null); if ($sortedByDimensionValue) { toggleSort(SortType.VALUE); } + setPrimaryDimension(undefined); }; function toggleFilterMode() { - cancelDashboardQueries(queryClient, metricViewName); - metricsExplorerStore.toggleFilterMode(metricViewName, dimensionName); + cancelDashboardQueries(queryClient, $metricsViewName); + metricsExplorerStore.toggleFilterMode($metricsViewName, dimensionName); } @@ -96,16 +108,7 @@ {areAllTableRowsSelected ? "Deselect all" : "Select all"} {/if} - {#if !searchToggle} - - {:else} + {#if searchBarOpen || (searchText && searchText !== "")}
+ {:else} + {/if} @@ -136,6 +148,6 @@ - +
diff --git a/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte b/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte index 64844b50d8d..380a901cc1d 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte @@ -24,20 +24,23 @@ TableCells – the cell contents. export let rows: DimensionTableRow[]; export let columns: VirtualizedTableColumns[]; - export let selectedValues: Array = []; + export let selectedValues: string[]; export let dimensionName: string; - export let excludeMode = false; - export let isBeingCompared = false; export let isFetching: boolean; const { - actions: { dimTable }, + actions: { dimensionTable }, selectors: { sorting: { sortMeasure }, + dimensionFilters: { isFilterExcludeMode }, + comparison: { isBeingCompared: isBeingComparedReadable }, }, } = getStateManagers(); + $: excludeMode = $isFilterExcludeMode(dimensionName); + $: isBeingCompared = $isBeingComparedReadable(dimensionName); + /** the overscan values tell us how much to render off-screen. These may be set by the consumer * in certain circumstances. The tradeoff: the higher the overscan amount, the more DOM elements we have * to render on initial load. @@ -58,7 +61,7 @@ TableCells – the cell contents. const CHARACTER_LIMIT_FOR_WRAPPING = 9; const FILTER_COLUMN_WIDTH = config.indexWidth; - $: selectedIndices = selectedValues + $: selectedIndex = selectedValues .map((label) => { return rows.findIndex((row) => row[dimensionName] === label); }) @@ -87,6 +90,8 @@ TableCells – the cell contents. let estimateColumnSize: number[] = []; /* Separate out dimension column */ + // SAFETY: cast should be safe because if dimensionName is undefined, + // we should not be in a dimension table sub-component $: dimensionColumn = columns?.find( (c) => c.name == dimensionName ) as VirtualizedTableColumns; @@ -181,7 +186,7 @@ TableCells – the cell contents. async function handleColumnHeaderClick(event) { colScrollOffset = $columnVirtualizer.scrollOffset; const columnName = event.detail; - dimTable.handleMeasureColumnHeaderClick(columnName); + dimensionTable.handleMeasureColumnHeaderClick(columnName); } async function handleResizeDimensionColumn(event) { @@ -238,7 +243,7 @@ TableCells – the cell contents. 0} @@ -253,7 +258,7 @@ TableCells – the cell contents. column={dimensionColumn} {rows} {activeIndex} - selectedIndex={selectedIndices} + {selectedIndex} {excludeMode} {scrolling} {horizontalScrolling} @@ -270,7 +275,7 @@ TableCells – the cell contents. columns={measureColumns} {rows} {activeIndex} - selectedIndex={selectedIndices} + {selectedIndex} {scrolling} {excludeMode} on:select-item={(event) => onSelectItem(event)} diff --git a/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts b/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts index 8c05849606f..56c52c61932 100644 --- a/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts +++ b/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts @@ -7,6 +7,7 @@ import type { MetricsViewDimension, MetricsViewSpecMeasureV2, V1MetricsViewComparisonRow, + V1MetricsViewComparisonValue, V1MetricsViewFilter, V1MetricsViewToplistResponseDataItem, } from "../../../runtime-client"; @@ -17,7 +18,7 @@ import type { VirtualizedTableConfig } from "@rilldata/web-common/components/vir import type { SvelteComponent } from "svelte"; import { getDimensionColumn } from "../dashboard-utils"; import type { DimensionTableRow } from "./dimension-table-types"; -import { getFilterForDimension } from "../selectors"; +import { getFiltersForOtherDimensions } from "../selectors"; import { SortType } from "../proto-state/derived-types"; import type { MetricsExplorerEntity } from "../stores/metrics-explorer-entity"; import { createMeasureValueFormatter } from "@rilldata/web-common/lib/number-formatting/format-measure-value"; @@ -26,10 +27,10 @@ import { formatMeasurePercentageDifference } from "@rilldata/web-common/lib/numb /** Returns an updated filter set for a given dimension on search */ export function updateFilterOnSearch( - filterForDimension, - searchText, - dimensionName -) { + filterForDimension: V1MetricsViewFilter, + searchText: string, + dimensionName: string +): V1MetricsViewFilter { const filterSet = JSON.parse(JSON.stringify(filterForDimension)); const addNull = "null".includes(searchText); if (searchText !== "") { @@ -64,7 +65,10 @@ export function getDimensionFilterWithSearch( searchText: string, dimensionName: string ) { - const filterForDimension = getFilterForDimension(filters, dimensionName); + const filterForDimension = getFiltersForOtherDimensions( + filters, + dimensionName + ); return updateFilterOnSearch(filterForDimension, searchText, dimensionName); } @@ -231,13 +235,14 @@ export function prepareVirtualizedDimTableColumns( const selectedMeasure = allMeasures.find( (m) => m.name === leaderboardMeasureName ); + const dimensionColumn = getDimensionColumn(dimension); // copy column names so we don't mutate the original const columnNames = [...dash.visibleMeasureKeys]; // don't add context columns if sorting by dimension - if (sortType !== SortType.DIMENSION) { + if (selectedMeasure && sortType !== SortType.DIMENSION) { addContextColumnNames( columnNames, timeComparison, @@ -248,20 +253,20 @@ export function prepareVirtualizedDimTableColumns( // Make dimension the first column columnNames.unshift(dimensionColumn); - return columnNames + const columns = columnNames .map((name) => { let highlight = false; if (sortType === SortType.DIMENSION) { highlight = name === dimensionColumn; } else { highlight = - name === selectedMeasure.name || + name === selectedMeasure?.name || name.endsWith("_delta") || name.endsWith("_delta_perc") || name.endsWith("_percent_of_total"); } - let sorted = undefined; + let sorted; if (name.endsWith("_delta") && sortType === SortType.DELTA_ABSOLUTE) { sorted = sortDirection; } else if ( @@ -274,19 +279,23 @@ export function prepareVirtualizedDimTableColumns( sortType === SortType.PERCENT ) { sorted = sortDirection; - } else if (name === selectedMeasure.name && sortType === SortType.VALUE) { + } else if ( + name === selectedMeasure?.name && + sortType === SortType.VALUE + ) { sorted = sortDirection; } + let columnOut: VirtualizedTableColumns | undefined = undefined; if (measureNames.includes(name)) { // Handle all regular measures const measure = allMeasures.find((m) => m.name === name); - return { + columnOut = { name, type: "INT", label: measure?.label || measure?.expression, description: measure?.description, - total: referenceValues[measure.name] || 0, + total: referenceValues[measure?.name ?? ""] || 0, enableResize: false, format: measure?.formatPreset, highlight, @@ -294,7 +303,7 @@ export function prepareVirtualizedDimTableColumns( }; } else if (name === dimensionColumn) { // Handle dimension column - return { + columnOut = { name, type: "VARCHAR", label: dimension?.label, @@ -305,7 +314,7 @@ export function prepareVirtualizedDimTableColumns( } else if (selectedMeasure) { // Handle delta and delta_perc const comparison = getComparisonProperties(name, selectedMeasure); - return { + columnOut = { name, type: comparison.type, label: comparison.component, @@ -316,9 +325,12 @@ export function prepareVirtualizedDimTableColumns( sorted, }; } - return undefined; + return columnOut; }) - .filter((column) => !!column); + .filter((column) => column !== undefined); + + // cast is safe, because we filtered out undefined columns + return (columns as VirtualizedTableColumns[]) ?? []; } /** @@ -334,6 +346,7 @@ export function addContextColumnNames( selectedMeasure: MetricsViewSpecMeasureV2 ) { const name = selectedMeasure?.name; + if (!name) return; const sortByColumnIndex = columnNames.indexOf(name); // Add comparison columns if available @@ -379,55 +392,67 @@ export function prepareDimensionTableRows( ): DimensionTableRow[] { if (!queryRows || !queryRows.length) return []; - const formattersForMeasures = Object.fromEntries( - allMeasuresForSpec.map((m) => [m.name, createMeasureValueFormatter(m)]) - ); - - const tableRows: DimensionTableRow[] = queryRows.map((row) => { - if (!row.measureValues) return {}; - - const rawVals: [string, number][] = row.measureValues.map((m) => { - return [m.measureName ?? "", m.baseValue ? (m.baseValue as number) : 0]; - }); - - const formattedVals: [string, string | number][] = rawVals.map( - ([name, val]) => ["__formatted_" + name, formattersForMeasures[name](val)] + const formattersForMeasures: { [key: string]: (val: number) => string } = + Object.fromEntries( + allMeasuresForSpec.map((m) => [m.name, createMeasureValueFormatter(m)]) ); - const rowOut: DimensionTableRow = Object.fromEntries([ - [dimensionColumn, row.dimensionValue as string], - ...rawVals, - ...formattedVals, - ]); + const tableRows: DimensionTableRow[] = queryRows + .filter( + (row) => row.measureValues !== undefined && row.measureValues !== null + ) + .map((row) => { + // cast is safe since we filtered out rows without measureValues + const measureValues = row.measureValues as V1MetricsViewComparisonValue[]; + + const rawVals: [string, number][] = measureValues.map((m) => [ + m.measureName?.toString() ?? "", + m.baseValue ? (m.baseValue as number) : 0, + ]); + + const formattedVals: [string, string | number][] = rawVals.map( + ([name, val]) => [ + "__formatted_" + name, + formattersForMeasures[name](val), + ] + ); - const activeMeasure = row.measureValues?.find( - (m) => m.measureName === activeMeasureName - ); + const rowOut: DimensionTableRow = Object.fromEntries([ + [dimensionColumn, row.dimensionValue as string], + ...rawVals, + ...formattedVals, + ]); - if (addDeltas && activeMeasure) { - rowOut[`${activeMeasureName}_delta`] = activeMeasure.deltaAbs - ? formattersForMeasures[activeMeasureName](activeMeasure.deltaAbs) - : PERC_DIFF.PREV_VALUE_NO_DATA; + const activeMeasure = measureValues.find( + (m) => m.measureName === activeMeasureName + ); - rowOut[`${activeMeasureName}_delta_perc`] = activeMeasure.deltaRel - ? formatMeasurePercentageDifference(activeMeasure.deltaRel as number) - : PERC_DIFF.PREV_VALUE_NO_DATA; - } + if (addDeltas && activeMeasure) { + rowOut[`${activeMeasureName}_delta`] = activeMeasure.deltaAbs + ? formattersForMeasures[activeMeasureName]( + activeMeasure.deltaAbs as number + ) + : PERC_DIFF.PREV_VALUE_NO_DATA; - if (addPercentOfTotal && activeMeasure) { - const value = activeMeasure.baseValue as number; + rowOut[`${activeMeasureName}_delta_perc`] = activeMeasure.deltaRel + ? formatMeasurePercentageDifference(activeMeasure.deltaRel as number) + : PERC_DIFF.PREV_VALUE_NO_DATA; + } - if (unfilteredTotal === 0 || !unfilteredTotal) { - rowOut[activeMeasureName + "_percent_of_total"] = - PERC_DIFF.CURRENT_VALUE_NO_DATA; - } else { - rowOut[activeMeasureName + "_percent_of_total"] = - formatMeasurePercentageDifference(value / unfilteredTotal); + if (addPercentOfTotal && activeMeasure) { + const value = activeMeasure.baseValue as number; + + if (unfilteredTotal === 0 || !unfilteredTotal) { + rowOut[activeMeasureName + "_percent_of_total"] = + PERC_DIFF.CURRENT_VALUE_NO_DATA; + } else { + rowOut[activeMeasureName + "_percent_of_total"] = + formatMeasurePercentageDifference(value / unfilteredTotal); + } } - } - return rowOut; - }); + return rowOut; + }); return tableRows; } @@ -439,8 +464,8 @@ export function getSelectedRowIndicesFromFilters( ): number[] { const selectedDimValues = ((excludeMode - ? filters.exclude.find((d) => d.name === dimensionName)?.in - : filters.include.find((d) => d.name === dimensionName) + ? filters?.exclude?.find((d) => d.name === dimensionName)?.in + : filters?.include?.find((d) => d.name === dimensionName) ?.in) as string[]) ?? []; return selectedDimValues diff --git a/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte b/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte index d4f4ac60f27..3961d868981 100644 --- a/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte +++ b/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte @@ -3,16 +3,30 @@ import Compare from "@rilldata/web-common/components/icons/Compare.svelte"; import Tooltip from "@rilldata/web-common/components/tooltip/Tooltip.svelte"; import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; - import { createEventDispatcher } from "svelte"; + import { getStateManagers } from "../state-managers/state-managers"; - export let isBeingCompared = false; + export let dimensionName: string | undefined; - const dispatch = createEventDispatcher(); + const { + selectors: { + comparison: { isBeingCompared: isBeingComparedReadable }, + }, + actions: { + comparison: { setComparisonDimension }, + }, + } = getStateManagers(); + + $: isBeingCompared = + dimensionName !== undefined && $isBeingComparedReadable(dimensionName); { - dispatch("toggle-dimension-comparison"); + if (isBeingCompared) { + setComparisonDimension(undefined); + } else { + setComparisonDimension(dimensionName); + } e.stopPropagation(); }} > diff --git a/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte b/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte index 86e9cf8fa76..0683f231cc7 100644 --- a/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte +++ b/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte @@ -7,24 +7,12 @@ */ import Tooltip from "@rilldata/web-common/components/tooltip/Tooltip.svelte"; import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; - import { - getFilterForDimension, - useMetaDimension, - useMetaMeasure, - } from "@rilldata/web-common/features/dashboards/selectors"; import { getStateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; - import { useTimeControlStore } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; import { createQueryServiceMetricsViewComparison, - MetricsViewDimension, - MetricsViewSpecMeasureV2, + createQueryServiceMetricsViewTotals, } from "@rilldata/web-common/runtime-client"; - import { runtime } from "../../../runtime-client/runtime-store"; - import { SortDirection } from "../proto-state/derived-types"; - import { - metricsExplorerStore, - useDashboardStore, - } from "web-common/src/features/dashboards/stores/dashboard-stores"; + import LeaderboardHeader from "./LeaderboardHeader.svelte"; import { LeaderboardItemData, @@ -32,125 +20,63 @@ prepareLeaderboardItemData, } from "./leaderboard-utils"; import LeaderboardListItem from "./LeaderboardListItem.svelte"; - import { prepareSortedQueryBody } from "../dashboard-utils"; - export let metricViewName: string; export let dimensionName: string; /** The reference value is the one that the bar in the LeaderboardListItem * gets scaled with. For a summable metric, the total is a reference value, * or for a count(*) metric, the reference value is the total number of rows. */ export let referenceValue: number; - export let unfilteredTotal: number; let slice = 7; - const stateManagers = getStateManagers(); - - $: dashboardStore = useDashboardStore(metricViewName); - - let filterExcludeMode: boolean; - $: filterExcludeMode = - $dashboardStore?.dimensionFilterExcludeMode.get(dimensionName) ?? false; - let filterKey: "exclude" | "include"; - $: filterKey = filterExcludeMode ? "exclude" : "include"; + const { + selectors: { + activeMeasure: { activeMeasureName }, + dimensionFilters: { selectedDimensionValues, isFilterExcludeMode }, + dashboardQueries: { + leaderboardSortedQueryBody, + leaderboardSortedQueryOptions, + leaderboardDimensionTotalQueryBody, + leaderboardDimensionTotalQueryOptions, + }, + }, + actions: { + dimensions: { setPrimaryDimension }, + }, + metricsViewName, + runtime, + } = getStateManagers(); - $: dimensionQuery = useMetaDimension( + $: sortedQuery = createQueryServiceMetricsViewComparison( $runtime.instanceId, - metricViewName, - dimensionName + $metricsViewName, + $leaderboardSortedQueryBody(dimensionName), + $leaderboardSortedQueryOptions(dimensionName) ); - let dimension: MetricsViewDimension; - $: dimension = $dimensionQuery?.data; - $: displayName = dimension?.label || dimension?.name || dimensionName; - $: measureQuery = useMetaMeasure( + $: totalsQuery = createQueryServiceMetricsViewTotals( $runtime.instanceId, - metricViewName, - $dashboardStore?.leaderboardMeasureName - ); - let measure: MetricsViewSpecMeasureV2; - $: measure = $measureQuery?.data; - - $: filterForDimension = getFilterForDimension( - $dashboardStore?.filters, - dimensionName - ); - - // FIXME: it is possible for this way of accessing the filters - // to return the same value twice, which would seem to indicate - // a bug in the way we're setting the filters / active values. - // Need to investigate further to determine whether this is a - // problem with the runtime or the client, but for now wrapping - // it in a set dedupes the values. - $: activeValues = new Set( - ($dashboardStore?.filters[filterKey]?.find( - (d) => d.name === dimension?.name - )?.in as (number | string)[]) ?? [] - ); - $: atLeastOneActive = activeValues?.size > 0; - - const timeControlsStore = useTimeControlStore(stateManagers); - - function selectDimension(dimensionName) { - metricsExplorerStore.setMetricDimensionName(metricViewName, dimensionName); - } - - function toggleComparisonDimension(dimensionName, isBeingCompared) { - metricsExplorerStore.setComparisonDimension( - metricViewName, - isBeingCompared ? undefined : dimensionName - ); - } - - function toggleSort(evt) { - metricsExplorerStore.toggleSort(metricViewName, evt.detail); - } - - $: isBeingCompared = - $dashboardStore?.selectedComparisonDimension === dimensionName; - - $: sortAscending = $dashboardStore.sortDirection === SortDirection.ASCENDING; - $: sortType = $dashboardStore.dashboardSortType; - - $: sortedQueryBody = prepareSortedQueryBody( - dimensionName, - [measure?.name], - $timeControlsStore, - measure?.name, - sortType, - sortAscending, - filterForDimension + $metricsViewName, + $leaderboardDimensionTotalQueryBody(dimensionName), + $leaderboardDimensionTotalQueryOptions(dimensionName) ); - $: sortedQueryEnabled = $timeControlsStore.ready && !!filterForDimension; - - $: sortedQueryOptions = { - query: { - enabled: sortedQueryEnabled, - }, - }; - - $: sortedQuery = createQueryServiceMetricsViewComparison( - $runtime.instanceId, - metricViewName, - sortedQueryBody, - sortedQueryOptions - ); + $: leaderboardTotal = $totalsQuery?.data?.data?.[$activeMeasureName]; let aboveTheFold: LeaderboardItemData[] = []; let selectedBelowTheFold: LeaderboardItemData[] = []; let noAvailableValues = true; let showExpandTable = false; - $: if (!$sortedQuery?.isFetching) { + $: if (sortedQuery && !$sortedQuery?.isFetching) { const leaderboardData = prepareLeaderboardItemData( $sortedQuery?.data?.rows?.map((r) => - getLabeledComparisonFromComparisonRow(r, measure.name) + getLabeledComparisonFromComparisonRow(r, $activeMeasureName) ) ?? [], slice, - [...activeValues], - unfilteredTotal, - filterExcludeMode + $selectedDimensionValues(dimensionName), + leaderboardTotal, + $isFilterExcludeMode(dimensionName) ); aboveTheFold = leaderboardData.aboveTheFold; @@ -162,7 +88,7 @@ let hovered: boolean; -{#if sortedQuery} +{#if $sortedQuery !== undefined}
(hovered = true)} @@ -170,28 +96,19 @@ > - toggleComparisonDimension(dimensionName, isBeingCompared)} - {isBeingCompared} + {dimensionName} {hovered} - dimensionDescription={dimension?.description || ""} - on:open-dimension-details={() => selectDimension(dimensionName)} - on:toggle-sort={toggleSort} /> {#if aboveTheFold || selectedBelowTheFold}
{#each aboveTheFold as itemData (itemData.dimensionValue)} {/each} @@ -199,14 +116,11 @@
{#each selectedBelowTheFold as itemData (itemData.dimensionValue)} {/each} @@ -224,7 +138,7 @@ {#if showExpandTable}
diff --git a/web-common/src/lib/number-formatting/format-measure-value.ts b/web-common/src/lib/number-formatting/format-measure-value.ts index 5867c8d24cd..cfe81fa8578 100644 --- a/web-common/src/lib/number-formatting/format-measure-value.ts +++ b/web-common/src/lib/number-formatting/format-measure-value.ts @@ -11,6 +11,10 @@ import { } from "./strategies/intervals"; import { humanizedFormatterFactory } from "./humanizer"; +export function defaultHumanizer(value: number): string { + return humanizeDataType(value, FormatPreset.HUMANIZE); +} + /** * This function is intended to provides a compact, * potentially lossy, humanized string representation of a number. diff --git a/web-local/src/lib/types.ts b/web-local/src/lib/types.ts index 0a4680dedd9..a2e4c3e1060 100644 --- a/web-local/src/lib/types.ts +++ b/web-local/src/lib/types.ts @@ -15,6 +15,7 @@ export interface VirtualizedTableColumns { // Is this the table sorted by this column, and if so, in what direction? // Leave undefined if the table is not sorted by this column. sorted?: SortDirection; + format?: string; } export type ProfileColumnSummary =