diff --git a/.github/workflows/rill-cloud.yml b/.github/workflows/rill-cloud.yml index be266bececc..2559b177501 100644 --- a/.github/workflows/rill-cloud.yml +++ b/.github/workflows/rill-cloud.yml @@ -45,9 +45,7 @@ jobs: gcloud auth configure-docker docker build -t gcr.io/rilldata/rill-headless:${GITHUB_SHA} . - docker tag gcr.io/rilldata/rill-headless:${GITHUB_SHA} gcr.io/rilldata/rill-headless docker push gcr.io/rilldata/rill-headless:${GITHUB_SHA} - docker push gcr.io/rilldata/rill-headless if [ ${RELEASE} == "true" ]; then docker tag gcr.io/rilldata/rill-headless:${GITHUB_SHA} gcr.io/rilldata/rill-headless:${GITHUB_REF_NAME} diff --git a/admin/deployments.go b/admin/deployments.go index e53658c09c8..24888e33976 100644 --- a/admin/deployments.go +++ b/admin/deployments.go @@ -64,7 +64,7 @@ func (s *Service) createDeployment(ctx context.Context, opts *createDeploymentOp olapConfig["cpu"] = strconv.Itoa(alloc.CPU) olapConfig["memory_limit_gb"] = strconv.Itoa(alloc.MemoryGB) olapConfig["storage_limit_bytes"] = strconv.FormatInt(alloc.StorageBytes, 10) - embedCatalog = true + embedCatalog = false case "duckdb-ext-storage": // duckdb driver having capability to store table as view if opts.ProdOLAPDSN != "" { return nil, fmt.Errorf("passing a DSN is not allowed for driver 'duckdb-ext-storage'") @@ -79,7 +79,7 @@ func (s *Service) createDeployment(ctx context.Context, opts *createDeploymentOp olapConfig["memory_limit_gb"] = strconv.Itoa(alloc.MemoryGB) olapConfig["storage_limit_bytes"] = strconv.FormatInt(alloc.StorageBytes, 10) olapConfig["external_table_storage"] = strconv.FormatBool(true) - embedCatalog = true + embedCatalog = false default: olapConfig["dsn"] = opts.ProdOLAPDSN embedCatalog = false diff --git a/cli/cmd/project/project.go b/cli/cmd/project/project.go index 97c528ac735..ca5f41d8691 100644 --- a/cli/cmd/project/project.go +++ b/cli/cmd/project/project.go @@ -27,6 +27,8 @@ func ProjectCmd(cfg *config.Config) *cobra.Command { projectCmd.AddCommand(DeleteCmd(cfg)) projectCmd.AddCommand(ListCmd(cfg)) projectCmd.AddCommand(ReconcileCmd(cfg)) + projectCmd.AddCommand(RefreshCmd(cfg)) + projectCmd.AddCommand(ResetCmd(cfg)) projectCmd.AddCommand(JwtCmd(cfg)) projectCmd.AddCommand(RenameCmd(cfg)) projectCmd.AddCommand(LogsCmd(cfg)) diff --git a/cli/cmd/project/reconcile.go b/cli/cmd/project/reconcile.go index a49839a21c2..609dc2d765d 100644 --- a/cli/cmd/project/reconcile.go +++ b/cli/cmd/project/reconcile.go @@ -11,13 +11,14 @@ import ( func ReconcileCmd(cfg *config.Config) *cobra.Command { var project, path string - var refresh, reset bool + var refresh, reset, force bool var refreshSources []string reconcileCmd := &cobra.Command{ Use: "reconcile []", Args: cobra.MaximumNArgs(1), Short: "Send trigger to deployment", + Hidden: true, PersistentPreRunE: cmdutil.CheckChain(cmdutil.CheckAuth(cfg), cmdutil.CheckOrganization(cfg)), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -49,6 +50,13 @@ func ReconcileCmd(cfg *config.Config) *cobra.Command { } if reset || resp.ProdDeployment == nil { + if !force { + msg := "This will create a new deployment, causing downtime as data sources are reloaded from scratch. If you just need to refresh data, use `rill project refresh`. Do you want to continue?" + if !cmdutil.ConfirmPrompt(msg, "", false) { + return nil + } + } + _, err = client.TriggerRedeploy(ctx, &adminv1.TriggerRedeployRequest{Organization: cfg.Org, Project: project}) if err != nil { return err @@ -85,6 +93,7 @@ func ReconcileCmd(cfg *config.Config) *cobra.Command { reconcileCmd.Flags().BoolVar(&refresh, "refresh", false, "Refresh all sources") reconcileCmd.Flags().StringSliceVar(&refreshSources, "refresh-source", nil, "Refresh specific source(s)") reconcileCmd.Flags().BoolVar(&reset, "reset", false, "Reset and redeploy the project from scratch") + reconcileCmd.Flags().BoolVar(&force, "force", false, "Force the operation") reconcileCmd.MarkFlagsMutuallyExclusive("reset", "refresh") reconcileCmd.MarkFlagsMutuallyExclusive("reset", "refresh-source") diff --git a/cli/cmd/project/refresh.go b/cli/cmd/project/refresh.go new file mode 100644 index 00000000000..7064e4afa18 --- /dev/null +++ b/cli/cmd/project/refresh.go @@ -0,0 +1,67 @@ +package project + +import ( + "fmt" + + "github.com/rilldata/rill/cli/pkg/cmdutil" + "github.com/rilldata/rill/cli/pkg/config" + adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1" + "github.com/spf13/cobra" +) + +func RefreshCmd(cfg *config.Config) *cobra.Command { + var project, path string + var source []string + + refreshCmd := &cobra.Command{ + Use: "refresh []", + Args: cobra.MaximumNArgs(1), + Short: "Refresh project", + PersistentPreRunE: cmdutil.CheckChain(cmdutil.CheckAuth(cfg), cmdutil.CheckOrganization(cfg)), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + client, err := cmdutil.Client(cfg) + if err != nil { + return err + } + defer client.Close() + + if len(args) > 0 { + project = args[0] + } + + if !cmd.Flags().Changed("project") && len(args) == 0 && cfg.Interactive { + var err error + project, err = inferProjectName(ctx, client, cfg.Org, path) + if err != nil { + return err + } + } + + resp, err := client.GetProject(ctx, &adminv1.GetProjectRequest{ + OrganizationName: cfg.Org, + Name: project, + }) + if err != nil { + return err + } + + _, err = client.TriggerRefreshSources(ctx, &adminv1.TriggerRefreshSourcesRequest{DeploymentId: resp.ProdDeployment.Id, Sources: source}) + if err != nil { + return fmt.Errorf("failed to trigger refresh: %w", err) + } + + fmt.Printf("Triggered refresh. To see status, run `rill project status --project %s`.\n", project) + + return nil + }, + } + + refreshCmd.Flags().SortFlags = false + refreshCmd.Flags().StringVar(&project, "project", "", "Project name") + refreshCmd.Flags().StringVar(&path, "path", ".", "Project directory") + refreshCmd.Flags().StringSliceVar(&source, "source", nil, "Refresh specific source(s)") + + return refreshCmd +} diff --git a/cli/cmd/project/reset.go b/cli/cmd/project/reset.go new file mode 100644 index 00000000000..e6de1452b1c --- /dev/null +++ b/cli/cmd/project/reset.go @@ -0,0 +1,65 @@ +package project + +import ( + "fmt" + + "github.com/rilldata/rill/cli/pkg/cmdutil" + "github.com/rilldata/rill/cli/pkg/config" + adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1" + "github.com/spf13/cobra" +) + +func ResetCmd(cfg *config.Config) *cobra.Command { + var project, path string + var force bool + + resetCmd := &cobra.Command{ + Use: "reset []", + Args: cobra.MaximumNArgs(1), + Short: "Reset project", + PersistentPreRunE: cmdutil.CheckChain(cmdutil.CheckAuth(cfg), cmdutil.CheckOrganization(cfg)), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + client, err := cmdutil.Client(cfg) + if err != nil { + return err + } + defer client.Close() + + if len(args) > 0 { + project = args[0] + } + + if !cmd.Flags().Changed("project") && len(args) == 0 && cfg.Interactive { + var err error + project, err = inferProjectName(ctx, client, cfg.Org, path) + if err != nil { + return err + } + } + + if !force { + msg := "This will create a new deployment, causing downtime as data sources are reloaded from scratch. If you just need to refresh data, use `rill project refresh`. Do you want to continue?" + if !cmdutil.ConfirmPrompt(msg, "", false) { + return nil + } + } + + _, err = client.TriggerRedeploy(ctx, &adminv1.TriggerRedeployRequest{Organization: cfg.Org, Project: project}) + if err != nil { + return err + } + + fmt.Printf("Triggered project reset. To see status, run `rill project status --project %s`.\n", project) + + return nil + }, + } + + resetCmd.Flags().SortFlags = false + resetCmd.Flags().StringVar(&project, "project", "", "Project name") + resetCmd.Flags().StringVar(&path, "path", ".", "Project directory") + resetCmd.Flags().BoolVar(&force, "force", false, "Force reset even if project is already deployed") + return resetCmd +} diff --git a/cli/pkg/local/app.go b/cli/pkg/local/app.go index e5108f2bcfa..116632cbd30 100644 --- a/cli/pkg/local/app.go +++ b/cli/pkg/local/app.go @@ -194,15 +194,6 @@ func NewApp(ctx context.Context, ver config.Version, verbose, strict, reset bool activity: client, } - // Wait for the initial reconcile - if isInit { - err = app.AwaitInitialReconcile(strict) - if err != nil { - app.Close() - return nil, fmt.Errorf("reconcile project: %w", err) - } - } - return app, nil } @@ -226,80 +217,6 @@ func (a *App) Close() error { return nil } -func (a *App) AwaitInitialReconcile(strict bool) (err error) { - defer func() { - if a.Context.Err() != nil { - a.Logger.Errorf("Hydration canceled") - err = nil - } - }() - - controller, err := a.Runtime.Controller(a.Context, a.Instance.ID) - if err != nil { - return err - } - - // We need to do some extra work to ensure we don't return until all resources have been reconciled. - // We can't call WaitUntilIdle until the parser has initially parsed and created the resources for the project. - // We know the global project parser is created immediately, and should only be IDLE initially or if a fatal error occurs with the watcher. - // So we poll for it's state to transition to Watching. - start := time.Now() - for { - if a.Context.Err() != nil { - return nil - } - - if time.Since(start) >= 5*time.Second { - // Starting the watcher should take just a few ms. This is just meant to serve as an extra safety net in case something goes wrong. - return fmt.Errorf("timed out waiting for project parser to start watching") - } - - r, err := controller.Get(a.Context, runtime.GlobalProjectParserName, false) - if err != nil { - return fmt.Errorf("could not find project parser: %w", err) - } - - if r.Meta.ReconcileStatus == runtimev1.ReconcileStatus_RECONCILE_STATUS_IDLE && r.Meta.ReconcileError != "" { - return fmt.Errorf("parser failed: %s", r.Meta.ReconcileError) - } - - if r.GetProjectParser().State.Watching { - break - } - - time.Sleep(100 * time.Millisecond) - } - - err = a.Runtime.WaitUntilIdle(a.Context, a.Instance.ID, true) - if err != nil { - return err - } - - rs, err := controller.List(a.Context, "", false) - if err != nil { - return err - } - - hasError := false - for _, r := range rs { - if r.Meta.ReconcileError != "" { - hasError = true - break - } - } - - if hasError { - a.Logger.Named("console").Errorf("Hydration failed") - if strict { - return fmt.Errorf("strict mode exit") - } - } else { - a.Logger.Named("console").Infof("Hydration completed!") - } - - return nil -} - func (a *App) Serve(httpPort, grpcPort int, enableUI, openBrowser, readonly bool, userID string) error { // Get analytics info installID, enabled, err := dotrill.AnalyticsInfo() diff --git a/proto/gen/rill/runtime/v1/resources.pb.go b/proto/gen/rill/runtime/v1/resources.pb.go index 1ae9197ac43..f3b54e67493 100644 --- a/proto/gen/rill/runtime/v1/resources.pb.go +++ b/proto/gen/rill/runtime/v1/resources.pb.go @@ -74,6 +74,58 @@ func (ReconcileStatus) EnumDescriptor() ([]byte, []int) { return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{0} } +type MetricsViewSpec_ComparisonMode int32 + +const ( + MetricsViewSpec_COMPARISON_MODE_UNSPECIFIED MetricsViewSpec_ComparisonMode = 0 + MetricsViewSpec_COMPARISON_MODE_NONE MetricsViewSpec_ComparisonMode = 1 + MetricsViewSpec_COMPARISON_MODE_TIME MetricsViewSpec_ComparisonMode = 2 + MetricsViewSpec_COMPARISON_MODE_DIMENSION MetricsViewSpec_ComparisonMode = 3 +) + +// Enum value maps for MetricsViewSpec_ComparisonMode. +var ( + MetricsViewSpec_ComparisonMode_name = map[int32]string{ + 0: "COMPARISON_MODE_UNSPECIFIED", + 1: "COMPARISON_MODE_NONE", + 2: "COMPARISON_MODE_TIME", + 3: "COMPARISON_MODE_DIMENSION", + } + MetricsViewSpec_ComparisonMode_value = map[string]int32{ + "COMPARISON_MODE_UNSPECIFIED": 0, + "COMPARISON_MODE_NONE": 1, + "COMPARISON_MODE_TIME": 2, + "COMPARISON_MODE_DIMENSION": 3, + } +) + +func (x MetricsViewSpec_ComparisonMode) Enum() *MetricsViewSpec_ComparisonMode { + p := new(MetricsViewSpec_ComparisonMode) + *p = x + return p +} + +func (x MetricsViewSpec_ComparisonMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MetricsViewSpec_ComparisonMode) Descriptor() protoreflect.EnumDescriptor { + return file_rill_runtime_v1_resources_proto_enumTypes[1].Descriptor() +} + +func (MetricsViewSpec_ComparisonMode) Type() protoreflect.EnumType { + return &file_rill_runtime_v1_resources_proto_enumTypes[1] +} + +func (x MetricsViewSpec_ComparisonMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MetricsViewSpec_ComparisonMode.Descriptor instead. +func (MetricsViewSpec_ComparisonMode) EnumDescriptor() ([]byte, []int) { + return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{13, 0} +} + type BucketExtractPolicy_Strategy int32 const ( @@ -107,11 +159,11 @@ func (x BucketExtractPolicy_Strategy) String() string { } func (BucketExtractPolicy_Strategy) Descriptor() protoreflect.EnumDescriptor { - return file_rill_runtime_v1_resources_proto_enumTypes[1].Descriptor() + return file_rill_runtime_v1_resources_proto_enumTypes[2].Descriptor() } func (BucketExtractPolicy_Strategy) Type() protoreflect.EnumType { - return &file_rill_runtime_v1_resources_proto_enumTypes[1] + return &file_rill_runtime_v1_resources_proto_enumTypes[2] } func (x BucketExtractPolicy_Strategy) Number() protoreflect.EnumNumber { @@ -1243,6 +1295,10 @@ type MetricsViewSpec struct { FirstDayOfWeek uint32 `protobuf:"varint,12,opt,name=first_day_of_week,json=firstDayOfWeek,proto3" json:"first_day_of_week,omitempty"` // Month number to use as the base for time aggregations by year. Defaults to 1 (January). FirstMonthOfYear uint32 `protobuf:"varint,13,opt,name=first_month_of_year,json=firstMonthOfYear,proto3" json:"first_month_of_year,omitempty"` + // Selected default comparison mode. + DefaultComparisonMode MetricsViewSpec_ComparisonMode `protobuf:"varint,14,opt,name=default_comparison_mode,json=defaultComparisonMode,proto3,enum=rill.runtime.v1.MetricsViewSpec_ComparisonMode" json:"default_comparison_mode,omitempty"` + // If comparison mode is dimension then this determines which is the default dimension + DefaultComparisonDimension string `protobuf:"bytes,15,opt,name=default_comparison_dimension,json=defaultComparisonDimension,proto3" json:"default_comparison_dimension,omitempty"` } func (x *MetricsViewSpec) Reset() { @@ -1368,6 +1424,20 @@ func (x *MetricsViewSpec) GetFirstMonthOfYear() uint32 { return 0 } +func (x *MetricsViewSpec) GetDefaultComparisonMode() MetricsViewSpec_ComparisonMode { + if x != nil { + return x.DefaultComparisonMode + } + return MetricsViewSpec_COMPARISON_MODE_UNSPECIFIED +} + +func (x *MetricsViewSpec) GetDefaultComparisonDimension() string { + if x != nil { + return x.DefaultComparisonDimension + } + return "" +} + type MetricsViewState struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2743,6 +2813,7 @@ type MetricsViewSpec_DimensionV2 struct { Column string `protobuf:"bytes,2,opt,name=column,proto3" json:"column,omitempty"` Label string `protobuf:"bytes,3,opt,name=label,proto3" json:"label,omitempty"` Description string `protobuf:"bytes,4,opt,name=description,proto3" json:"description,omitempty"` + Unnest bool `protobuf:"varint,5,opt,name=unnest,proto3" json:"unnest,omitempty"` } func (x *MetricsViewSpec_DimensionV2) Reset() { @@ -2805,6 +2876,13 @@ func (x *MetricsViewSpec_DimensionV2) GetDescription() string { return "" } +func (x *MetricsViewSpec_DimensionV2) GetUnnest() bool { + if x != nil { + return x.Unnest + } + return false +} + // Measures are aggregated computed values type MetricsViewSpec_MeasureV2 struct { state protoimpl.MessageState @@ -3255,7 +3333,7 @@ var file_rill_runtime_v1_resources_proto_rawDesc = []byte{ 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x22, 0xab, 0x0a, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, + 0x65, 0x22, 0xf6, 0x0c, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, @@ -3295,262 +3373,283 @@ var file_rill_runtime_v1_resources_proto_rawDesc = []byte{ 0x66, 0x69, 0x72, 0x73, 0x74, 0x44, 0x61, 0x79, 0x4f, 0x66, 0x57, 0x65, 0x65, 0x6b, 0x12, 0x2d, 0x0a, 0x13, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x5f, 0x6f, 0x66, 0x5f, 0x79, 0x65, 0x61, 0x72, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x66, 0x69, 0x72, - 0x73, 0x74, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x4f, 0x66, 0x59, 0x65, 0x61, 0x72, 0x1a, 0x71, 0x0a, - 0x0b, 0x44, 0x69, 0x6d, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x20, - 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x1a, 0xee, 0x01, 0x0a, 0x09, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x56, 0x32, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x50, 0x72, 0x65, 0x73, 0x65, 0x74, 0x12, - 0x1b, 0x0a, 0x09, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x64, 0x33, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x44, 0x33, 0x12, 0x33, 0x0a, 0x16, - 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6f, 0x66, - 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x76, 0x61, - 0x6c, 0x69, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4f, 0x66, 0x54, 0x6f, 0x74, 0x61, - 0x6c, 0x1a, 0xbb, 0x02, 0x0a, 0x0a, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x56, 0x32, - 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x6f, 0x77, 0x5f, - 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x6f, - 0x77, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x56, 0x0a, 0x07, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, - 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, - 0x69, 0x74, 0x79, 0x56, 0x32, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x52, 0x07, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x12, - 0x56, 0x0a, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3c, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x70, - 0x65, 0x63, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x56, 0x32, 0x2e, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x52, 0x07, - 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x1a, 0x46, 0x0a, 0x10, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x22, - 0x53, 0x0a, 0x10, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x73, 0x70, 0x65, - 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, - 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, 0x64, - 0x53, 0x70, 0x65, 0x63, 0x22, 0x76, 0x0a, 0x09, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x32, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1e, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, - 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x35, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x59, 0x0a, 0x0d, - 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, - 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x73, - 0x71, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x2a, 0x0a, 0x0e, 0x4d, 0x69, 0x67, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x22, 0x6d, 0x0a, 0x06, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2f, 0x0a, - 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x72, 0x69, - 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x70, 0x6f, 0x72, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x32, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, - 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x22, 0x94, 0x04, 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x70, 0x65, - 0x63, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74, - 0x69, 0x74, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x69, 0x74, 0x6c, - 0x65, 0x12, 0x44, 0x0a, 0x10, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x72, 0x69, - 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, - 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x26, 0x0a, 0x0f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x6a, 0x73, - 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x71, 0x75, 0x65, 0x72, 0x79, 0x41, - 0x72, 0x67, 0x73, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x70, 0x6f, 0x72, - 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x65, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x42, 0x0a, 0x0d, 0x65, 0x78, - 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, - 0x52, 0x0c, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x29, - 0x0a, 0x10, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, - 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x52, - 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4e, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, - 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, + 0x73, 0x74, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x4f, 0x66, 0x59, 0x65, 0x61, 0x72, 0x12, 0x67, 0x0a, + 0x17, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, + 0x73, 0x6f, 0x6e, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, - 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, - 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x90, 0x02, 0x0a, 0x0b, 0x52, 0x65, - 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3a, 0x0a, 0x0b, 0x6e, 0x65, 0x78, - 0x74, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x6e, 0x65, 0x78, 0x74, - 0x52, 0x75, 0x6e, 0x4f, 0x6e, 0x12, 0x4d, 0x0a, 0x11, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, - 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x10, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4d, 0x0a, 0x11, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x20, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x10, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x69, 0x73, 0x74, - 0x6f, 0x72, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x81, 0x02, 0x0a, - 0x0f, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x14, 0x0a, 0x05, 0x61, 0x64, 0x68, 0x6f, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x05, 0x61, 0x64, 0x68, 0x6f, 0x63, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3b, 0x0a, 0x0b, 0x72, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x72, 0x65, - 0x70, 0x6f, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, - 0x64, 0x4f, 0x6e, 0x12, 0x3b, 0x0a, 0x0b, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f, - 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x4f, 0x6e, - 0x22, 0x7c, 0x0a, 0x0b, 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, - 0x34, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, - 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, - 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x11, - 0x0a, 0x0f, 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x70, 0x65, - 0x63, 0x22, 0x12, 0x0a, 0x10, 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0x85, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, - 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, - 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, - 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, - 0x63, 0x12, 0x3a, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x24, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x52, 0x0a, - 0x12, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, - 0x70, 0x65, 0x63, 0x12, 0x3c, 0x0a, 0x0a, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, - 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x09, 0x6f, 0x6e, 0x6c, 0x79, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x22, 0x15, 0x0a, 0x13, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, - 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x0d, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x36, 0x0a, 0x04, 0x73, 0x70, - 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, - 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, - 0x65, 0x63, 0x12, 0x39, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, + 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x52, + 0x15, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, + 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x40, 0x0a, 0x1c, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x5f, 0x64, 0x69, 0x6d, + 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x44, + 0x69, 0x6d, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x89, 0x01, 0x0a, 0x0b, 0x44, 0x69, 0x6d, + 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, + 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, + 0x75, 0x6e, 0x6e, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, 0x6e, + 0x6e, 0x65, 0x73, 0x74, 0x1a, 0xee, 0x01, 0x0a, 0x09, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, + 0x56, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x72, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x20, 0x0a, 0x0b, + 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, + 0x0a, 0x0d, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x50, 0x72, 0x65, + 0x73, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x64, 0x33, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x44, 0x33, + 0x12, 0x33, 0x0a, 0x16, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, + 0x74, 0x5f, 0x6f, 0x66, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x13, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4f, 0x66, + 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x1a, 0xbb, 0x02, 0x0a, 0x0a, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, + 0x74, 0x79, 0x56, 0x32, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, + 0x72, 0x6f, 0x77, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x72, 0x6f, 0x77, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x56, 0x0a, 0x07, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x72, + 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x53, + 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x56, 0x32, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x43, + 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x52, 0x07, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x12, 0x56, 0x0a, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, + 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, + 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x56, + 0x32, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x56, 0x32, 0x52, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x1a, 0x46, 0x0a, 0x10, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x32, 0x12, + 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, + 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, + 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, + 0x49, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x41, + 0x52, 0x49, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, + 0x01, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, 0x4e, 0x5f, + 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x1d, 0x0a, 0x19, 0x43, + 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x44, + 0x49, 0x4d, 0x45, 0x4e, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x03, 0x22, 0x53, 0x0a, 0x10, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3f, + 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x56, 0x69, 0x65, 0x77, + 0x53, 0x70, 0x65, 0x63, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x70, 0x65, 0x63, 0x22, + 0x76, 0x0a, 0x09, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x04, + 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x72, 0x69, 0x6c, + 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, + 0x12, 0x35, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x59, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x71, 0x6c, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x2a, 0x0a, 0x0e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x6d, + 0x0a, 0x06, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, + 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, + 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x94, 0x04, + 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, + 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x74, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x44, 0x0a, 0x10, + 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, + 0x65, 0x52, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, + 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x71, 0x75, 0x65, 0x72, 0x79, 0x41, 0x72, 0x67, 0x73, 0x4a, 0x73, + 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x42, 0x0a, 0x0d, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x5f, + 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x72, + 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0c, 0x65, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6d, 0x61, + 0x69, 0x6c, 0x5f, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x69, 0x70, 0x69, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4e, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x72, 0x69, 0x6c, 0x6c, + 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x90, 0x02, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x3a, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x72, 0x75, 0x6e, + 0x5f, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x52, 0x75, 0x6e, 0x4f, 0x6e, + 0x12, 0x4d, 0x0a, 0x11, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x69, + 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x63, + 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x4d, 0x0a, 0x11, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x68, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x69, 0x6c, + 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x27, + 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x81, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x61, + 0x64, 0x68, 0x6f, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x61, 0x64, 0x68, 0x6f, + 0x63, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3b, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x6f, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x6e, 0x12, 0x3b, + 0x0a, 0x0b, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x6f, 0x6e, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x0a, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x4f, 0x6e, 0x22, 0x7c, 0x0a, 0x0b, 0x50, + 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x34, 0x0a, 0x04, 0x73, 0x70, + 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, + 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x54, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, + 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x21, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x50, 0x75, 0x6c, + 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x22, 0x12, 0x0a, 0x10, + 0x50, 0x75, 0x6c, 0x6c, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x22, 0x85, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, + 0x67, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x60, 0x0a, - 0x11, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x70, - 0x65, 0x63, 0x12, 0x4b, 0x0a, 0x0e, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x70, 0x6f, - 0x6c, 0x69, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x72, 0x69, 0x6c, - 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, - 0x52, 0x0d, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, - 0x2c, 0x0a, 0x12, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xd6, 0x02, - 0x0a, 0x13, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x52, 0x0a, 0x0d, 0x72, 0x6f, 0x77, 0x73, 0x5f, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x72, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x72, 0x69, + 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x52, 0x0a, 0x12, 0x52, 0x65, 0x66, 0x72, + 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3c, + 0x0a, 0x0a, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x52, 0x09, 0x6f, 0x6e, 0x6c, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x15, 0x0a, 0x13, + 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x0d, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, + 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x36, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, + 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, + 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x39, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, - 0x63, 0x79, 0x2e, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x52, 0x0c, 0x72, 0x6f, 0x77, - 0x73, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x72, 0x6f, 0x77, - 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x0e, 0x72, 0x6f, 0x77, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x79, - 0x74, 0x65, 0x73, 0x12, 0x54, 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x72, 0x69, - 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x2e, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x52, 0x0d, 0x66, 0x69, 0x6c, 0x65, - 0x73, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, - 0x65, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, - 0x66, 0x69, 0x6c, 0x65, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x4a, 0x0a, 0x08, 0x53, 0x74, - 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x54, 0x52, 0x41, 0x54, 0x45, - 0x47, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x52, 0x41, 0x54, 0x45, 0x47, 0x59, 0x5f, 0x48, 0x45, 0x41, - 0x44, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x52, 0x41, 0x54, 0x45, 0x47, 0x59, 0x5f, - 0x54, 0x41, 0x49, 0x4c, 0x10, 0x02, 0x22, 0x62, 0x0a, 0x08, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x63, 0x6b, 0x65, 0x72, - 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, - 0x74, 0x69, 0x63, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1b, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x5a, 0x6f, 0x6e, 0x65, 0x22, 0xa5, 0x01, 0x0a, 0x0a, 0x50, - 0x61, 0x72, 0x73, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, - 0x12, 0x44, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, - 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x72, 0x4c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4c, 0x6f, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x22, 0x50, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, - 0x23, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, - 0x50, 0x61, 0x74, 0x68, 0x22, 0x4b, 0x0a, 0x0f, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, - 0x63, 0x79, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, - 0x79, 0x22, 0x2a, 0x0a, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x22, 0x0a, - 0x0c, 0x43, 0x68, 0x61, 0x72, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, - 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x2a, 0x8a, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x65, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x0a, 0x1c, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, - 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x43, 0x4f, 0x4e, - 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x44, 0x4c, 0x45, - 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x02, - 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x42, 0xc1, - 0x01, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, - 0x6c, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, - 0x6c, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, - 0x74, 0x69, 0x6d, 0x65, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x52, 0x58, 0xaa, 0x02, 0x0f, 0x52, - 0x69, 0x6c, 0x6c, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, - 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, - 0xe2, 0x02, 0x1b, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, - 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, - 0x11, 0x52, 0x69, 0x6c, 0x6c, 0x3a, 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, - 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x60, 0x0a, 0x11, 0x42, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x4b, 0x0a, + 0x0e, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x45, 0x78, + 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0d, 0x65, 0x78, 0x74, + 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x2c, 0x0a, 0x12, 0x42, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xd6, 0x02, 0x0a, 0x13, 0x42, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, + 0x12, 0x52, 0x0a, 0x0d, 0x72, 0x6f, 0x77, 0x73, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2e, 0x53, 0x74, + 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x52, 0x0c, 0x72, 0x6f, 0x77, 0x73, 0x53, 0x74, 0x72, 0x61, + 0x74, 0x65, 0x67, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x72, 0x6f, 0x77, 0x73, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, + 0x72, 0x6f, 0x77, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x54, + 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x45, + 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2e, 0x53, 0x74, 0x72, + 0x61, 0x74, 0x65, 0x67, 0x79, 0x52, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x53, 0x74, 0x72, 0x61, + 0x74, 0x65, 0x67, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x6c, 0x69, + 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x4a, 0x0a, 0x08, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, + 0x79, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x54, 0x52, 0x41, 0x54, 0x45, 0x47, 0x59, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x53, + 0x54, 0x52, 0x41, 0x54, 0x45, 0x47, 0x59, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x10, 0x01, 0x12, 0x11, + 0x0a, 0x0d, 0x53, 0x54, 0x52, 0x41, 0x54, 0x45, 0x47, 0x59, 0x5f, 0x54, 0x41, 0x49, 0x4c, 0x10, + 0x02, 0x22, 0x62, 0x0a, 0x08, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x72, 0x6f, + 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, 0x69, 0x63, 0x6b, 0x65, + 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x69, 0x6d, + 0x65, 0x5a, 0x6f, 0x6e, 0x65, 0x22, 0xa5, 0x01, 0x0a, 0x0a, 0x50, 0x61, 0x72, 0x73, 0x65, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1b, + 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x44, 0x0a, 0x0e, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, + 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x72, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x08, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x22, 0x50, 0x0a, + 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x50, 0x61, 0x74, 0x68, 0x22, + 0x4b, 0x0a, 0x0f, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x22, 0x2a, 0x0a, 0x0e, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, + 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x22, 0x0a, 0x0c, 0x43, 0x68, 0x61, 0x72, + 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x2a, 0x8a, 0x01, 0x0a, + 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x20, 0x0a, 0x1c, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, + 0x18, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x52, + 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x42, 0xc1, 0x01, 0x0a, 0x13, 0x63, 0x6f, + 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, + 0x31, 0x42, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x76, + 0x31, 0xa2, 0x02, 0x03, 0x52, 0x52, 0x58, 0xaa, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x2e, 0x52, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, + 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1b, 0x52, 0x69, + 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, + 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x11, 0x52, 0x69, 0x6c, 0x6c, + 0x3a, 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3565,130 +3664,132 @@ func file_rill_runtime_v1_resources_proto_rawDescGZIP() []byte { return file_rill_runtime_v1_resources_proto_rawDescData } -var file_rill_runtime_v1_resources_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_rill_runtime_v1_resources_proto_enumTypes = make([]protoimpl.EnumInfo, 3) var file_rill_runtime_v1_resources_proto_msgTypes = make([]protoimpl.MessageInfo, 43) var file_rill_runtime_v1_resources_proto_goTypes = []interface{}{ (ReconcileStatus)(0), // 0: rill.runtime.v1.ReconcileStatus - (BucketExtractPolicy_Strategy)(0), // 1: rill.runtime.v1.BucketExtractPolicy.Strategy - (*Resource)(nil), // 2: rill.runtime.v1.Resource - (*ResourceMeta)(nil), // 3: rill.runtime.v1.ResourceMeta - (*ResourceName)(nil), // 4: rill.runtime.v1.ResourceName - (*ProjectParser)(nil), // 5: rill.runtime.v1.ProjectParser - (*ProjectParserSpec)(nil), // 6: rill.runtime.v1.ProjectParserSpec - (*ProjectParserState)(nil), // 7: rill.runtime.v1.ProjectParserState - (*SourceV2)(nil), // 8: rill.runtime.v1.SourceV2 - (*SourceSpec)(nil), // 9: rill.runtime.v1.SourceSpec - (*SourceState)(nil), // 10: rill.runtime.v1.SourceState - (*ModelV2)(nil), // 11: rill.runtime.v1.ModelV2 - (*ModelSpec)(nil), // 12: rill.runtime.v1.ModelSpec - (*ModelState)(nil), // 13: rill.runtime.v1.ModelState - (*MetricsViewV2)(nil), // 14: rill.runtime.v1.MetricsViewV2 - (*MetricsViewSpec)(nil), // 15: rill.runtime.v1.MetricsViewSpec - (*MetricsViewState)(nil), // 16: rill.runtime.v1.MetricsViewState - (*Migration)(nil), // 17: rill.runtime.v1.Migration - (*MigrationSpec)(nil), // 18: rill.runtime.v1.MigrationSpec - (*MigrationState)(nil), // 19: rill.runtime.v1.MigrationState - (*Report)(nil), // 20: rill.runtime.v1.Report - (*ReportSpec)(nil), // 21: rill.runtime.v1.ReportSpec - (*ReportState)(nil), // 22: rill.runtime.v1.ReportState - (*ReportExecution)(nil), // 23: rill.runtime.v1.ReportExecution - (*PullTrigger)(nil), // 24: rill.runtime.v1.PullTrigger - (*PullTriggerSpec)(nil), // 25: rill.runtime.v1.PullTriggerSpec - (*PullTriggerState)(nil), // 26: rill.runtime.v1.PullTriggerState - (*RefreshTrigger)(nil), // 27: rill.runtime.v1.RefreshTrigger - (*RefreshTriggerSpec)(nil), // 28: rill.runtime.v1.RefreshTriggerSpec - (*RefreshTriggerState)(nil), // 29: rill.runtime.v1.RefreshTriggerState - (*BucketPlanner)(nil), // 30: rill.runtime.v1.BucketPlanner - (*BucketPlannerSpec)(nil), // 31: rill.runtime.v1.BucketPlannerSpec - (*BucketPlannerState)(nil), // 32: rill.runtime.v1.BucketPlannerState - (*BucketExtractPolicy)(nil), // 33: rill.runtime.v1.BucketExtractPolicy - (*Schedule)(nil), // 34: rill.runtime.v1.Schedule - (*ParseError)(nil), // 35: rill.runtime.v1.ParseError - (*ValidationError)(nil), // 36: rill.runtime.v1.ValidationError - (*DependencyError)(nil), // 37: rill.runtime.v1.DependencyError - (*ExecutionError)(nil), // 38: rill.runtime.v1.ExecutionError - (*CharLocation)(nil), // 39: rill.runtime.v1.CharLocation - (*MetricsViewSpec_DimensionV2)(nil), // 40: rill.runtime.v1.MetricsViewSpec.DimensionV2 - (*MetricsViewSpec_MeasureV2)(nil), // 41: rill.runtime.v1.MetricsViewSpec.MeasureV2 - (*MetricsViewSpec_SecurityV2)(nil), // 42: rill.runtime.v1.MetricsViewSpec.SecurityV2 - (*MetricsViewSpec_SecurityV2_FieldConditionV2)(nil), // 43: rill.runtime.v1.MetricsViewSpec.SecurityV2.FieldConditionV2 - nil, // 44: rill.runtime.v1.ReportSpec.AnnotationsEntry - (*timestamppb.Timestamp)(nil), // 45: google.protobuf.Timestamp - (*structpb.Struct)(nil), // 46: google.protobuf.Struct - (TimeGrain)(0), // 47: rill.runtime.v1.TimeGrain - (ExportFormat)(0), // 48: rill.runtime.v1.ExportFormat + (MetricsViewSpec_ComparisonMode)(0), // 1: rill.runtime.v1.MetricsViewSpec.ComparisonMode + (BucketExtractPolicy_Strategy)(0), // 2: rill.runtime.v1.BucketExtractPolicy.Strategy + (*Resource)(nil), // 3: rill.runtime.v1.Resource + (*ResourceMeta)(nil), // 4: rill.runtime.v1.ResourceMeta + (*ResourceName)(nil), // 5: rill.runtime.v1.ResourceName + (*ProjectParser)(nil), // 6: rill.runtime.v1.ProjectParser + (*ProjectParserSpec)(nil), // 7: rill.runtime.v1.ProjectParserSpec + (*ProjectParserState)(nil), // 8: rill.runtime.v1.ProjectParserState + (*SourceV2)(nil), // 9: rill.runtime.v1.SourceV2 + (*SourceSpec)(nil), // 10: rill.runtime.v1.SourceSpec + (*SourceState)(nil), // 11: rill.runtime.v1.SourceState + (*ModelV2)(nil), // 12: rill.runtime.v1.ModelV2 + (*ModelSpec)(nil), // 13: rill.runtime.v1.ModelSpec + (*ModelState)(nil), // 14: rill.runtime.v1.ModelState + (*MetricsViewV2)(nil), // 15: rill.runtime.v1.MetricsViewV2 + (*MetricsViewSpec)(nil), // 16: rill.runtime.v1.MetricsViewSpec + (*MetricsViewState)(nil), // 17: rill.runtime.v1.MetricsViewState + (*Migration)(nil), // 18: rill.runtime.v1.Migration + (*MigrationSpec)(nil), // 19: rill.runtime.v1.MigrationSpec + (*MigrationState)(nil), // 20: rill.runtime.v1.MigrationState + (*Report)(nil), // 21: rill.runtime.v1.Report + (*ReportSpec)(nil), // 22: rill.runtime.v1.ReportSpec + (*ReportState)(nil), // 23: rill.runtime.v1.ReportState + (*ReportExecution)(nil), // 24: rill.runtime.v1.ReportExecution + (*PullTrigger)(nil), // 25: rill.runtime.v1.PullTrigger + (*PullTriggerSpec)(nil), // 26: rill.runtime.v1.PullTriggerSpec + (*PullTriggerState)(nil), // 27: rill.runtime.v1.PullTriggerState + (*RefreshTrigger)(nil), // 28: rill.runtime.v1.RefreshTrigger + (*RefreshTriggerSpec)(nil), // 29: rill.runtime.v1.RefreshTriggerSpec + (*RefreshTriggerState)(nil), // 30: rill.runtime.v1.RefreshTriggerState + (*BucketPlanner)(nil), // 31: rill.runtime.v1.BucketPlanner + (*BucketPlannerSpec)(nil), // 32: rill.runtime.v1.BucketPlannerSpec + (*BucketPlannerState)(nil), // 33: rill.runtime.v1.BucketPlannerState + (*BucketExtractPolicy)(nil), // 34: rill.runtime.v1.BucketExtractPolicy + (*Schedule)(nil), // 35: rill.runtime.v1.Schedule + (*ParseError)(nil), // 36: rill.runtime.v1.ParseError + (*ValidationError)(nil), // 37: rill.runtime.v1.ValidationError + (*DependencyError)(nil), // 38: rill.runtime.v1.DependencyError + (*ExecutionError)(nil), // 39: rill.runtime.v1.ExecutionError + (*CharLocation)(nil), // 40: rill.runtime.v1.CharLocation + (*MetricsViewSpec_DimensionV2)(nil), // 41: rill.runtime.v1.MetricsViewSpec.DimensionV2 + (*MetricsViewSpec_MeasureV2)(nil), // 42: rill.runtime.v1.MetricsViewSpec.MeasureV2 + (*MetricsViewSpec_SecurityV2)(nil), // 43: rill.runtime.v1.MetricsViewSpec.SecurityV2 + (*MetricsViewSpec_SecurityV2_FieldConditionV2)(nil), // 44: rill.runtime.v1.MetricsViewSpec.SecurityV2.FieldConditionV2 + nil, // 45: rill.runtime.v1.ReportSpec.AnnotationsEntry + (*timestamppb.Timestamp)(nil), // 46: google.protobuf.Timestamp + (*structpb.Struct)(nil), // 47: google.protobuf.Struct + (TimeGrain)(0), // 48: rill.runtime.v1.TimeGrain + (ExportFormat)(0), // 49: rill.runtime.v1.ExportFormat } var file_rill_runtime_v1_resources_proto_depIdxs = []int32{ - 3, // 0: rill.runtime.v1.Resource.meta:type_name -> rill.runtime.v1.ResourceMeta - 5, // 1: rill.runtime.v1.Resource.project_parser:type_name -> rill.runtime.v1.ProjectParser - 8, // 2: rill.runtime.v1.Resource.source:type_name -> rill.runtime.v1.SourceV2 - 11, // 3: rill.runtime.v1.Resource.model:type_name -> rill.runtime.v1.ModelV2 - 14, // 4: rill.runtime.v1.Resource.metrics_view:type_name -> rill.runtime.v1.MetricsViewV2 - 17, // 5: rill.runtime.v1.Resource.migration:type_name -> rill.runtime.v1.Migration - 20, // 6: rill.runtime.v1.Resource.report:type_name -> rill.runtime.v1.Report - 24, // 7: rill.runtime.v1.Resource.pull_trigger:type_name -> rill.runtime.v1.PullTrigger - 27, // 8: rill.runtime.v1.Resource.refresh_trigger:type_name -> rill.runtime.v1.RefreshTrigger - 30, // 9: rill.runtime.v1.Resource.bucket_planner:type_name -> rill.runtime.v1.BucketPlanner - 4, // 10: rill.runtime.v1.ResourceMeta.name:type_name -> rill.runtime.v1.ResourceName - 4, // 11: rill.runtime.v1.ResourceMeta.refs:type_name -> rill.runtime.v1.ResourceName - 4, // 12: rill.runtime.v1.ResourceMeta.owner:type_name -> rill.runtime.v1.ResourceName - 45, // 13: rill.runtime.v1.ResourceMeta.created_on:type_name -> google.protobuf.Timestamp - 45, // 14: rill.runtime.v1.ResourceMeta.spec_updated_on:type_name -> google.protobuf.Timestamp - 45, // 15: rill.runtime.v1.ResourceMeta.state_updated_on:type_name -> google.protobuf.Timestamp - 45, // 16: rill.runtime.v1.ResourceMeta.deleted_on:type_name -> google.protobuf.Timestamp + 4, // 0: rill.runtime.v1.Resource.meta:type_name -> rill.runtime.v1.ResourceMeta + 6, // 1: rill.runtime.v1.Resource.project_parser:type_name -> rill.runtime.v1.ProjectParser + 9, // 2: rill.runtime.v1.Resource.source:type_name -> rill.runtime.v1.SourceV2 + 12, // 3: rill.runtime.v1.Resource.model:type_name -> rill.runtime.v1.ModelV2 + 15, // 4: rill.runtime.v1.Resource.metrics_view:type_name -> rill.runtime.v1.MetricsViewV2 + 18, // 5: rill.runtime.v1.Resource.migration:type_name -> rill.runtime.v1.Migration + 21, // 6: rill.runtime.v1.Resource.report:type_name -> rill.runtime.v1.Report + 25, // 7: rill.runtime.v1.Resource.pull_trigger:type_name -> rill.runtime.v1.PullTrigger + 28, // 8: rill.runtime.v1.Resource.refresh_trigger:type_name -> rill.runtime.v1.RefreshTrigger + 31, // 9: rill.runtime.v1.Resource.bucket_planner:type_name -> rill.runtime.v1.BucketPlanner + 5, // 10: rill.runtime.v1.ResourceMeta.name:type_name -> rill.runtime.v1.ResourceName + 5, // 11: rill.runtime.v1.ResourceMeta.refs:type_name -> rill.runtime.v1.ResourceName + 5, // 12: rill.runtime.v1.ResourceMeta.owner:type_name -> rill.runtime.v1.ResourceName + 46, // 13: rill.runtime.v1.ResourceMeta.created_on:type_name -> google.protobuf.Timestamp + 46, // 14: rill.runtime.v1.ResourceMeta.spec_updated_on:type_name -> google.protobuf.Timestamp + 46, // 15: rill.runtime.v1.ResourceMeta.state_updated_on:type_name -> google.protobuf.Timestamp + 46, // 16: rill.runtime.v1.ResourceMeta.deleted_on:type_name -> google.protobuf.Timestamp 0, // 17: rill.runtime.v1.ResourceMeta.reconcile_status:type_name -> rill.runtime.v1.ReconcileStatus - 45, // 18: rill.runtime.v1.ResourceMeta.reconcile_on:type_name -> google.protobuf.Timestamp - 4, // 19: rill.runtime.v1.ResourceMeta.renamed_from:type_name -> rill.runtime.v1.ResourceName - 6, // 20: rill.runtime.v1.ProjectParser.spec:type_name -> rill.runtime.v1.ProjectParserSpec - 7, // 21: rill.runtime.v1.ProjectParser.state:type_name -> rill.runtime.v1.ProjectParserState - 35, // 22: rill.runtime.v1.ProjectParserState.parse_errors:type_name -> rill.runtime.v1.ParseError - 9, // 23: rill.runtime.v1.SourceV2.spec:type_name -> rill.runtime.v1.SourceSpec - 10, // 24: rill.runtime.v1.SourceV2.state:type_name -> rill.runtime.v1.SourceState - 46, // 25: rill.runtime.v1.SourceSpec.properties:type_name -> google.protobuf.Struct - 34, // 26: rill.runtime.v1.SourceSpec.refresh_schedule:type_name -> rill.runtime.v1.Schedule - 45, // 27: rill.runtime.v1.SourceState.refreshed_on:type_name -> google.protobuf.Timestamp - 12, // 28: rill.runtime.v1.ModelV2.spec:type_name -> rill.runtime.v1.ModelSpec - 13, // 29: rill.runtime.v1.ModelV2.state:type_name -> rill.runtime.v1.ModelState - 34, // 30: rill.runtime.v1.ModelSpec.refresh_schedule:type_name -> rill.runtime.v1.Schedule - 45, // 31: rill.runtime.v1.ModelState.refreshed_on:type_name -> google.protobuf.Timestamp - 15, // 32: rill.runtime.v1.MetricsViewV2.spec:type_name -> rill.runtime.v1.MetricsViewSpec - 16, // 33: rill.runtime.v1.MetricsViewV2.state:type_name -> rill.runtime.v1.MetricsViewState - 40, // 34: rill.runtime.v1.MetricsViewSpec.dimensions:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionV2 - 41, // 35: rill.runtime.v1.MetricsViewSpec.measures:type_name -> rill.runtime.v1.MetricsViewSpec.MeasureV2 - 47, // 36: rill.runtime.v1.MetricsViewSpec.smallest_time_grain:type_name -> rill.runtime.v1.TimeGrain - 42, // 37: rill.runtime.v1.MetricsViewSpec.security:type_name -> rill.runtime.v1.MetricsViewSpec.SecurityV2 - 15, // 38: rill.runtime.v1.MetricsViewState.valid_spec:type_name -> rill.runtime.v1.MetricsViewSpec - 18, // 39: rill.runtime.v1.Migration.spec:type_name -> rill.runtime.v1.MigrationSpec - 19, // 40: rill.runtime.v1.Migration.state:type_name -> rill.runtime.v1.MigrationState - 21, // 41: rill.runtime.v1.Report.spec:type_name -> rill.runtime.v1.ReportSpec - 22, // 42: rill.runtime.v1.Report.state:type_name -> rill.runtime.v1.ReportState - 34, // 43: rill.runtime.v1.ReportSpec.refresh_schedule:type_name -> rill.runtime.v1.Schedule - 48, // 44: rill.runtime.v1.ReportSpec.export_format:type_name -> rill.runtime.v1.ExportFormat - 44, // 45: rill.runtime.v1.ReportSpec.annotations:type_name -> rill.runtime.v1.ReportSpec.AnnotationsEntry - 45, // 46: rill.runtime.v1.ReportState.next_run_on:type_name -> google.protobuf.Timestamp - 23, // 47: rill.runtime.v1.ReportState.current_execution:type_name -> rill.runtime.v1.ReportExecution - 23, // 48: rill.runtime.v1.ReportState.execution_history:type_name -> rill.runtime.v1.ReportExecution - 45, // 49: rill.runtime.v1.ReportExecution.report_time:type_name -> google.protobuf.Timestamp - 45, // 50: rill.runtime.v1.ReportExecution.started_on:type_name -> google.protobuf.Timestamp - 45, // 51: rill.runtime.v1.ReportExecution.finished_on:type_name -> google.protobuf.Timestamp - 25, // 52: rill.runtime.v1.PullTrigger.spec:type_name -> rill.runtime.v1.PullTriggerSpec - 26, // 53: rill.runtime.v1.PullTrigger.state:type_name -> rill.runtime.v1.PullTriggerState - 28, // 54: rill.runtime.v1.RefreshTrigger.spec:type_name -> rill.runtime.v1.RefreshTriggerSpec - 29, // 55: rill.runtime.v1.RefreshTrigger.state:type_name -> rill.runtime.v1.RefreshTriggerState - 4, // 56: rill.runtime.v1.RefreshTriggerSpec.only_names:type_name -> rill.runtime.v1.ResourceName - 31, // 57: rill.runtime.v1.BucketPlanner.spec:type_name -> rill.runtime.v1.BucketPlannerSpec - 32, // 58: rill.runtime.v1.BucketPlanner.state:type_name -> rill.runtime.v1.BucketPlannerState - 33, // 59: rill.runtime.v1.BucketPlannerSpec.extract_policy:type_name -> rill.runtime.v1.BucketExtractPolicy - 1, // 60: rill.runtime.v1.BucketExtractPolicy.rows_strategy:type_name -> rill.runtime.v1.BucketExtractPolicy.Strategy - 1, // 61: rill.runtime.v1.BucketExtractPolicy.files_strategy:type_name -> rill.runtime.v1.BucketExtractPolicy.Strategy - 39, // 62: rill.runtime.v1.ParseError.start_location:type_name -> rill.runtime.v1.CharLocation - 43, // 63: rill.runtime.v1.MetricsViewSpec.SecurityV2.include:type_name -> rill.runtime.v1.MetricsViewSpec.SecurityV2.FieldConditionV2 - 43, // 64: rill.runtime.v1.MetricsViewSpec.SecurityV2.exclude:type_name -> rill.runtime.v1.MetricsViewSpec.SecurityV2.FieldConditionV2 - 65, // [65:65] is the sub-list for method output_type - 65, // [65:65] is the sub-list for method input_type - 65, // [65:65] is the sub-list for extension type_name - 65, // [65:65] is the sub-list for extension extendee - 0, // [0:65] is the sub-list for field type_name + 46, // 18: rill.runtime.v1.ResourceMeta.reconcile_on:type_name -> google.protobuf.Timestamp + 5, // 19: rill.runtime.v1.ResourceMeta.renamed_from:type_name -> rill.runtime.v1.ResourceName + 7, // 20: rill.runtime.v1.ProjectParser.spec:type_name -> rill.runtime.v1.ProjectParserSpec + 8, // 21: rill.runtime.v1.ProjectParser.state:type_name -> rill.runtime.v1.ProjectParserState + 36, // 22: rill.runtime.v1.ProjectParserState.parse_errors:type_name -> rill.runtime.v1.ParseError + 10, // 23: rill.runtime.v1.SourceV2.spec:type_name -> rill.runtime.v1.SourceSpec + 11, // 24: rill.runtime.v1.SourceV2.state:type_name -> rill.runtime.v1.SourceState + 47, // 25: rill.runtime.v1.SourceSpec.properties:type_name -> google.protobuf.Struct + 35, // 26: rill.runtime.v1.SourceSpec.refresh_schedule:type_name -> rill.runtime.v1.Schedule + 46, // 27: rill.runtime.v1.SourceState.refreshed_on:type_name -> google.protobuf.Timestamp + 13, // 28: rill.runtime.v1.ModelV2.spec:type_name -> rill.runtime.v1.ModelSpec + 14, // 29: rill.runtime.v1.ModelV2.state:type_name -> rill.runtime.v1.ModelState + 35, // 30: rill.runtime.v1.ModelSpec.refresh_schedule:type_name -> rill.runtime.v1.Schedule + 46, // 31: rill.runtime.v1.ModelState.refreshed_on:type_name -> google.protobuf.Timestamp + 16, // 32: rill.runtime.v1.MetricsViewV2.spec:type_name -> rill.runtime.v1.MetricsViewSpec + 17, // 33: rill.runtime.v1.MetricsViewV2.state:type_name -> rill.runtime.v1.MetricsViewState + 41, // 34: rill.runtime.v1.MetricsViewSpec.dimensions:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionV2 + 42, // 35: rill.runtime.v1.MetricsViewSpec.measures:type_name -> rill.runtime.v1.MetricsViewSpec.MeasureV2 + 48, // 36: rill.runtime.v1.MetricsViewSpec.smallest_time_grain:type_name -> rill.runtime.v1.TimeGrain + 43, // 37: rill.runtime.v1.MetricsViewSpec.security:type_name -> rill.runtime.v1.MetricsViewSpec.SecurityV2 + 1, // 38: rill.runtime.v1.MetricsViewSpec.default_comparison_mode:type_name -> rill.runtime.v1.MetricsViewSpec.ComparisonMode + 16, // 39: rill.runtime.v1.MetricsViewState.valid_spec:type_name -> rill.runtime.v1.MetricsViewSpec + 19, // 40: rill.runtime.v1.Migration.spec:type_name -> rill.runtime.v1.MigrationSpec + 20, // 41: rill.runtime.v1.Migration.state:type_name -> rill.runtime.v1.MigrationState + 22, // 42: rill.runtime.v1.Report.spec:type_name -> rill.runtime.v1.ReportSpec + 23, // 43: rill.runtime.v1.Report.state:type_name -> rill.runtime.v1.ReportState + 35, // 44: rill.runtime.v1.ReportSpec.refresh_schedule:type_name -> rill.runtime.v1.Schedule + 49, // 45: rill.runtime.v1.ReportSpec.export_format:type_name -> rill.runtime.v1.ExportFormat + 45, // 46: rill.runtime.v1.ReportSpec.annotations:type_name -> rill.runtime.v1.ReportSpec.AnnotationsEntry + 46, // 47: rill.runtime.v1.ReportState.next_run_on:type_name -> google.protobuf.Timestamp + 24, // 48: rill.runtime.v1.ReportState.current_execution:type_name -> rill.runtime.v1.ReportExecution + 24, // 49: rill.runtime.v1.ReportState.execution_history:type_name -> rill.runtime.v1.ReportExecution + 46, // 50: rill.runtime.v1.ReportExecution.report_time:type_name -> google.protobuf.Timestamp + 46, // 51: rill.runtime.v1.ReportExecution.started_on:type_name -> google.protobuf.Timestamp + 46, // 52: rill.runtime.v1.ReportExecution.finished_on:type_name -> google.protobuf.Timestamp + 26, // 53: rill.runtime.v1.PullTrigger.spec:type_name -> rill.runtime.v1.PullTriggerSpec + 27, // 54: rill.runtime.v1.PullTrigger.state:type_name -> rill.runtime.v1.PullTriggerState + 29, // 55: rill.runtime.v1.RefreshTrigger.spec:type_name -> rill.runtime.v1.RefreshTriggerSpec + 30, // 56: rill.runtime.v1.RefreshTrigger.state:type_name -> rill.runtime.v1.RefreshTriggerState + 5, // 57: rill.runtime.v1.RefreshTriggerSpec.only_names:type_name -> rill.runtime.v1.ResourceName + 32, // 58: rill.runtime.v1.BucketPlanner.spec:type_name -> rill.runtime.v1.BucketPlannerSpec + 33, // 59: rill.runtime.v1.BucketPlanner.state:type_name -> rill.runtime.v1.BucketPlannerState + 34, // 60: rill.runtime.v1.BucketPlannerSpec.extract_policy:type_name -> rill.runtime.v1.BucketExtractPolicy + 2, // 61: rill.runtime.v1.BucketExtractPolicy.rows_strategy:type_name -> rill.runtime.v1.BucketExtractPolicy.Strategy + 2, // 62: rill.runtime.v1.BucketExtractPolicy.files_strategy:type_name -> rill.runtime.v1.BucketExtractPolicy.Strategy + 40, // 63: rill.runtime.v1.ParseError.start_location:type_name -> rill.runtime.v1.CharLocation + 44, // 64: rill.runtime.v1.MetricsViewSpec.SecurityV2.include:type_name -> rill.runtime.v1.MetricsViewSpec.SecurityV2.FieldConditionV2 + 44, // 65: rill.runtime.v1.MetricsViewSpec.SecurityV2.exclude:type_name -> rill.runtime.v1.MetricsViewSpec.SecurityV2.FieldConditionV2 + 66, // [66:66] is the sub-list for method output_type + 66, // [66:66] is the sub-list for method input_type + 66, // [66:66] is the sub-list for extension type_name + 66, // [66:66] is the sub-list for extension extendee + 0, // [0:66] is the sub-list for field type_name } func init() { file_rill_runtime_v1_resources_proto_init() } @@ -4222,7 +4323,7 @@ func file_rill_runtime_v1_resources_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_rill_runtime_v1_resources_proto_rawDesc, - NumEnums: 2, + NumEnums: 3, NumMessages: 43, NumExtensions: 0, NumServices: 0, diff --git a/proto/gen/rill/runtime/v1/resources.pb.validate.go b/proto/gen/rill/runtime/v1/resources.pb.validate.go index 3a188987524..ba49a7de621 100644 --- a/proto/gen/rill/runtime/v1/resources.pb.validate.go +++ b/proto/gen/rill/runtime/v1/resources.pb.validate.go @@ -2620,6 +2620,10 @@ func (m *MetricsViewSpec) validate(all bool) error { // no validation rules for FirstMonthOfYear + // no validation rules for DefaultComparisonMode + + // no validation rules for DefaultComparisonDimension + if len(errors) > 0 { return MetricsViewSpecMultiError(errors) } @@ -5810,6 +5814,8 @@ func (m *MetricsViewSpec_DimensionV2) validate(all bool) error { // no validation rules for Description + // no validation rules for Unnest + if len(errors) > 0 { return MetricsViewSpec_DimensionV2MultiError(errors) } diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml index 614ff14b38b..c376bd4ec97 100644 --- a/proto/gen/rill/runtime/v1/runtime.swagger.yaml +++ b/proto/gen/rill/runtime/v1/runtime.swagger.yaml @@ -2359,6 +2359,14 @@ definitions: type: object $ref: '#/definitions/SecurityFieldCondition' title: Security for the metrics view + MetricsViewSpecComparisonMode: + type: string + enum: + - COMPARISON_MODE_UNSPECIFIED + - COMPARISON_MODE_NONE + - COMPARISON_MODE_TIME + - COMPARISON_MODE_DIMENSION + default: COMPARISON_MODE_UNSPECIFIED MetricsViewSpecDimensionV2: type: object properties: @@ -2370,6 +2378,8 @@ definitions: type: string description: type: string + unnest: + type: boolean title: Dimensions are columns to filter and group by MetricsViewSpecMeasureV2: type: object @@ -3587,6 +3597,12 @@ definitions: type: integer format: int64 description: Month number to use as the base for time aggregations by year. Defaults to 1 (January). + defaultComparisonMode: + $ref: '#/definitions/MetricsViewSpecComparisonMode' + description: Selected default comparison mode. + defaultComparisonDimension: + type: string + title: If comparison mode is dimension then this determines which is the default dimension v1MetricsViewState: type: object properties: diff --git a/proto/rill/runtime/v1/resources.proto b/proto/rill/runtime/v1/resources.proto index 6fe9f04612e..6f80583b217 100644 --- a/proto/rill/runtime/v1/resources.proto +++ b/proto/rill/runtime/v1/resources.proto @@ -127,6 +127,7 @@ message MetricsViewSpec { string column = 2; string label = 3; string description = 4; + bool unnest = 5; } // Measures are aggregated computed values message MeasureV2 { @@ -153,6 +154,12 @@ message MetricsViewSpec { repeated FieldConditionV2 include = 3; repeated FieldConditionV2 exclude = 4; } + enum ComparisonMode { + COMPARISON_MODE_UNSPECIFIED = 0; + COMPARISON_MODE_NONE = 1; + COMPARISON_MODE_TIME = 2; + COMPARISON_MODE_DIMENSION = 3; + } // Connector containing the table string connector = 1; // Name of the table the metrics view is based on @@ -179,6 +186,10 @@ message MetricsViewSpec { uint32 first_day_of_week = 12; // Month number to use as the base for time aggregations by year. Defaults to 1 (January). uint32 first_month_of_year = 13; + // Selected default comparison mode. + ComparisonMode default_comparison_mode = 14; + // If comparison mode is dimension then this determines which is the default dimension + string default_comparison_dimension = 15; } message MetricsViewState { diff --git a/runtime/compilers/rillv1/parse_metrics_view.go b/runtime/compilers/rillv1/parse_metrics_view.go index 2926a3d2d3b..0436a9ca719 100644 --- a/runtime/compilers/rillv1/parse_metrics_view.go +++ b/runtime/compilers/rillv1/parse_metrics_view.go @@ -36,6 +36,7 @@ type MetricsViewYAML struct { Property string // For backwards compatibility Description string Ignore bool `yaml:"ignore"` + Unnest bool } Measures []*struct { Name string @@ -59,8 +60,20 @@ type MetricsViewYAML struct { Condition string `yaml:"if"` } } + DefaultComparison struct { + Mode string `yaml:"mode"` + Dimension string `yaml:"dimension"` + } `yaml:"default_comparison"` } +var comparisonModesMap = map[string]runtimev1.MetricsViewSpec_ComparisonMode{ + "": runtimev1.MetricsViewSpec_COMPARISON_MODE_UNSPECIFIED, + "none": runtimev1.MetricsViewSpec_COMPARISON_MODE_NONE, + "time": runtimev1.MetricsViewSpec_COMPARISON_MODE_TIME, + "dimension": runtimev1.MetricsViewSpec_COMPARISON_MODE_DIMENSION, +} +var validComparisonModes = []string{"none", "time", "dimension"} + // parseMetricsView parses a metrics view (dashboard) definition and adds the resulting resource to p.Resources. func (p *Parser) parseMetricsView(ctx context.Context, node *Node) error { // Parse YAML @@ -180,6 +193,16 @@ func (p *Parser) parseMetricsView(ctx context.Context, node *Node) error { return fmt.Errorf("must define at least one measure") } + tmp.DefaultComparison.Mode = strings.ToLower(tmp.DefaultComparison.Mode) + if _, ok := comparisonModesMap[tmp.DefaultComparison.Mode]; !ok { + return fmt.Errorf("invalid mode: %q. allowed values: %s", tmp.DefaultComparison.Mode, strings.Join(validComparisonModes, ",")) + } + if tmp.DefaultComparison.Dimension != "" { + if ok := names[tmp.DefaultComparison.Dimension]; !ok { + return fmt.Errorf("default comparison dimension %q doesn't exist", tmp.DefaultComparison.Dimension) + } + } + if tmp.Security != nil { templateData := TemplateData{User: map[string]interface{}{ "name": "dummy", @@ -292,6 +315,7 @@ func (p *Parser) parseMetricsView(ctx context.Context, node *Node) error { Column: dim.Column, Label: dim.Label, Description: dim.Description, + Unnest: dim.Unnest, }) } @@ -311,6 +335,11 @@ func (p *Parser) parseMetricsView(ctx context.Context, node *Node) error { }) } + spec.DefaultComparisonMode = comparisonModesMap[tmp.DefaultComparison.Mode] + if tmp.DefaultComparison.Dimension != "" { + spec.DefaultComparisonDimension = tmp.DefaultComparison.Dimension + } + if tmp.Security != nil { if spec.Security == nil { spec.Security = &runtimev1.MetricsViewSpec_SecurityV2{} diff --git a/runtime/drivers/bigquery/sql_store.go b/runtime/drivers/bigquery/sql_store.go index 773da88484b..0d428bae089 100644 --- a/runtime/drivers/bigquery/sql_store.go +++ b/runtime/drivers/bigquery/sql_store.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "regexp" "strings" "time" @@ -19,6 +20,7 @@ import ( "github.com/rilldata/rill/runtime/pkg/observability" "go.uber.org/zap" "google.golang.org/api/iterator" + "google.golang.org/api/option" ) // recommended size is 512MB - 1GB, entire data is buffered in memory before its written to disk @@ -26,6 +28,9 @@ const rowGroupBufferSize = int64(datasize.MB) * 512 const _jsonDownloadLimitBytes = 100 * int64(datasize.MB) +// Regex to parse BigQuery SELECT ALL statement: SELECT * FROM `project_id.dataset.table` +var selectQueryRegex = regexp.MustCompile("(?i)^\\s*SELECT\\s+\\*\\s+FROM\\s+(`?[a-zA-Z0-9_.-]+`?)\\s*$") + // Query implements drivers.SQLStore func (c *Connection) Query(ctx context.Context, props map[string]any) (drivers.RowIterator, error) { return nil, drivers.ErrNotImplemented @@ -43,42 +48,85 @@ func (c *Connection) QueryAsFiles(ctx context.Context, props map[string]any, opt return nil, err } - client, err := bigquery.NewClient(ctx, srcProps.ProjectID, opts...) - if err != nil { - if strings.Contains(err.Error(), "unable to detect projectID") { - return nil, fmt.Errorf("projectID not detected in credentials. Please set `project_id` in source yaml") + var client *bigquery.Client + var it *bigquery.RowIterator + + match := selectQueryRegex.FindStringSubmatch(srcProps.SQL) + if match != nil { + // "SELECT * FROM `project_id.dataset.table`" statement so storage api might be used + // project_id and backticks are optional + fullTableName := match[1] + fullTableName = strings.Trim(fullTableName, "`") + + var projectID, dataset, tableID string + + parts := strings.Split(fullTableName, ".") + switch len(parts) { + case 2: + dataset, tableID = parts[0], parts[1] + projectID = srcProps.ProjectID + case 3: + projectID, dataset, tableID = parts[0], parts[1], parts[2] + default: + return nil, fmt.Errorf("invalid table format, `project_id.dataset.table` is expected") } - return nil, fmt.Errorf("failed to create bigquery client: %w", err) - } - if err := client.EnableStorageReadClient(ctx, opts...); err != nil { - client.Close() - return nil, err - } + client, err = createClient(ctx, srcProps.ProjectID, opts) + if err != nil { + return nil, err + } - now := time.Now() - q := client.Query(srcProps.SQL) - it, err := q.Read(ctx) - if err != nil && !strings.Contains(err.Error(), "Syntax error") { - // close the read storage API client - client.Close() - c.logger.Info("query failed, retrying without storage api", zap.Error(err)) - // the query results are always cached in a temporary table that storage api can use - // there are some exceptions when results aren't cached - // so we also try without storage api - client, err = bigquery.NewClient(ctx, srcProps.ProjectID, opts...) + if err = client.EnableStorageReadClient(ctx, opts...); err != nil { + client.Close() + return nil, err + } + + it = client.DatasetInProject(projectID, dataset).Table(tableID).Read(ctx) + } else { + now := time.Now() + + client, err = createClient(ctx, srcProps.ProjectID, opts) if err != nil { - return nil, fmt.Errorf("failed to create bigquery client: %w", err) + return nil, err + } + + if err := client.EnableStorageReadClient(ctx, opts...); err != nil { + client.Close() + return nil, err } q := client.Query(srcProps.SQL) it, err = q.Read(ctx) + + if err != nil && strings.Contains(err.Error(), "Response too large to return") { + // https://cloud.google.com/knowledge/kb/bigquery-response-too-large-to-return-consider-setting-allowlargeresults-to-true-in-your-job-configuration-000004266 + client.Close() + return nil, fmt.Errorf("response too large, consider ingesting the entire table with 'select * from `project_id.dataset.tablename`'") + } + + if err != nil && !strings.Contains(err.Error(), "Syntax error") { + // close the read storage API client + client.Close() + c.logger.Info("query failed, retrying without storage api", zap.Error(err)) + // the query results are always cached in a temporary table that storage api can use + // there are some exceptions when results aren't cached + // so we also try without storage api + client, err = bigquery.NewClient(ctx, srcProps.ProjectID, opts...) + if err != nil { + return nil, fmt.Errorf("failed to create bigquery client: %w", err) + } + + q := client.Query(srcProps.SQL) + it, err = q.Read(ctx) + } + + if err != nil { + client.Close() + return nil, err + } + + c.logger.Info("query took", zap.Duration("duration", time.Since(now)), observability.ZapCtx(ctx)) } - if err != nil { - client.Close() - return nil, err - } - c.logger.Info("query took", zap.Duration("duration", time.Since(now)), observability.ZapCtx(ctx)) p.Target(int64(it.TotalRows), drivers.ProgressUnitRecord) return &fileIterator{ @@ -92,6 +140,17 @@ func (c *Connection) QueryAsFiles(ctx context.Context, props map[string]any, opt }, nil } +func createClient(ctx context.Context, projectID string, opts []option.ClientOption) (*bigquery.Client, error) { + client, err := bigquery.NewClient(ctx, projectID, opts...) + if err != nil { + if strings.Contains(err.Error(), "unable to detect projectID") { + return nil, fmt.Errorf("projectID not detected in credentials. Please set `project_id` in source yaml") + } + return nil, fmt.Errorf("failed to create bigquery client: %w", err) + } + return client, nil +} + type fileIterator struct { client *bigquery.Client bqIter *bigquery.RowIterator @@ -111,10 +170,6 @@ func (f *fileIterator) Close() error { return os.Remove(f.tempFilePath) } -// KeepFilesUntilClose implements drivers.FileIterator. -func (f *fileIterator) KeepFilesUntilClose(keepFilesUntilClose bool) { -} - // Next implements drivers.FileIterator. // TODO :: currently it downloads all records in a single file. Need to check if it is efficient to ingest a single file with size in tens of GBs or more. func (f *fileIterator) Next() ([]string, error) { diff --git a/runtime/drivers/blob/blobdownloader.go b/runtime/drivers/blob/blobdownloader.go index 5674ba7b913..923e8aa2688 100644 --- a/runtime/drivers/blob/blobdownloader.go +++ b/runtime/drivers/blob/blobdownloader.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "time" "cloud.google.com/go/storage" @@ -149,7 +150,7 @@ func NewIterator(ctx context.Context, bucket *blob.Bucket, opts Options, l *zap. // For cases where there's only one file, we want to prefetch it to return the error early (from NewIterator instead of Next) if len(objects) == 1 { - it.KeepFilesUntilClose(true) + it.opts.KeepFilesUntilClose = true batch, err := it.Next() if err != nil { it.Close() @@ -161,10 +162,6 @@ func NewIterator(ctx context.Context, bucket *blob.Bucket, opts Options, l *zap. return it, nil } -func (it *blobIterator) KeepFilesUntilClose(keepFilesUntilClose bool) { - it.opts.KeepFilesUntilClose = keepFilesUntilClose -} - func (it *blobIterator) Close() error { // Cancel the background downloads (this will eventually close downloadsCh, which eventually closes batchCh) it.cancel() @@ -345,35 +342,35 @@ func (it *blobIterator) downloadFiles() { // Download the file and send it on downloadsCh. // NOTE: Errors returned here will be assigned to it.downloadErr after the loop. g.Go(func() error { - file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, os.ModePerm) - if err != nil { - return err - } - defer file.Close() - ext := filepath.Ext(obj.obj.Key) partialReader, isPartialDownloadSupported := _partialDownloadReaders[ext] downloadFull := obj.full || !isPartialDownloadSupported startTime := time.Now() + var file *os.File + err := retry(5, 10*time.Second, func() error { + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer file.Close() - if downloadFull { - err = downloadObject(ctx, it.bucket, obj.obj.Key, file) - } else { + if downloadFull { + return downloadObject(ctx, it.bucket, obj.obj.Key, file) + } // download partial file switch partialReader { case "parquet": - err = downloadParquet(ctx, it.bucket, obj.obj, obj.extractOption, file) + return downloadParquet(ctx, it.bucket, obj.obj, obj.extractOption, file) case "csv": - err = downloadText(ctx, it.bucket, obj.obj, &textExtractOption{extractOption: obj.extractOption, hasCSVHeader: true}, file) + return downloadText(ctx, it.bucket, obj.obj, &textExtractOption{extractOption: obj.extractOption, hasCSVHeader: true}, file) case "json": - err = downloadText(ctx, it.bucket, obj.obj, &textExtractOption{extractOption: obj.extractOption, hasCSVHeader: false}, file) + return downloadText(ctx, it.bucket, obj.obj, &textExtractOption{extractOption: obj.extractOption, hasCSVHeader: false}, file) default: // should not reach here panic(fmt.Errorf("partial download not supported for extension: %q", ext)) } - } - + }) // Returning the err will cancel the errgroup and propagate the error to it.downloadErr if err != nil { return err @@ -449,10 +446,6 @@ type prefetchedIterator struct { underlying *blobIterator } -func (it *prefetchedIterator) KeepFilesUntilClose(keep bool) { - // Nothing to do – already set on the underlying iterator -} - func (it *prefetchedIterator) Close() error { return it.underlying.Close() } @@ -514,3 +507,18 @@ func downloadObject(ctx context.Context, bucket *blob.Bucket, objpath string, fi _, err = io.Copy(file, rc) return err } + +func retry(maxRetries int, delay time.Duration, fn func() error) error { + var err error + for i := 0; i < maxRetries; i++ { + err = fn() + if err == nil { + return nil // success + } else if strings.Contains(err.Error(), "stream error: stream ID") { + time.Sleep(delay) // retry + } else { + break // return error + } + } + return err +} diff --git a/runtime/drivers/duckdb/config.go b/runtime/drivers/duckdb/config.go index 25071520e2b..c1da86deb6f 100644 --- a/runtime/drivers/duckdb/config.go +++ b/runtime/drivers/duckdb/config.go @@ -9,8 +9,11 @@ import ( "github.com/mitchellh/mapstructure" ) -// We need to give one thread 1GB memory to operate on. -const memoryThreadRatio = 1 +const ( + cpuThreadRatio float64 = 0.5 + poolSizeMin int = 2 + poolSizeMax int = 5 +) // config represents the DuckDB driver config type config struct { @@ -24,14 +27,18 @@ type config struct { ErrorOnIncompatibleVersion bool `mapstructure:"error_on_incompatible_version"` // ExtTableStorage controls if every table is stored in a different db file ExtTableStorage bool `mapstructure:"external_table_storage"` - // MemoryLimitGB is duckdb max_memory config - MemoryLimitGB int `mapstructure:"memory_limit_gb"` - // CPU is the limit on cpu which determines number of threads based on cpuThreadRatio constant + // CPU cores available for the DB CPU int `mapstructure:"cpu"` - // DisableThreadLimit disables any thread limit on duckdb - DisableThreadLimit bool `mapstructure:"disable_thread_limit"` - // StorageLimitBytes is the maximum size of all database files + // MemoryLimitGB is the amount of memory available for the DB + MemoryLimitGB int `mapstructure:"memory_limit_gb"` + // StorageLimitBytes is the amount of disk storage available for the DB StorageLimitBytes int64 `mapstructure:"storage_limit_bytes"` + // MaxMemoryOverride sets a hard override for the "max_memory" DuckDB setting + MaxMemoryGBOverride int `mapstructure:"max_memory_gb_override"` + // ThreadsOverride sets a hard override for the "threads" DuckDB setting. Set to -1 for unlimited threads. + ThreadsOverride int `mapstructure:"threads_override"` + // BootQueries is queries to run on boot. Use ; to separate multiple queries. Common use case is to provide project specific memory and threads ratios. + BootQueries string `mapstructure:"boot_queries"` // DBFilePath is the path where the database is stored. It is inferred from the DSN (can't be provided by user). DBFilePath string `mapstructure:"-"` // ExtStoragePath is the path where the database files are stored in case external_table_storage is true. It is inferred from the DSN (can't be provided by user). @@ -39,10 +46,7 @@ type config struct { } func newConfig(cfgMap map[string]any) (*config, error) { - cfg := &config{ - PoolSize: 2, // Default value - DisableThreadLimit: false, - } + cfg := &config{} err := mapstructure.WeakDecode(cfgMap, cfg) if err != nil { return nil, fmt.Errorf("could not decode config: %w", err) @@ -64,36 +68,57 @@ func newConfig(cfgMap map[string]any) (*config, error) { cfg.ExtStoragePath = filepath.Dir(cfg.DBFilePath) } - // We also support overriding the pool size via the DSN by setting "rill_pool_size" as a query argument. + // Set memory limit + maxMemory := cfg.MemoryLimitGB + if cfg.MaxMemoryGBOverride != 0 { + maxMemory = cfg.MaxMemoryGBOverride + } + if maxMemory > 0 { + qry.Add("max_memory", fmt.Sprintf("%dGB", maxMemory)) + } + + // Set threads limit + var threads int + if cfg.ThreadsOverride != 0 { + threads = cfg.ThreadsOverride + } else if cfg.CPU > 0 { + threads = int(cpuThreadRatio * float64(cfg.CPU)) + if threads <= 0 { + threads = 1 + } + } + if threads > 0 { // NOTE: threads=0 or threads=-1 means no limit + qry.Add("threads", strconv.Itoa(threads)) + } + + // Set pool size + poolSize := cfg.PoolSize if qry.Has("rill_pool_size") { + // For backwards compatibility, we also support overriding the pool size via the DSN when "rill_pool_size" is a query argument. + + // Remove from query string (so not passed into DuckDB config) + val := qry.Get("rill_pool_size") + qry.Del("rill_pool_size") + // Parse as integer - cfg.PoolSize, err = strconv.Atoi(qry.Get("rill_pool_size")) + poolSize, err = strconv.Atoi(val) if err != nil { return nil, fmt.Errorf("could not parse dsn: 'rill_pool_size' is not an integer") } - - // Remove from query string (so not passed into DuckDB config) - qry.Del("rill_pool_size") } - threads := 0 - if cfg.MemoryLimitGB > 0 { - qry.Add("max_memory", fmt.Sprintf("%dGB", cfg.MemoryLimitGB)) - threads = memoryThreadRatio * cfg.MemoryLimitGB - if !cfg.DisableThreadLimit { - qry.Add("threads", strconv.Itoa(threads)) + if poolSize == 0 && threads != 0 { + poolSize = threads + if cfg.CPU != 0 && cfg.CPU < poolSize { + poolSize = cfg.CPU } + poolSize = min(poolSizeMax, poolSize) // Only enforce max pool size when inferred from threads/CPU } - if cfg.CPU > 0 { - cfg.PoolSize = max(2, min(cfg.CPU, threads)) - } + poolSize = max(poolSizeMin, poolSize) // Always enforce min pool size + cfg.PoolSize = poolSize // Rebuild DuckDB DSN (which should be "path?key=val&...") // this is required since spaces and other special characters are valid in db file path but invalid and hence encoded in URL cfg.DSN = generateDSN(uri.Path, qry.Encode()) - // Check pool size - if cfg.PoolSize < 2 { - return nil, fmt.Errorf("duckdb pool size must be >= 1") - } return cfg, nil } diff --git a/runtime/drivers/duckdb/config_test.go b/runtime/drivers/duckdb/config_test.go index 247f25c97cd..4c35921ac5a 100644 --- a/runtime/drivers/duckdb/config_test.go +++ b/runtime/drivers/duckdb/config_test.go @@ -64,9 +64,6 @@ func TestConfig(t *testing.T) { _, err = newConfig(map[string]any{"dsn": "path/to/duck.db?max_memory=4GB", "pool_size": "abc"}) require.Error(t, err) - _, err = newConfig(map[string]any{"dsn": "path/to/duck.db?max_memory=4GB", "pool_size": 0}) - require.Error(t, err) - cfg, err = newConfig(map[string]any{"dsn": "duck.db"}) require.NoError(t, err) require.Equal(t, "duck.db", cfg.DBFilePath) @@ -78,7 +75,7 @@ func TestConfig(t *testing.T) { cfg, err = newConfig(map[string]any{"dsn": "duck.db", "memory_limit_gb": "4", "cpu": "2"}) require.NoError(t, err) require.Equal(t, "duck.db", cfg.DBFilePath) - require.Equal(t, "duck.db?max_memory=4GB&threads=4", cfg.DSN) + require.Equal(t, "duck.db?max_memory=4GB&threads=1", cfg.DSN) require.Equal(t, 2, cfg.PoolSize) cfg, err = newConfig(map[string]any{"dsn": "duck.db?max_memory=2GB&rill_pool_size=4"}) @@ -98,7 +95,7 @@ func Test_specialCharInPath(t *testing.T) { conn, err := Driver{}.Open(map[string]any{"dsn": dbFile, "memory_limit_gb": "4", "cpu": "2"}, false, activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) config := conn.(*connection).config - require.Equal(t, filepath.Join(path, "st@g3's.db?max_memory=4GB&threads=4"), config.DSN) + require.Equal(t, filepath.Join(path, "st@g3's.db?max_memory=4GB&threads=1"), config.DSN) require.Equal(t, 2, config.PoolSize) olap, ok := conn.AsOLAP("") @@ -109,3 +106,20 @@ func Test_specialCharInPath(t *testing.T) { require.NoError(t, res.Close()) require.NoError(t, conn.Close()) } + +func TestOverrides(t *testing.T) { + cfgMap := map[string]any{"dsn": "duck.db", "memory_limit_gb": "4", "cpu": "2", "max_memory_gb_override": "2", "threads_override": "10"} + handle, err := Driver{}.Open(cfgMap, false, activity.NewNoopClient(), zap.NewNop()) + require.NoError(t, err) + + olap, ok := handle.AsOLAP("") + require.True(t, ok) + + res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT value FROM duckdb_settings() WHERE name='max_memory'"}) + require.NoError(t, err) + require.True(t, res.Next()) + var mem string + require.NoError(t, res.Scan(&mem)) + + require.Equal(t, "2.0GB", mem) +} diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 716bc0393f6..bd38276a546 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -263,7 +263,6 @@ type connection struct { // driverConfig is input config passed during Open driverConfig map[string]any driverName string - instanceID string // populated after call to AsOLAP // config is parsed configs config *config logger *zap.Logger @@ -314,6 +313,8 @@ func (c *connection) Config() map[string]any { func (c *connection) Close() error { c.cancel() _ = c.registration.Unregister() + // detach all attached DBs otherwise duckdb leaks memory + c.detachAllDBs() return c.db.Close() } @@ -347,12 +348,6 @@ func (c *connection) AsOLAP(instanceID string) (drivers.OLAPStore, bool) { // duckdb olap is instance specific return nil, false } - // TODO Add this back once every call passes instanceID correctly. - // Example incorrect usage : runtime/services/catalog/migrator/sources/sources.go - // if c.instanceID != "" && c.instanceID != instanceID { - // return nil, false - // } - c.instanceID = instanceID return c, true } @@ -369,7 +364,7 @@ func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { // AsTransporter implements drivers.Connection. func (c *connection) AsTransporter(from, to drivers.Handle) (drivers.Transporter, bool) { - olap, _ := to.AsOLAP(c.instanceID) // if c == to, connection is instance specific + olap, _ := to.(*connection) if c == to { if from == to { return transporter.NewDuckDBToDuckDB(olap, c.logger), true @@ -398,6 +393,8 @@ func (c *connection) AsFileStore() (drivers.FileStore, bool) { func (c *connection) reopenDB() error { // If c.db is already open, close it first if c.db != nil { + // detach all attached DBs otherwise duckdb leaks memory + c.detachAllDBs() err := c.db.Close() if err != nil { return err @@ -427,6 +424,10 @@ func (c *connection) reopenDB() error { bootQueries = append(bootQueries, "SET preserve_insertion_order TO false") } + if c.config.BootQueries != "" { + bootQueries = append(bootQueries, c.config.BootQueries) + } + // DuckDB extensions need to be loaded separately on each connection, but the built-in connection pool in database/sql doesn't enable that. // So we use go-duckdb's custom connector to pass a callback that it invokes for each new connection. connector, err := duckdb.NewConnector(c.config.DSN, func(execer driver.ExecerContext) error { @@ -472,6 +473,8 @@ func (c *connection) reopenDB() error { } defer conn.Close() + c.logLimits(conn) + // List the directories directly in the external storage directory // Load the version.txt from each sub-directory // If version.txt is found, attach only the .db file matching the version.txt. @@ -759,6 +762,48 @@ func (c *connection) periodicallyEmitStats(d time.Duration) { } } +// detachAllDBs detaches all attached dbs if external_table_storage config is true +func (c *connection) detachAllDBs() { + if !c.config.ExtTableStorage { + return + } + entries, err := os.ReadDir(c.config.ExtStoragePath) + if err != nil { + c.logger.Error("unable to read ExtStoragePath", zap.String("path", c.config.ExtStoragePath), zap.Error(err)) + return + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + version, exist, err := c.tableVersion(entry.Name()) + if err != nil { + continue + } + if !exist { + continue + } + + db := dbName(entry.Name(), version) + _, err = c.db.ExecContext(context.Background(), fmt.Sprintf("DETACH %s", safeSQLName(db))) + if err != nil { + c.logger.Error("detach failed", zap.String("db", db), zap.Error(err)) + } + } +} + +func (c *connection) logLimits(conn *sqlx.Conn) { + row := conn.QueryRowContext(context.Background(), "SELECT value FROM duckdb_settings() WHERE name='max_memory'") + var memory string + _ = row.Scan(&memory) + + row = conn.QueryRowContext(context.Background(), "SELECT value FROM duckdb_settings() WHERE name='threads'") + var threads string + _ = row.Scan(&threads) + + c.logger.Info("duckdb limits", zap.String("memory", memory), zap.String("threads", threads)) +} + // Regex to parse human-readable size returned by DuckDB // nolint var humanReadableSizeRegex = regexp.MustCompile(`^([\d.]+)\s*(\S+)$`) diff --git a/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go b/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go index 8a20a1eee5b..0b4bf9a0445 100644 --- a/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go @@ -70,7 +70,13 @@ func (t *objectStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps srcCfg.DuckDB["union_by_name"] = true } - a := newAppender(t.to, sinkCfg, srcCfg.DuckDB, srcCfg.AllowSchemaRelaxation, t.logger) + a := newAppender(t.to, sinkCfg, srcCfg.AllowSchemaRelaxation, t.logger, func(files []string) (string, error) { + from, err := sourceReader(files, format, srcCfg.DuckDB) + if err != nil { + return "", err + } + return fmt.Sprintf("SELECT * FROM %s", from), nil + }) for { files, err := iterator.Next() @@ -88,7 +94,7 @@ func (t *objectStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps st := time.Now() t.logger.Info("ingesting files", zap.Strings("files", files), observability.ZapCtx(ctx)) if appendToTable { - if err := a.appendData(ctx, files, format); err != nil { + if err := a.appendData(ctx, files); err != nil { return err } } else { @@ -111,57 +117,116 @@ func (t *objectStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps return nil } +func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL string, iterator drivers.FileIterator, srcCfg *fileSourceProperties, dbSink *sinkProperties, opts *drivers.TransferOptions) error { + ast, err := duckdbsql.Parse(originalSQL) + if err != nil { + return err + } + + // Validate the sql is supported for sources + // TODO: find a good common place for this validation and avoid code duplication here and in sources packages as well + refs := ast.GetTableRefs() + if len(refs) != 1 { + return errors.New("sql source should have exactly one table reference") + } + ref := refs[0] + + if len(ref.Paths) == 0 { + return errors.New("only read_* functions with a single path is supported") + } + if len(ref.Paths) > 1 { + return errors.New("invalid source, only a single path for source is supported") + } + a := newAppender(t.to, dbSink, srcCfg.AllowSchemaRelaxation, t.logger, func(files []string) (string, error) { + return rewriteSQL(ast, files) + }) + appendToTable := false + for { + files, err := iterator.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return err + } + + st := time.Now() + t.logger.Info("ingesting files", zap.Strings("files", files), observability.ZapCtx(ctx)) + if appendToTable { + if err := a.appendData(ctx, files); err != nil { + return err + } + } else { + sql, err := rewriteSQL(ast, files) + if err != nil { + return err + } + + err = t.to.CreateTableAsSelect(ctx, dbSink.Table, false, sql) + if err != nil { + return err + } + } + + size := fileSize(files) + t.logger.Info("ingested files", zap.Strings("files", files), zap.Int64("bytes_ingested", size), zap.Duration("duration", time.Since(st)), observability.ZapCtx(ctx)) + opts.Progress.Observe(size, drivers.ProgressUnitByte) + appendToTable = true + } + return nil +} + type appender struct { to drivers.OLAPStore sink *sinkProperties - ingestionProps map[string]any allowSchemaRelaxation bool tableSchema map[string]string logger *zap.Logger + sqlFunc func([]string) (string, error) } -func newAppender(to drivers.OLAPStore, sink *sinkProperties, ingestionProps map[string]any, allowSchemaRelaxation bool, logger *zap.Logger) *appender { +func newAppender(to drivers.OLAPStore, sink *sinkProperties, allowSchemaRelaxation bool, logger *zap.Logger, sqlFunc func([]string) (string, error)) *appender { return &appender{ to: to, sink: sink, - ingestionProps: ingestionProps, allowSchemaRelaxation: allowSchemaRelaxation, logger: logger, tableSchema: nil, + sqlFunc: sqlFunc, } } -func (a *appender) appendData(ctx context.Context, files []string, format string) error { - from, err := sourceReader(files, format, a.ingestionProps) +func (a *appender) appendData(ctx context.Context, files []string) error { + sql, err := a.sqlFunc(files) if err != nil { return err } - err = a.to.InsertTableAsSelect(ctx, a.sink.Table, a.allowSchemaRelaxation, fmt.Sprintf("SELECT * FROM %s", from)) + err = a.to.InsertTableAsSelect(ctx, a.sink.Table, a.allowSchemaRelaxation, sql) if err == nil || !a.allowSchemaRelaxation || !containsAny(err.Error(), []string{"binder error", "conversion error"}) { return err } // error is of type binder error (more or less columns than current table schema) // or of type conversion error (datatype changed or column sequence changed) - err = a.updateSchema(ctx, from, files) + err = a.updateSchema(ctx, sql, files) if err != nil { return fmt.Errorf("failed to update schema %w", err) } - return a.to.InsertTableAsSelect(ctx, a.sink.Table, true, fmt.Sprintf("SELECT * FROM %s", from)) + return a.to.InsertTableAsSelect(ctx, a.sink.Table, true, sql) } // updateSchema updates the schema of the table in case new file adds a new column or // updates the datatypes of an existing columns with a wider datatype. -func (a *appender) updateSchema(ctx context.Context, from string, fileNames []string) error { +func (a *appender) updateSchema(ctx context.Context, sql string, fileNames []string) error { // schema of new files - srcSchema, err := a.scanSchemaFromQuery(ctx, fmt.Sprintf("DESCRIBE (SELECT * FROM %s LIMIT 0);", from)) + srcSchema, err := a.scanSchemaFromQuery(ctx, fmt.Sprintf("DESCRIBE (%s);", sql)) if err != nil { return err } // combined schema - qry := fmt.Sprintf("DESCRIBE ((SELECT * FROM %s limit 0) UNION ALL BY NAME (SELECT * FROM %s limit 0));", safeName(a.sink.Table), from) + qry := fmt.Sprintf("DESCRIBE ((SELECT * FROM %s LIMIT 0) UNION ALL BY NAME (%s));", safeName(a.sink.Table), sql) unionSchema, err := a.scanSchemaFromQuery(ctx, qry) if err != nil { return err @@ -246,41 +311,8 @@ func (a *appender) scanSchemaFromQuery(ctx context.Context, qry string) (map[str return schema, nil } -func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL string, iterator drivers.FileIterator, srcCfg *fileSourceProperties, dbSink *sinkProperties, opts *drivers.TransferOptions) error { - iterator.KeepFilesUntilClose(true) - allFiles := make([]string, 0) - for { - files, err := iterator.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return err - } - allFiles = append(allFiles, files...) - } - - ast, err := duckdbsql.Parse(originalSQL) - if err != nil { - return err - } - - // Validate the sql is supported for sources - // TODO: find a good common place for this validation and avoid code duplication here and in sources packages as well - refs := ast.GetTableRefs() - if len(refs) != 1 { - return errors.New("sql source should have exactly one table reference") - } - ref := refs[0] - - if len(ref.Paths) == 0 { - return errors.New("only read_* functions with a single path is supported") - } - if len(ref.Paths) > 1 { - return errors.New("invalid source, only a single path for source is supported") - } - - err = ast.RewriteTableRefs(func(table *duckdbsql.TableRef) (*duckdbsql.TableRef, bool) { +func rewriteSQL(ast *duckdbsql.AST, allFiles []string) (string, error) { + err := ast.RewriteTableRefs(func(table *duckdbsql.TableRef) (*duckdbsql.TableRef, bool) { return &duckdbsql.TableRef{ Paths: allFiles, Function: table.Function, @@ -289,21 +321,11 @@ func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL s }, true }) if err != nil { - return err + return "", err } sql, err := ast.Format() if err != nil { - return err - } - - st := time.Now() - err = t.to.CreateTableAsSelect(ctx, dbSink.Table, false, sql) - if err != nil { - return err + return "", err } - - size := fileSize(allFiles) - t.logger.Info("ingested files", zap.Strings("files", allFiles), zap.Int64("bytes_ingested", size), zap.Duration("duration", time.Since(st)), observability.ZapCtx(ctx)) - opts.Progress.Observe(size, drivers.ProgressUnitByte) - return nil + return sql, nil } diff --git a/runtime/drivers/duckdb/transporter/transporter_test.go b/runtime/drivers/duckdb/transporter/transporter_test.go index 240208c2572..01950b912a4 100644 --- a/runtime/drivers/duckdb/transporter/transporter_test.go +++ b/runtime/drivers/duckdb/transporter/transporter_test.go @@ -45,9 +45,6 @@ func (m *mockIterator) Size(unit drivers.ProgressUnit) (int64, bool) { return 0, false } -func (m *mockIterator) KeepFilesUntilClose(keepFilesUntilClose bool) { -} - func (m *mockIterator) Format() string { return "" } @@ -156,8 +153,8 @@ mum,8.2`) tests = append(tests, queryTests...) mockConnector := &mockObjectStore{} - for i, test := range tests { - t.Run(fmt.Sprintf("%d - query=%v", i, test.query), func(t *testing.T) { + for _, test := range tests { + t.Run(fmt.Sprintf("%s - query=%v", test.name, test.query), func(t *testing.T) { mockConnector.mockIterator = &mockIterator{batches: test.files} olap := runOLAPStore(t) ctx := context.Background() @@ -165,7 +162,7 @@ mum,8.2`) var src map[string]any if test.query { - src = map[string]any{"sql": "select * from read_csv_auto('path',union_by_name=true,sample_size=200000)"} + src = map[string]any{"sql": "select * from read_csv_auto('path',union_by_name=true,sample_size=200000)", "allow_schema_relaxation": true} } else { src = map[string]any{"allow_schema_relaxation": true} } @@ -409,7 +406,7 @@ func TestIterativeParquetIngestionWithVariableSchema(t *testing.T) { var src map[string]any if test.query { - src = map[string]any{"sql": "select * from read_parquet('path',union_by_name=true,hive_partitioning=true)"} + src = map[string]any{"sql": "select * from read_parquet('path',union_by_name=true,hive_partitioning=true)", "allow_schema_relaxation": true} } else { src = map[string]any{"allow_schema_relaxation": true} } @@ -541,16 +538,22 @@ func TestIterativeJSONIngestionWithVariableSchema(t *testing.T) { tests = append(tests, queryTests...) mockConnector := &mockObjectStore{} - for i, test := range tests { - t.Run(fmt.Sprintf("%d - query=%v", i, test.query), func(t *testing.T) { - mockConnector.mockIterator = &mockIterator{batches: test.files} + for _, test := range tests { + t.Run(fmt.Sprintf("%s - query=%v", test.name, test.query), func(t *testing.T) { + m := &mockIterator{batches: test.files} + mockConnector.mockIterator = m olap := runOLAPStore(t) ctx := context.Background() tr := transporter.NewObjectStoreToDuckDB(mockConnector, olap, zap.NewNop()) var src map[string]any if test.query { - src = map[string]any{"sql": "select * from read_json('path',format='auto',union_by_name=true,auto_detect=true,sample_size=200000)"} + files := make([]string, 0) + for _, f := range test.files { + files = append(files, f...) + } + m.batches = [][]string{files} + src = map[string]any{"sql": "select * from read_json('path',format='auto',union_by_name=true,auto_detect=true,sample_size=200000)", "batch_size": "-1"} } else { src = map[string]any{"allow_schema_relaxation": true} } diff --git a/runtime/drivers/duckdb/transporter/utils.go b/runtime/drivers/duckdb/transporter/utils.go index 44ba5969737..ce3524fb482 100644 --- a/runtime/drivers/duckdb/transporter/utils.go +++ b/runtime/drivers/duckdb/transporter/utils.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" - "github.com/c2h5oh/datasize" "github.com/mitchellh/mapstructure" "github.com/rilldata/rill/runtime/drivers" ) @@ -65,7 +64,6 @@ type fileSourceProperties struct { Format string `mapstructure:"format"` AllowSchemaRelaxation bool `mapstructure:"allow_schema_relaxation"` BatchSize string `mapstructure:"batch_size"` - BatchSizeBytes int64 `mapstructure:"-"` // Inferred from BatchSize // Backwards compatibility HivePartitioning *bool `mapstructure:"hive_partitioning"` @@ -107,15 +105,6 @@ func parseFileSourceProperties(props map[string]any) (*fileSourceProperties, err return nil, fmt.Errorf("if any of `columns`,`types`,`dtypes` is set `allow_schema_relaxation` must be disabled") } } - - if cfg.BatchSize != "" { - b, err := datasize.ParseString(cfg.BatchSize) - if err != nil { - return nil, err - } - cfg.BatchSizeBytes = int64(b.Bytes()) - } - return cfg, nil } diff --git a/runtime/drivers/gcs/gcs.go b/runtime/drivers/gcs/gcs.go index a78815095a6..7e1703946da 100644 --- a/runtime/drivers/gcs/gcs.go +++ b/runtime/drivers/gcs/gcs.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "net/http" "os" "strings" @@ -288,9 +289,14 @@ func (c *Connection) DownloadFiles(ctx context.Context, props map[string]any) (d return nil, fmt.Errorf("failed to open bucket %q, %w", conf.url.Host, err) } - batchSize, err := datasize.ParseString(conf.BatchSize) - if err != nil { - return nil, err + var batchSize datasize.ByteSize + if conf.BatchSize == "-1" { + batchSize = math.MaxInt64 // download everything in one batch + } else { + batchSize, err = datasize.ParseString(conf.BatchSize) + if err != nil { + return nil, err + } } // prepare fetch configs opts := rillblob.Options{ @@ -301,6 +307,7 @@ func (c *Connection) DownloadFiles(ctx context.Context, props map[string]any) (d GlobPattern: conf.url.Path, ExtractPolicy: conf.extractPolicy, BatchSizeBytes: int64(batchSize.Bytes()), + KeepFilesUntilClose: conf.BatchSize == "-1", } iter, err := rillblob.NewIterator(ctx, bucketObj, opts, c.logger) diff --git a/runtime/drivers/object_store.go b/runtime/drivers/object_store.go index 2621155c0b3..90c07ad0963 100644 --- a/runtime/drivers/object_store.go +++ b/runtime/drivers/object_store.go @@ -18,9 +18,6 @@ type FileIterator interface { // Size returns size of data downloaded in unit. // Returns 0,false if not able to compute size in given unit Size(unit ProgressUnit) (int64, bool) - // KeepFilesUntilClose marks the iterator to keep the files until close is called. - // This is used when the entire list of files is used at once in certain cases. - KeepFilesUntilClose(keepFilesUntilClose bool) // Format returns general file format (json, csv, parquet, etc) // Returns an empty string if there is no general format Format() string diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 9fa0fb39cc9..0724b6e99c6 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "net/http" "github.com/aws/aws-sdk-go/aws" @@ -292,9 +293,14 @@ func (c *Connection) DownloadFiles(ctx context.Context, src map[string]any) (dri return nil, fmt.Errorf("failed to open bucket %q, %w", conf.url.Host, err) } - batchSize, err := datasize.ParseString(conf.BatchSize) - if err != nil { - return nil, err + var batchSize datasize.ByteSize + if conf.BatchSize == "-1" { + batchSize = math.MaxInt64 // download everything in one batch + } else { + batchSize, err = datasize.ParseString(conf.BatchSize) + if err != nil { + return nil, err + } } // prepare fetch configs opts := rillblob.Options{ @@ -305,6 +311,7 @@ func (c *Connection) DownloadFiles(ctx context.Context, src map[string]any) (dri GlobPattern: conf.url.Path, ExtractPolicy: conf.extractPolicy, BatchSizeBytes: int64(batchSize.Bytes()), + KeepFilesUntilClose: conf.BatchSize == "-1", } it, err := rillblob.NewIterator(ctx, bucketObj, opts, c.logger) diff --git a/runtime/queries/column_numeric_histogram.go b/runtime/queries/column_numeric_histogram.go index 743cf74bc39..cad73e289bc 100644 --- a/runtime/queries/column_numeric_histogram.go +++ b/runtime/queries/column_numeric_histogram.go @@ -204,6 +204,7 @@ func (q *ColumnNumericHistogram) calculateFDMethod(ctx context.Context, rt *runt -- fill in the case where we've filtered out the highest value and need to recompute it, otherwise use count. CASE WHEN high = (SELECT max(high) from histogram_stage) THEN count + (select c from right_edge) ELSE count END AS count FROM histogram_stage + ORDER BY bucket `, selectColumn, sanitizedColumnName, @@ -332,6 +333,7 @@ func (q *ColumnNumericHistogram) calculateDiagnosticMethod(ctx context.Context, -- fill in the case where we've filtered out the highest value and need to recompute it, otherwise use count. CASE WHEN high = (SELECT max(high) from histogram_stage) THEN count + (select c from right_edge) ELSE count END AS count FROM histogram_stage + ORDER BY bucket `, selectColumn, sanitizedColumnName, diff --git a/runtime/queries/metricsview.go b/runtime/queries/metricsview.go index ffadb7ac468..18e19f31519 100644 --- a/runtime/queries/metricsview.go +++ b/runtime/queries/metricsview.go @@ -197,10 +197,11 @@ func buildFilterClauseForCondition(mv *runtimev1.MetricsViewSpec, cond *runtimev // NOTE: Looking up for dimension like this will lead to O(nm). // Ideal way would be to create a map, but we need to find a clean solution down the line - name, err := metricsViewDimensionToSafeColumn(mv, cond.Name) + dim, err := metricsViewDimension(mv, cond.Name) if err != nil { return "", nil, err } + name := safeName(metricsViewDimensionColumn(dim)) notKeyword := "" if exclude { @@ -228,7 +229,13 @@ func buildFilterClauseForCondition(mv *runtimev1.MetricsViewSpec, cond *runtimev // If there were non-null args, add a "dim [NOT] IN (...)" clause if len(args) > 0 { questionMarks := strings.Join(repeatString("?", len(args)), ",") - clause := fmt.Sprintf("%s %s IN (%s)", name, notKeyword, questionMarks) + var clause string + // Build [NOT] list_has_any("dim", ARRAY[?, ?, ...]) + if dim.Unnest && dialect != drivers.DialectDruid { + clause = fmt.Sprintf("%s list_has_any(%s, ARRAY[%s])", notKeyword, name, questionMarks) + } else { + clause = fmt.Sprintf("%s %s IN (%s)", name, notKeyword, questionMarks) + } clauses = append(clauses, clause) } } @@ -237,11 +244,16 @@ func buildFilterClauseForCondition(mv *runtimev1.MetricsViewSpec, cond *runtimev if len(cond.Like) > 0 { for _, val := range cond.Like { var clause string - if dialect == drivers.DialectDruid { - // Druid does not support ILIKE - clause = fmt.Sprintf("LOWER(%s) %s LIKE LOWER(?)", name, notKeyword) + // Build [NOT] len(list_filter("dim", x -> x ILIKE ?)) > 0 + if dim.Unnest && dialect != drivers.DialectDruid { + clause = fmt.Sprintf("%s len(list_filter(%s, x -> x %s ILIKE ?)) > 0", notKeyword, name, notKeyword) } else { - clause = fmt.Sprintf("%s %s ILIKE ?", name, notKeyword) + if dialect == drivers.DialectDruid { + // Druid does not support ILIKE + clause = fmt.Sprintf("LOWER(%s) %s LIKE LOWER(?)", name, notKeyword) + } else { + clause = fmt.Sprintf("%s %s ILIKE ?", name, notKeyword) + } } args = append(args, val) @@ -322,17 +334,29 @@ func convertToXLSXValue(pbvalue *structpb.Value) (interface{}, error) { func metricsViewDimensionToSafeColumn(mv *runtimev1.MetricsViewSpec, dimName string) (string, error) { dimName = strings.ToLower(dimName) + dimension, err := metricsViewDimension(mv, dimName) + if err != nil { + return "", err + } + return safeName(metricsViewDimensionColumn(dimension)), nil +} + +func metricsViewDimension(mv *runtimev1.MetricsViewSpec, dimName string) (*runtimev1.MetricsViewSpec_DimensionV2, error) { for _, dimension := range mv.Dimensions { if strings.EqualFold(dimension.Name, dimName) { - if dimension.Column != "" { - return safeName(dimension.Column), nil - } - // backwards compatibility for older projects that have not run reconcile on this dashboard - // in that case `column` will not be present - return safeName(dimension.Name), nil + return dimension, nil } } - return "", fmt.Errorf("dimension %s not found", dimName) + return nil, fmt.Errorf("dimension %s not found", dimName) +} + +func metricsViewDimensionColumn(dimension *runtimev1.MetricsViewSpec_DimensionV2) string { + if dimension.Column != "" { + return dimension.Column + } + // backwards compatibility for older projects that have not run reconcile on this dashboard + // in that case `column` will not be present + return dimension.Name } func metricsViewMeasureExpression(mv *runtimev1.MetricsViewSpec, measureName string) (string, error) { diff --git a/runtime/queries/metricsview_aggregation.go b/runtime/queries/metricsview_aggregation.go index c40d14c1435..5185ca561d0 100644 --- a/runtime/queries/metricsview_aggregation.go +++ b/runtime/queries/metricsview_aggregation.go @@ -137,18 +137,28 @@ func (q *MetricsViewAggregation) buildMetricsAggregationSQL(mv *runtimev1.Metric selectCols := make([]string, 0, len(q.Dimensions)+len(q.Measures)) groupCols := make([]string, 0, len(q.Dimensions)) + unnestClauses := make([]string, 0) args := []any{} for _, d := range q.Dimensions { // Handle regular dimensions if d.TimeGrain == runtimev1.TimeGrain_TIME_GRAIN_UNSPECIFIED { - col, err := metricsViewDimensionToSafeColumn(mv, d.Name) + dim, err := metricsViewDimension(mv, d.Name) if err != nil { return "", nil, err } - - selectCols = append(selectCols, fmt.Sprintf("%s as %s", col, safeName(d.Name))) - groupCols = append(groupCols, col) + rawColName := metricsViewDimensionColumn(dim) + col := safeName(rawColName) + + if dim.Unnest && dialect != drivers.DialectDruid { + // select "unnested_colName" as "colName" ... FROM "mv_table", LATERAL UNNEST("mv_table"."colName") tbl("unnested_colName") ... + unnestColName := safeName(tempName(fmt.Sprintf("%s_%s_", "unnested", rawColName))) + selectCols = append(selectCols, fmt.Sprintf(`%s as %s`, unnestColName, col)) + unnestClauses = append(unnestClauses, fmt.Sprintf(`, LATERAL UNNEST(%s.%s) tbl(%s)`, safeName(mv.Table), col, unnestColName)) + } else { + selectCols = append(selectCols, fmt.Sprintf("%s as %s", col, safeName(d.Name))) + groupCols = append(groupCols, col) + } continue } @@ -194,7 +204,7 @@ func (q *MetricsViewAggregation) buildMetricsAggregationSQL(mv *runtimev1.Metric whereClause := "" if mv.TimeDimension != "" { timeCol := safeName(mv.TimeDimension) - clause, err := timeRangeClause(q.TimeRange, dialect, timeCol, &args) + clause, err := timeRangeClause(q.TimeRange, mv, dialect, timeCol, &args) if err != nil { return "", nil, err } @@ -236,9 +246,10 @@ func (q *MetricsViewAggregation) buildMetricsAggregationSQL(mv *runtimev1.Metric limitClause = fmt.Sprintf("LIMIT %d", *q.Limit) } - sql := fmt.Sprintf("SELECT %s FROM %s %s %s %s %s OFFSET %d", + sql := fmt.Sprintf("SELECT %s FROM %s %s %s %s %s %s OFFSET %d", strings.Join(selectCols, ", "), safeName(mv.Table), + strings.Join(unnestClauses, ""), whereClause, groupClause, orderClause, diff --git a/runtime/queries/metricsview_comparison_toplist.go b/runtime/queries/metricsview_comparison_toplist.go index 453834dc8df..9dffbad8d10 100644 --- a/runtime/queries/metricsview_comparison_toplist.go +++ b/runtime/queries/metricsview_comparison_toplist.go @@ -6,12 +6,10 @@ import ( "fmt" "io" "strings" - "time" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" "github.com/rilldata/rill/runtime/drivers" - "github.com/rilldata/rill/runtime/pkg/duration" "github.com/rilldata/rill/runtime/pkg/pbutil" "google.golang.org/protobuf/types/known/structpb" @@ -222,12 +220,23 @@ func (q *MetricsViewComparison) executeComparisonToplist(ctx context.Context, ol } func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsViewSpec, dialect drivers.Dialect, policy *runtime.ResolvedMetricsViewSecurity) (string, []any, error) { - colName, err := metricsViewDimensionToSafeColumn(mv, q.DimensionName) + dim, err := metricsViewDimension(mv, q.DimensionName) if err != nil { return "", nil, err } + rawColName := metricsViewDimensionColumn(dim) + colName := safeName(rawColName) + unnestColName := safeName(tempName(fmt.Sprintf("%s_%s_", "unnested", rawColName))) - selectCols := []string{colName} + var selectCols []string + unnestClause := "" + if dim.Unnest && dialect != drivers.DialectDruid { + // select "unnested_colName" as "colName" ... FROM "mv_table", LATERAL UNNEST("mv_table"."colName") tbl("unnested_colName") ... + selectCols = append(selectCols, fmt.Sprintf(`%s as %s`, unnestColName, colName)) + unnestClause = fmt.Sprintf(`, LATERAL UNNEST(%s.%s) tbl(%s)`, safeName(mv.Table), colName, unnestColName) + } else { + selectCols = append(selectCols, colName) + } for _, m := range q.Measures { switch m.BuiltinMeasure { @@ -259,7 +268,7 @@ func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsView args := []any{} td := safeName(mv.TimeDimension) - trc, err := timeRangeClause(q.TimeRange, dialect, td, &args) + trc, err := timeRangeClause(q.TimeRange, mv, dialect, td, &args) if err != nil { return "", nil, err } @@ -302,27 +311,44 @@ func (q *MetricsViewComparison) buildMetricsTopListSQL(mv *runtimev1.MetricsView limitClause = fmt.Sprintf(" LIMIT %d", q.Limit) } + groupByCol := colName + if dim.Unnest && dialect != drivers.DialectDruid { + groupByCol = unnestColName + } + sql := fmt.Sprintf( - `SELECT %[1]s FROM %[3]q WHERE %[4]s GROUP BY %[2]s ORDER BY %[5]s %[6]s OFFSET %[7]d`, - selectClause, // 1 - colName, // 2 - mv.Table, // 3 - baseWhereClause, // 4 - orderClause, // 5 - limitClause, // 6 - q.Offset, // 7 + `SELECT %[1]s FROM %[3]s %[8]s WHERE %[4]s GROUP BY %[2]s ORDER BY %[5]s %[6]s OFFSET %[7]d`, + selectClause, // 1 + groupByCol, // 2 + safeName(mv.Table), // 3 + baseWhereClause, // 4 + orderClause, // 5 + limitClause, // 6 + q.Offset, // 7 + unnestClause, // 8 ) return sql, args, nil } func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.MetricsViewSpec, dialect drivers.Dialect, policy *runtime.ResolvedMetricsViewSecurity) (string, []any, error) { - colName, err := metricsViewDimensionToSafeColumn(mv, q.DimensionName) + dim, err := metricsViewDimension(mv, q.DimensionName) if err != nil { return "", nil, err } + rawColName := metricsViewDimensionColumn(dim) + colName := safeName(rawColName) + unnestColName := safeName(tempName(fmt.Sprintf("%s_%s_", "unnested", rawColName))) - selectCols := []string{colName} + var selectCols []string + unnestClause := "" + if dim.Unnest && dialect != drivers.DialectDruid { + // select "unnested_colName" as "colName" ... FROM "mv_table", LATERAL UNNEST("mv_table"."colName") tbl("unnested_colName") ... + selectCols = append(selectCols, fmt.Sprintf(`%s as %s`, unnestColName, colName)) + unnestClause = fmt.Sprintf(`, LATERAL UNNEST(%s.%s) tbl(%s)`, safeName(mv.Table), colName, unnestColName) + } else { + selectCols = append(selectCols, colName) + } for _, m := range q.Measures { switch m.BuiltinMeasure { @@ -386,7 +412,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M td := safeName(mv.TimeDimension) - trc, err := timeRangeClause(q.TimeRange, dialect, td, &args) + trc, err := timeRangeClause(q.TimeRange, mv, dialect, td, &args) if err != nil { return "", nil, err } @@ -402,7 +428,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M args = append(args, clauseArgs...) } - trc, err = timeRangeClause(q.ComparisonTimeRange, dialect, td, &args) + trc, err = timeRangeClause(q.ComparisonTimeRange, mv, dialect, td, &args) if err != nil { return "", nil, err } @@ -525,16 +551,21 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M LIMIT 10 OFFSET 0 */ + groupByCol := colName + if dim.Unnest && dialect != drivers.DialectDruid { + groupByCol = unnestColName + } + var sql string if dialect != drivers.DialectDruid { sql = fmt.Sprintf(` SELECT COALESCE(base.%[2]s, comparison.%[2]s) AS %[10]s, %[9]s FROM ( - SELECT %[1]s FROM %[3]q WHERE %[4]s GROUP BY %[2]s %[12]s + SELECT %[1]s FROM %[3]s %[14]s WHERE %[4]s GROUP BY %[15]s %[12]s ) base %[11]s JOIN ( - SELECT %[1]s FROM %[3]q WHERE %[5]s GROUP BY %[2]s %[13]s + SELECT %[1]s FROM %[3]s %[14]s WHERE %[5]s GROUP BY %[15]s %[13]s ) comparison ON base.%[2]s = comparison.%[2]s OR (base.%[2]s is null and comparison.%[2]s is null) @@ -546,7 +577,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M `, subSelectClause, // 1 colName, // 2 - mv.Table, // 3 + safeName(mv.Table), // 3 baseWhereClause, // 4 comparisonWhereClause, // 5 orderClause, // 6 @@ -556,7 +587,9 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M safeName(q.DimensionName), // 10 joinType, // 11 baseLimitClause, // 12 - comparisonLimitClause, // 12 + comparisonLimitClause, // 13 + unnestClause, // 14 + groupByCol, // 15 ) } else { /* @@ -605,11 +638,11 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M sql = fmt.Sprintf(` SELECT %[11]s.%[2]s, %[9]s FROM ( - SELECT %[1]s FROM %[3]q WHERE %[4]s GROUP BY %[2]s ORDER BY %[13]s %[10]s OFFSET %[8]d + SELECT %[1]s FROM %[3]s WHERE %[4]s GROUP BY %[2]s ORDER BY %[13]s %[10]s OFFSET %[8]d ) %[11]s LEFT OUTER JOIN ( - SELECT %[1]s FROM %[3]q WHERE %[5]s GROUP BY %[2]s + SELECT %[1]s FROM %[3]s WHERE %[5]s GROUP BY %[2]s ) %[12]s ON base.%[2]s = comparison.%[2]s @@ -624,7 +657,7 @@ func (q *MetricsViewComparison) buildMetricsComparisonTopListSQL(mv *runtimev1.M subSelectClause, // 1 colName, // 2 - mv.Table, // 3 + safeName(mv.Table), // 3 leftWhereClause, // 4 rightWhereClause, // 5 orderClause, // 6 @@ -800,60 +833,15 @@ func (q *MetricsViewComparison) generateFilename() string { // TODO: a) Ensure correct time zone handling, b) Implement support for tr.RoundToGrain // (Maybe consider pushing all this logic into the SQL instead?) -func timeRangeClause(tr *runtimev1.TimeRange, dialect drivers.Dialect, timeCol string, args *[]any) (string, error) { +func timeRangeClause(tr *runtimev1.TimeRange, mv *runtimev1.MetricsViewSpec, dialect drivers.Dialect, timeCol string, args *[]any) (string, error) { var clause string if isTimeRangeNil(tr) { return clause, nil } - tz := time.UTC - if tr.TimeZone != "" { - var err error - tz, err = time.LoadLocation(tr.TimeZone) - if err != nil { - return "", fmt.Errorf("invalid time_range.time_zone %q: %w", tr.TimeZone, err) - } - } - - var start, end time.Time - if tr.Start != nil { - start = tr.Start.AsTime().In(tz) - } - if tr.End != nil { - end = tr.End.AsTime().In(tz) - } - - if tr.IsoDuration != "" { - if !start.IsZero() && !end.IsZero() { - return "", fmt.Errorf("only two of time_range.{start,end,iso_duration} can be specified") - } - - d, err := duration.ParseISO8601(tr.IsoDuration) - if err != nil { - return "", fmt.Errorf("invalid iso_duration %q: %w", tr.IsoDuration, err) - } - - if !start.IsZero() { - end = d.Add(start) - } else if !end.IsZero() { - start = d.Sub(end) - } else { - return "", fmt.Errorf("one of time_range.{start,end} must be specified with time_range.iso_duration") - } - } - - if tr.IsoOffset != "" { - d, err := duration.ParseISO8601(tr.IsoOffset) - if err != nil { - return "", fmt.Errorf("invalid iso_offset %q: %w", tr.IsoOffset, err) - } - - if !start.IsZero() { - start = d.Add(start) - } - if !end.IsZero() { - end = d.Add(end) - } + start, end, err := ResolveTimeRange(tr, mv) + if err != nil { + return "", err } if !start.IsZero() { diff --git a/runtime/queries/metricsview_rows.go b/runtime/queries/metricsview_rows.go index 2e71737f5fe..b38e8702b33 100644 --- a/runtime/queries/metricsview_rows.go +++ b/runtime/queries/metricsview_rows.go @@ -279,9 +279,9 @@ func (q *MetricsViewRows) buildMetricsRowsSQL(mv *runtimev1.MetricsViewSpec, dia selectColumns = append([]string{rollup}, selectColumns...) } - sql := fmt.Sprintf("SELECT %s FROM %q WHERE %s %s %s OFFSET %d", + sql := fmt.Sprintf("SELECT %s FROM %s WHERE %s %s %s OFFSET %d", strings.Join(selectColumns, ","), - mv.Table, + safeName(mv.Table), whereClause, orderClause, limitClause, diff --git a/runtime/queries/metricsview_timeseries.go b/runtime/queries/metricsview_timeseries.go index 255271bf3a4..e9e702a80e0 100644 --- a/runtime/queries/metricsview_timeseries.go +++ b/runtime/queries/metricsview_timeseries.go @@ -161,21 +161,27 @@ func (q *MetricsViewTimeSeries) Resolve(ctx context.Context, rt *runtime.Runtime if zeroTime.Equal(start) { if q.TimeStart != nil { - start = truncateTime(q.TimeStart.AsTime(), q.TimeGranularity, tz, int(fdow), int(fmoy)) - data = addNulls(data, nullRecords, start, t, q.TimeGranularity) + start = TruncateTime(q.TimeStart.AsTime(), q.TimeGranularity, tz, int(fdow), int(fmoy)) + data = addNulls(data, nullRecords, start, t, q.TimeGranularity, tz) } } else { - data = addNulls(data, nullRecords, start, t, q.TimeGranularity) + data = addNulls(data, nullRecords, start, t, q.TimeGranularity, tz) } data = append(data, &runtimev1.TimeSeriesValue{ Ts: timestamppb.New(t), Records: records, }) - start = addTo(t, q.TimeGranularity) + start = addTo(t, q.TimeGranularity, tz) } if q.TimeEnd != nil && nullRecords != nil { - data = addNulls(data, nullRecords, start, q.TimeEnd.AsTime(), q.TimeGranularity) + if start.Equal(zeroTime) && q.TimeStart != nil { + start = q.TimeStart.AsTime() + } + + if !start.Equal(zeroTime) { + data = addNulls(data, nullRecords, start, q.TimeEnd.AsTime(), q.TimeGranularity, tz) + } } meta := structTypeToMetricsViewColumn(rows.Schema) @@ -268,7 +274,7 @@ func (q *MetricsViewTimeSeries) buildMetricsTimeseriesSQL(olap drivers.OLAPStore } if q.Filter != nil { - clause, clauseArgs, err := buildFilterClauseForMetricsViewFilter(mv, q.Filter, drivers.DialectDruid, policy) + clause, clauseArgs, err := buildFilterClauseForMetricsViewFilter(mv, q.Filter, olap.Dialect(), policy) if err != nil { return "", "", nil, err } @@ -310,11 +316,11 @@ func (q *MetricsViewTimeSeries) buildDruidSQL(args []any, mv *runtimev1.MetricsV } sql := fmt.Sprintf( - `SELECT %s AS %s, %s FROM %q WHERE %s GROUP BY 1 ORDER BY 1`, + `SELECT %s AS %s, %s FROM %s WHERE %s GROUP BY 1 ORDER BY 1`, timeClause, tsAlias, strings.Join(selectCols, ", "), - mv.Table, + safeName(mv.Table), whereClause, ) @@ -334,12 +340,12 @@ func (q *MetricsViewTimeSeries) buildDuckDBSQL(args []any, mv *runtimev1.Metrics } sql := fmt.Sprintf( - `SELECT timezone(?, date_trunc('%[1]s', timezone(?, %[2]s::TIMESTAMPTZ) + INTERVAL %[7]s) - INTERVAL %[7]s) as %[3]s, %[4]s FROM %[5]q WHERE %[6]s GROUP BY 1 ORDER BY 1`, + `SELECT timezone(?, date_trunc('%[1]s', timezone(?, %[2]s::TIMESTAMPTZ) + INTERVAL %[7]s) - INTERVAL %[7]s) as %[3]s, %[4]s FROM %[5]s WHERE %[6]s GROUP BY 1 ORDER BY 1`, dateTruncSpecifier, // 1 safeName(mv.TimeDimension), // 2 tsAlias, // 3 strings.Join(selectCols, ", "), // 4 - mv.Table, // 5 + safeName(mv.Table), // 5 whereClause, // 6 shift, // 7 ) @@ -347,67 +353,6 @@ func (q *MetricsViewTimeSeries) buildDuckDBSQL(args []any, mv *runtimev1.Metrics return sql } -func truncateTime(start time.Time, tg runtimev1.TimeGrain, tz *time.Location, firstDay, firstMonth int) time.Time { - switch tg { - case runtimev1.TimeGrain_TIME_GRAIN_MILLISECOND: - return start.Truncate(time.Millisecond) - case runtimev1.TimeGrain_TIME_GRAIN_SECOND: - return start.Truncate(time.Second) - case runtimev1.TimeGrain_TIME_GRAIN_MINUTE: - return start.Truncate(time.Minute) - case runtimev1.TimeGrain_TIME_GRAIN_HOUR: - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), start.Day(), start.Hour(), 0, 0, 0, tz) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_DAY: - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_WEEK: - start = start.In(tz) - weekday := int(start.Weekday()) - if weekday == 0 { - weekday = 7 - } - if firstDay < 1 { - firstDay = 1 - } - if firstDay > 7 { - firstDay = 7 - } - - daysToSubtract := -(weekday - firstDay) - if weekday < firstDay { - daysToSubtract = -7 + daysToSubtract - } - start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) - start = start.AddDate(0, 0, daysToSubtract) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_MONTH: - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) - start = start.In(time.UTC) - return start - case runtimev1.TimeGrain_TIME_GRAIN_QUARTER: - monthsToSubtract := 1 - int(start.Month())%3 // todo first month of year - start = start.In(tz) - start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) - start = start.AddDate(0, monthsToSubtract, 0) - return start.In(time.UTC) - case runtimev1.TimeGrain_TIME_GRAIN_YEAR: - start = start.In(tz) - year := start.Year() - if int(start.Month()) < firstMonth { - year = start.Year() - 1 - } - - start = time.Date(year, time.Month(firstMonth), 1, 0, 0, 0, 0, tz) - return start.In(time.UTC) - } - - return start -} - func generateNullRecords(schema *runtimev1.StructType) *structpb.Struct { nullStruct := structpb.Struct{Fields: make(map[string]*structpb.Value, len(schema.Fields))} for _, f := range schema.Fields { @@ -416,18 +361,18 @@ func generateNullRecords(schema *runtimev1.StructType) *structpb.Struct { return &nullStruct } -func addNulls(data []*runtimev1.TimeSeriesValue, nullRecords *structpb.Struct, start, end time.Time, tg runtimev1.TimeGrain) []*runtimev1.TimeSeriesValue { +func addNulls(data []*runtimev1.TimeSeriesValue, nullRecords *structpb.Struct, start, end time.Time, tg runtimev1.TimeGrain, tz *time.Location) []*runtimev1.TimeSeriesValue { for start.Before(end) { data = append(data, &runtimev1.TimeSeriesValue{ Ts: timestamppb.New(start), Records: nullRecords, }) - start = addTo(start, tg) + start = addTo(start, tg, tz) } return data } -func addTo(start time.Time, tg runtimev1.TimeGrain) time.Time { +func addTo(start time.Time, tg runtimev1.TimeGrain, tz *time.Location) time.Time { switch tg { case runtimev1.TimeGrain_TIME_GRAIN_MILLISECOND: return start.Add(time.Millisecond) @@ -442,9 +387,13 @@ func addTo(start time.Time, tg runtimev1.TimeGrain) time.Time { case runtimev1.TimeGrain_TIME_GRAIN_WEEK: return start.AddDate(0, 0, 7) case runtimev1.TimeGrain_TIME_GRAIN_MONTH: - return start.AddDate(0, 1, 0) + start = start.In(tz) + start = start.AddDate(0, 1, 0) + return start.In(time.UTC) case runtimev1.TimeGrain_TIME_GRAIN_QUARTER: - return start.AddDate(0, 3, 0) + start = start.In(tz) + start = start.AddDate(0, 3, 0) + return start.In(time.UTC) case runtimev1.TimeGrain_TIME_GRAIN_YEAR: return start.AddDate(1, 0, 0) } diff --git a/runtime/queries/metricsview_timeseries_test.go b/runtime/queries/metricsview_timeseries_test.go index 02ddc421543..1a4927e61da 100644 --- a/runtime/queries/metricsview_timeseries_test.go +++ b/runtime/queries/metricsview_timeseries_test.go @@ -1,75 +1,183 @@ -package queries +package queries_test import ( + "context" + // "fmt" "testing" - "time" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime" + "github.com/rilldata/rill/runtime/queries" + "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" ) -func TestTruncateTime(t *testing.T) { - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:00:00Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T00:00:00Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), truncateTime(parseTestTime(t, "2019-01-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-04-01T00:00:00Z"), truncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, time.UTC, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), truncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, time.UTC, 1, 1)) -} +func TestMetricsViewsTimeseries_month_grain(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "timeseries") -func TestTruncateTime_Kathmandu(t *testing.T) { - tz, err := time.LoadLocation("Asia/Kathmandu") + ctrl, err := rt.Controller(context.Background(), instanceID) require.NoError(t, err) - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-07T04:15:00Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-06T18:15:00Z"), truncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2023-10-08T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-01-31T18:15:00Z"), truncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2019-03-31T18:15:00Z"), truncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 1, 1)) - require.Equal(t, parseTestTime(t, "2018-12-31T18:15:00Z"), truncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 1, 1)) -} + r, err := ctrl.Get(context.Background(), &runtimev1.ResourceName{Kind: runtime.ResourceKindMetricsView, Name: "timeseries_year"}, false) + require.NoError(t, err) + mv := r.GetMetricsView() -func TestTruncateTime_UTC_first_day(t *testing.T) { - tz := time.UTC - require.Equal(t, parseTestTime(t, "2023-10-08T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) - require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-10T00:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) -} + q := &queries.MetricsViewTimeSeries{ + MeasureNames: []string{"max_clicks"}, + MetricsViewName: "timeseries_year", + MetricsView: mv.Spec, + TimeStart: parseTime(t, "2023-01-01T00:00:00Z"), + TimeEnd: parseTime(t, "2024-01-01T00:00:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_MONTH, + Limit: 250, + } -func TestTruncateTime_Kathmandu_first_day(t *testing.T) { - tz, err := time.LoadLocation("Asia/Kathmandu") + err = q.Resolve(context.Background(), rt, instanceID, 0) require.NoError(t, err) - require.Equal(t, parseTestTime(t, "2023-10-07T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) - require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-09T18:16:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.NotEmpty(t, q.Result) + rows := q.Result.Data + i := 0 + require.Equal(t, parseTime(t, "2023-01-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-02-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-03-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-04-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-05-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-06-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-07-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-08-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-09-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-10-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-11-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-12-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) } -func TestTruncateTime_UTC_first_month(t *testing.T) { - tz := time.UTC - require.Equal(t, parseTestTime(t, "2023-02-01T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) - require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), truncateTime(parseTestTime(t, "2023-03-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), truncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) - require.Equal(t, parseTestTime(t, "2023-01-01T00:00:00Z"), truncateTime(parseTestTime(t, "2023-01-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) +func TestMetricsViewsTimeseries_month_grain_IST(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "timeseries") + + ctrl, err := rt.Controller(context.Background(), instanceID) + require.NoError(t, err) + r, err := ctrl.Get(context.Background(), &runtimev1.ResourceName{Kind: runtime.ResourceKindMetricsView, Name: "timeseries_year"}, false) + require.NoError(t, err) + mv := r.GetMetricsView() + + q := &queries.MetricsViewTimeSeries{ + MeasureNames: []string{"max_clicks"}, + MetricsViewName: "timeseries_year", + MetricsView: mv.Spec, + TimeStart: parseTime(t, "2022-12-31T18:30:00Z"), + TimeEnd: parseTime(t, "2024-01-31T18:30:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_MONTH, + TimeZone: "Asia/Kolkata", + Limit: 250, + } + + err = q.Resolve(context.Background(), rt, instanceID, 0) + require.NoError(t, err) + require.NotEmpty(t, q.Result) + rows := q.Result.Data + i := 0 + require.Equal(t, parseTime(t, "2022-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-01-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-02-28T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-03-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-04-30T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-05-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-06-30T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-07-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-08-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-09-30T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-10-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-11-30T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) } -func TestTruncateTime_Kathmandu_first_month(t *testing.T) { - tz, err := time.LoadLocation("Asia/Kathmandu") +func TestMetricsViewsTimeseries_quarter_grain_IST(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "timeseries") + + ctrl, err := rt.Controller(context.Background(), instanceID) require.NoError(t, err) - require.Equal(t, parseTestTime(t, "2023-01-31T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) - require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), truncateTime(parseTestTime(t, "2023-03-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) - require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), truncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) - require.Equal(t, parseTestTime(t, "2022-12-31T18:15:00Z"), truncateTime(parseTestTime(t, "2023-01-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) + r, err := ctrl.Get(context.Background(), &runtimev1.ResourceName{Kind: runtime.ResourceKindMetricsView, Name: "timeseries_year"}, false) + require.NoError(t, err) + mv := r.GetMetricsView() + + q := &queries.MetricsViewTimeSeries{ + MeasureNames: []string{"max_clicks"}, + MetricsViewName: "timeseries_year", + MetricsView: mv.Spec, + TimeStart: parseTime(t, "2022-12-31T18:30:00Z"), + TimeEnd: parseTime(t, "2024-01-31T18:30:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_QUARTER, + TimeZone: "Asia/Kolkata", + Limit: 250, + } + + err = q.Resolve(context.Background(), rt, instanceID, 0) + require.NoError(t, err) + require.NotEmpty(t, q.Result) + rows := q.Result.Data + require.Len(t, rows, 6) + i := 0 + require.Equal(t, parseTime(t, "2022-10-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2022-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-03-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-06-30T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-09-30T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) } -func parseTestTime(tst *testing.T, t string) time.Time { - ts, err := time.Parse(time.RFC3339, t) - require.NoError(tst, err) - return ts +func TestMetricsViewsTimeseries_year_grain_IST(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "timeseries") + + ctrl, err := rt.Controller(context.Background(), instanceID) + require.NoError(t, err) + r, err := ctrl.Get(context.Background(), &runtimev1.ResourceName{Kind: runtime.ResourceKindMetricsView, Name: "timeseries_year"}, false) + require.NoError(t, err) + mv := r.GetMetricsView() + + q := &queries.MetricsViewTimeSeries{ + MeasureNames: []string{"max_clicks"}, + MetricsViewName: "timeseries_year", + MetricsView: mv.Spec, + TimeStart: parseTime(t, "2022-12-31T18:30:00Z"), + TimeEnd: parseTime(t, "2024-12-31T00:00:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_YEAR, + TimeZone: "Asia/Kolkata", + Limit: 250, + } + + err = q.Resolve(context.Background(), rt, instanceID, 0) + require.NoError(t, err) + require.NotEmpty(t, q.Result) + rows := q.Result.Data + i := 0 + require.Equal(t, parseTime(t, "2022-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ + require.Equal(t, parseTime(t, "2023-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) } diff --git a/runtime/queries/metricsview_toplist.go b/runtime/queries/metricsview_toplist.go index 207bee92fbe..cbac2e637a3 100644 --- a/runtime/queries/metricsview_toplist.go +++ b/runtime/queries/metricsview_toplist.go @@ -178,12 +178,24 @@ func (q *MetricsViewToplist) buildMetricsTopListSQL(mv *runtimev1.MetricsViewSpe return "", nil, err } - colName, err := metricsViewDimensionToSafeColumn(mv, q.DimensionName) + dim, err := metricsViewDimension(mv, q.DimensionName) if err != nil { return "", nil, err } + rawColName := metricsViewDimensionColumn(dim) + colName := safeName(rawColName) + unnestColName := safeName(tempName(fmt.Sprintf("%s_%s_", "unnested", rawColName))) + + var selectCols []string + unnestClause := "" + if dim.Unnest && dialect != drivers.DialectDruid { + // select "unnested_colName" as "colName" ... FROM "mv_table", LATERAL UNNEST("mv_table"."colName") tbl("unnested_colName") ... + selectCols = append(selectCols, fmt.Sprintf(`%s as %s`, unnestColName, colName)) + unnestClause = fmt.Sprintf(`, LATERAL UNNEST(%s.%s) tbl(%s)`, safeName(mv.Table), colName, unnestColName) + } else { + selectCols = append(selectCols, colName) + } - selectCols := []string{colName} for _, m := range ms { expr := fmt.Sprintf(`%s as "%s"`, m.Expression, m.Name) selectCols = append(selectCols, expr) @@ -232,11 +244,17 @@ func (q *MetricsViewToplist) buildMetricsTopListSQL(mv *runtimev1.MetricsViewSpe limitClause = fmt.Sprintf("LIMIT %d", *q.Limit) } - sql := fmt.Sprintf("SELECT %s FROM %q WHERE %s GROUP BY %s %s %s OFFSET %d", + groupByCol := colName + if dim.Unnest && dialect != drivers.DialectDruid { + groupByCol = unnestColName + } + + sql := fmt.Sprintf("SELECT %s FROM %s %s WHERE %s GROUP BY %s %s %s OFFSET %d", strings.Join(selectCols, ", "), - mv.Table, + safeName(mv.Table), + unnestClause, whereClause, - colName, + groupByCol, orderClause, limitClause, q.Offset, diff --git a/runtime/queries/timeutil.go b/runtime/queries/timeutil.go new file mode 100644 index 00000000000..a0e2aca36b6 --- /dev/null +++ b/runtime/queries/timeutil.go @@ -0,0 +1,153 @@ +package queries + +import ( + "fmt" + "time" + + runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/pkg/duration" + + // Load IANA time zone data + _ "time/tzdata" +) + +func TruncateTime(start time.Time, tg runtimev1.TimeGrain, tz *time.Location, firstDay, firstMonth int) time.Time { + switch tg { + case runtimev1.TimeGrain_TIME_GRAIN_MILLISECOND: + return start.Truncate(time.Millisecond) + case runtimev1.TimeGrain_TIME_GRAIN_SECOND: + return start.Truncate(time.Second) + case runtimev1.TimeGrain_TIME_GRAIN_MINUTE: + return start.Truncate(time.Minute) + case runtimev1.TimeGrain_TIME_GRAIN_HOUR: + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), start.Day(), start.Hour(), 0, 0, 0, tz) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_DAY: + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_WEEK: + start = start.In(tz) + weekday := int(start.Weekday()) + if weekday == 0 { + weekday = 7 + } + if firstDay < 1 { + firstDay = 1 + } + if firstDay > 7 { + firstDay = 7 + } + + daysToSubtract := -(weekday - firstDay) + if weekday < firstDay { + daysToSubtract = -7 + daysToSubtract + } + start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz) + start = start.AddDate(0, 0, daysToSubtract) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_MONTH: + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) + start = start.In(time.UTC) + return start + case runtimev1.TimeGrain_TIME_GRAIN_QUARTER: + monthsToSubtract := (3 + int(start.Month()) - firstMonth%3) % 3 + start = start.In(tz) + start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz) + start = start.AddDate(0, -monthsToSubtract, 0) + return start.In(time.UTC) + case runtimev1.TimeGrain_TIME_GRAIN_YEAR: + start = start.In(tz) + year := start.Year() + if int(start.Month()) < firstMonth { + year = start.Year() - 1 + } + + start = time.Date(year, time.Month(firstMonth), 1, 0, 0, 0, 0, tz) + return start.In(time.UTC) + } + + return start +} + +func ResolveTimeRange(tr *runtimev1.TimeRange, mv *runtimev1.MetricsViewSpec) (time.Time, time.Time, error) { + tz := time.UTC + + if tr.TimeZone != "" { + var err error + tz, err = time.LoadLocation(tr.TimeZone) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid time_range.time_zone %q: %w", tr.TimeZone, err) + } + } + + var start, end time.Time + if tr.Start != nil { + start = tr.Start.AsTime().In(tz) + } + if tr.End != nil { + end = tr.End.AsTime().In(tz) + } + + isISO := false + + if tr.IsoDuration != "" { + if !start.IsZero() && !end.IsZero() { + return time.Time{}, time.Time{}, fmt.Errorf("only two of time_range.{start,end,iso_duration} can be specified") + } + + d, err := duration.ParseISO8601(tr.IsoDuration) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid iso_duration %q: %w", tr.IsoDuration, err) + } + + if !start.IsZero() { + end = d.Add(start) + } else if !end.IsZero() { + start = d.Sub(end) + } else { + return time.Time{}, time.Time{}, fmt.Errorf("one of time_range.{start,end} must be specified with time_range.iso_duration") + } + + isISO = true + } + + if tr.IsoOffset != "" { + d, err := duration.ParseISO8601(tr.IsoOffset) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid iso_offset %q: %w", tr.IsoOffset, err) + } + + if !start.IsZero() { + start = d.Add(start) + } + if !end.IsZero() { + end = d.Add(end) + } + + isISO = true + } + + // Only modify the start and end if ISO duration or offset was sent. + // This is to maintain backwards compatibility for calls from the UI. + if isISO { + fdow := int(mv.FirstDayOfWeek) + if mv.FirstDayOfWeek > 7 || mv.FirstDayOfWeek <= 0 { + fdow = 1 + } + fmoy := int(mv.FirstMonthOfYear) + if mv.FirstMonthOfYear > 12 || mv.FirstMonthOfYear <= 0 { + fmoy = 1 + } + if !start.IsZero() { + start = TruncateTime(start, tr.RoundToGrain, tz, fdow, fmoy) + } + if !end.IsZero() { + end = TruncateTime(end, tr.RoundToGrain, tz, fdow, fmoy) + } + } + + return start, end, nil +} diff --git a/runtime/queries/timeutil_test.go b/runtime/queries/timeutil_test.go new file mode 100644 index 00000000000..2b4dc407382 --- /dev/null +++ b/runtime/queries/timeutil_test.go @@ -0,0 +1,134 @@ +package queries + +import ( + "testing" + "time" + + runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestTruncateTime(t *testing.T) { + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-04-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, time.UTC, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, time.UTC, 1, 1)) +} + +func TestTruncateTime_Kathmandu(t *testing.T) { + tz, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-07T04:15:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-06T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2023-10-08T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-01-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2019-03-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 1, 1)) + require.Equal(t, parseTestTime(t, "2018-12-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 1, 1)) +} + +func TestTruncateTime_UTC_first_day(t *testing.T) { + tz := time.UTC + require.Equal(t, parseTestTime(t, "2023-10-08T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) + require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T00:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) +} + +func TestTruncateTime_Kathmandu_first_day(t *testing.T) { + tz, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.Equal(t, parseTestTime(t, "2023-10-07T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) + require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-09T18:16:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1)) +} + +func TestTruncateTime_UTC_first_month(t *testing.T) { + tz := time.UTC + require.Equal(t, parseTestTime(t, "2023-08-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-11-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 5)) + require.Equal(t, parseTestTime(t, "2023-09-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-09-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + + require.Equal(t, parseTestTime(t, "2023-02-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-03-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) + require.Equal(t, parseTestTime(t, "2023-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-01-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) +} + +func TestTruncateTime_Kathmandu_first_month(t *testing.T) { + tz, err := time.LoadLocation("Asia/Kathmandu") + require.NoError(t, err) + require.Equal(t, parseTestTime(t, "2023-07-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-10-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 5)) + require.Equal(t, parseTestTime(t, "2023-08-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-08-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6)) + + require.Equal(t, parseTestTime(t, "2023-01-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2)) + require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-03-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3)) + require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12)) + require.Equal(t, parseTestTime(t, "2022-12-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-01-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1)) +} + +func TestResolveTimeRange(t *testing.T) { + cases := []struct { + title string + tr *runtimev1.TimeRange + start, end string + }{ + { + "day light savings start US/Canada", + &runtimev1.TimeRange{End: timeToPB("2023-03-12T12:00:00Z"), IsoDuration: "PT4H", TimeZone: "America/Los_Angeles"}, + "2023-03-12T08:00:00Z", + "2023-03-12T12:00:00Z", + }, + { + "day light savings end US/Canada", + &runtimev1.TimeRange{Start: timeToPB("2023-11-05T08:00:00.000Z"), IsoDuration: "PT4H", TimeZone: "America/Los_Angeles"}, + "2023-11-05T08:00:00Z", + "2023-11-05T12:00:00Z", + }, + { + "going through feb", + &runtimev1.TimeRange{Start: timeToPB("2023-01-05T00:00:00Z"), IsoDuration: "P1M"}, + "2023-01-05T00:00:00Z", + "2023-02-05T00:00:00Z", + }, + } + + for _, tc := range cases { + t.Run(tc.title, func(t *testing.T) { + start, end, err := ResolveTimeRange(tc.tr, &runtimev1.MetricsViewSpec{ + FirstDayOfWeek: 1, + FirstMonthOfYear: 1, + }) + require.NoError(t, err) + require.Equal(t, parseTestTime(t, tc.start), start.UTC()) + require.Equal(t, parseTestTime(t, tc.end), end.UTC()) + }) + } +} + +func timeToPB(t string) *timestamppb.Timestamp { + ts, _ := time.Parse(time.RFC3339, t) + return timestamppb.New(ts) +} + +func parseTestTime(tst *testing.T, t string) time.Time { + ts, err := time.Parse(time.RFC3339, t) + require.NoError(tst, err) + return ts +} diff --git a/runtime/reconcilers/report.go b/runtime/reconcilers/report.go index 9397f5323a1..59f1d9fe689 100644 --- a/runtime/reconcilers/report.go +++ b/runtime/reconcilers/report.go @@ -320,6 +320,7 @@ func buildQuery(rep *runtimev1.Report, t time.Time) (*runtimev1.Query, error) { if err != nil { return nil, fmt.Errorf("invalid properties for query %q: %w", rep.Spec.QueryName, err) } + req.TimeRange = overrideTimeRange(req.TimeRange, t) case "MetricsViewToplist": req := &runtimev1.MetricsViewToplistRequest{} qry.Query = &runtimev1.Query_MetricsViewToplistRequest{MetricsViewToplistRequest: req} @@ -348,6 +349,7 @@ func buildQuery(rep *runtimev1.Report, t time.Time) (*runtimev1.Query, error) { if err != nil { return nil, fmt.Errorf("invalid properties for query %q: %w", rep.Spec.QueryName, err) } + req.TimeRange = overrideTimeRange(req.TimeRange, t) default: return nil, fmt.Errorf("query %q not supported for reports", rep.Spec.QueryName) } @@ -367,3 +369,13 @@ func formatExportFormat(f runtimev1.ExportFormat) string { return f.String() } } + +func overrideTimeRange(tr *runtimev1.TimeRange, t time.Time) *runtimev1.TimeRange { + if tr == nil { + tr = &runtimev1.TimeRange{} + } + + tr.End = timestamppb.New(t) + + return tr +} diff --git a/runtime/server/queries_metrics_timeseries_test.go b/runtime/server/queries_metrics_timeseries_test.go index 0eea4a3897c..26e4c273b77 100644 --- a/runtime/server/queries_metrics_timeseries_test.go +++ b/runtime/server/queries_metrics_timeseries_test.go @@ -745,6 +745,55 @@ func TestServer_Timeseries_1day(t *testing.T) { require.Equal(t, 2, len(results)) } +func TestServer_Timeseries_1day_no_data(t *testing.T) { + t.Parallel() + server, instanceID := getMetricsTestServer(t, "timeseries") + + response, err := server.MetricsViewTimeSeries(testCtx(), &runtimev1.MetricsViewTimeSeriesRequest{ + InstanceId: instanceID, + MetricsViewName: "timeseries", + MeasureNames: []string{"max_clicks"}, + TimeStart: parseTimeToProtoTimeStamps(t, "2018-01-01T00:00:00Z"), + TimeEnd: parseTimeToProtoTimeStamps(t, "2018-01-03T00:00:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_DAY, + }) + + require.NoError(t, err) + results := response.Data + require.Equal(t, 2, len(results)) + require.Equal(t, parseTime(t, "2018-01-01T00:00:00Z"), results[0].Ts.AsTime()) + require.Equal(t, parseTime(t, "2018-01-02T00:00:00Z"), results[1].Ts.AsTime()) +} + +func TestServer_Timeseries_1day_no_data_no_range(t *testing.T) { + t.Parallel() + server, instanceID := getMetricsTestServer(t, "timeseries") + + response, err := server.MetricsViewTimeSeries(testCtx(), &runtimev1.MetricsViewTimeSeriesRequest{ + InstanceId: instanceID, + MetricsViewName: "timeseries", + MeasureNames: []string{"max_clicks"}, + TimeEnd: parseTimeToProtoTimeStamps(t, "2018-01-03T00:00:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_DAY, + }) + + require.NoError(t, err) + results := response.Data + require.Equal(t, 0, len(results)) + + response, err = server.MetricsViewTimeSeries(testCtx(), &runtimev1.MetricsViewTimeSeriesRequest{ + InstanceId: instanceID, + MetricsViewName: "timeseries", + MeasureNames: []string{"max_clicks"}, + TimeStart: parseTimeToProtoTimeStamps(t, "2022-01-01T00:00:00Z"), + TimeGranularity: runtimev1.TimeGrain_TIME_GRAIN_DAY, + }) + + require.NoError(t, err) + results = response.Data + require.Equal(t, 0, len(results)) +} + func TestServer_Timeseries_1day_Count(t *testing.T) { t.Parallel() server, instanceID := getMetricsTestServer(t, "timeseries") diff --git a/runtime/testruntime/testdata/timeseries/dashboards/timeseries_year.yaml b/runtime/testruntime/testdata/timeseries/dashboards/timeseries_year.yaml new file mode 100644 index 00000000000..6ce2e5fd765 --- /dev/null +++ b/runtime/testruntime/testdata/timeseries/dashboards/timeseries_year.yaml @@ -0,0 +1,21 @@ +model: timeseries_year_model +display_name: Year time series +description: + +timeseries: timestamp +smallest_time_grain: + +dimensions: + - name: device + column: device + - name: publisher + column: publisher + - name: country + column: country +measures: + - name: max_clicks + expression: "max(clicks)" + - name: count + expression: "count(*)" + - name: sum_clicks + expression: "sum(clicks)" \ No newline at end of file diff --git a/runtime/testruntime/testdata/timeseries/models/timeseries_year_model.sql b/runtime/testruntime/testdata/timeseries/models/timeseries_year_model.sql new file mode 100644 index 00000000000..96c8ea3fe6a --- /dev/null +++ b/runtime/testruntime/testdata/timeseries/models/timeseries_year_model.sql @@ -0,0 +1,3 @@ +select * from (select generate_series as timestamp from generate_series(TIMESTAMP '2022-01-01 00:00:00', TIMESTAMP '2025-12-01 00:00:00', INTERVAL '1' MONTH)) a +cross join +(SELECT 1.0 AS clicks, 'android' AS device, 'Google' AS publisher, 'Canada' as country) b \ No newline at end of file diff --git a/web-admin/src/components/layout/ContentContainer.svelte b/web-admin/src/components/layout/ContentContainer.svelte index 36abcbd3ad1..4a9e587631c 100644 --- a/web-admin/src/components/layout/ContentContainer.svelte +++ b/web-admin/src/components/layout/ContentContainer.svelte @@ -1,3 +1,3 @@ -
+
diff --git a/web-admin/src/components/table/Table.svelte b/web-admin/src/components/table/Table.svelte index 42a4b4f9146..9bd4e4ebc58 100644 --- a/web-admin/src/components/table/Table.svelte +++ b/web-admin/src/components/table/Table.svelte @@ -63,35 +63,30 @@ $: data && rerender(); -
- - - - {#if $table.getRowModel().rows.length === 0} +
+ + + {#if $table.getRowModel().rows.length === 0} + + + + {:else} + {#each $table.getRowModel().rows as row} - + {#each row.getVisibleCells() as cell, i} + + {/each} - {:else} - {#each $table.getRowModel().rows as row} - - {#each row.getVisibleCells() as cell, i} - - {/each} - - {/each} - {/if} - -
+ +
- - + +
- -
-
+ {/each} + {/if} + + - {#if $page.route.id.endsWith("/-/logs")} - Logs - {/if} diff --git a/web-admin/src/features/navigation/TopNavigationBar.svelte b/web-admin/src/features/navigation/TopNavigationBar.svelte index 9ce975d4e46..243642d831d 100644 --- a/web-admin/src/features/navigation/TopNavigationBar.svelte +++ b/web-admin/src/features/navigation/TopNavigationBar.svelte @@ -8,19 +8,24 @@ import { viewAsUserStore } from "../../features/view-as-user/viewAsUserStore"; import AvatarButton from "../authentication/AvatarButton.svelte"; import SignIn from "../authentication/SignIn.svelte"; - import ShareButton from "../dashboards/share/ShareButton.svelte"; + import ShareDashboardButton from "../dashboards/share/ShareDashboardButton.svelte"; import { isErrorStoreEmpty } from "../errors/error-store"; + import ShareProjectButton from "../projects/ShareProjectButton.svelte"; import Breadcrumbs from "./Breadcrumbs.svelte"; - import { isDashboardPage } from "./nav-utils"; + import { isDashboardPage, isProjectPage } from "./nav-utils"; $: organization = $page.params.organization; + $: project = $page.params.project; + $: onProjectPage = isProjectPage($page); $: onDashboardPage = isDashboardPage($page); const user = createAdminServiceGetCurrentUser();
@@ -46,8 +51,11 @@ {#if $viewAsUserStore} {/if} + {#if onProjectPage} + + {/if} {#if onDashboardPage} - + {/if} {#if $user.isSuccess} {#if $user.data && $user.data.user} diff --git a/web-admin/src/features/navigation/nav-utils.ts b/web-admin/src/features/navigation/nav-utils.ts index fc231319502..eb4a38ba0d3 100644 --- a/web-admin/src/features/navigation/nav-utils.ts +++ b/web-admin/src/features/navigation/nav-utils.ts @@ -1,5 +1,12 @@ import type { Page } from "@sveltejs/kit"; +export function isProjectPage(page: Page): boolean { + return ( + page.route.id === "/[organization]/[project]" || + page.route.id === "/[organization]/[project]/-/reports" || + page.route.id === "/[organization]/[project]/-/logs" + ); +} export function isDashboardPage(page: Page): boolean { return page.route.id === "/[organization]/[project]/[dashboard]"; } diff --git a/web-admin/src/features/projects/ProjectDashboardsListener.svelte b/web-admin/src/features/projects/ProjectDashboardsListener.svelte index 5e77e2d7a48..004954f9107 100644 --- a/web-admin/src/features/projects/ProjectDashboardsListener.svelte +++ b/web-admin/src/features/projects/ProjectDashboardsListener.svelte @@ -1,5 +1,5 @@ -{#if $proj.isSuccess} -
    - {#if !hasReadAccess} -
  • - You don't have permission to view project logs -
    • - {:else if !errors || $errors.length === 0} -
  • - No logs present -
    • +
    + +
    + Logs +
    + + {#if $proj.isSuccess} + {#if !$errors || $errors.length === 0} +
    + +
    No logs
    +
    {:else} - -
  • - This project has - {$errors.length} - {$errors.length === 1 ? "error" : "errors"} -
    • - {#each $errors as error} -
  • - - {error.message} - - {#if error.filePath} - - {error.filePath} +
      + {#each $errors as error} +
    • + + {error.message} - {/if} -
      • - {/each} + {#if error.filePath} + + {error.filePath} + + {/if} + + {/each} +
    {/if} -
-{/if} + {/if} +
diff --git a/web-admin/src/features/projects/ProjectDeploymentStatus.svelte b/web-admin/src/features/projects/ProjectDeploymentStatus.svelte index bbab8c24d96..0007a526785 100644 --- a/web-admin/src/features/projects/ProjectDeploymentStatus.svelte +++ b/web-admin/src/features/projects/ProjectDeploymentStatus.svelte @@ -1,7 +1,5 @@
- Project status
@@ -41,9 +35,7 @@ {/if}
- {#if isProjectDeployed} -
- -
+ {#if !isProjectDeployed} +
This project is not deployed.
{/if}
diff --git a/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte b/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte index d0e9623dc81..a332b18c4ef 100644 --- a/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte +++ b/web-admin/src/features/projects/ProjectDeploymentStatusChip.svelte @@ -6,7 +6,7 @@ import { getDashboardsForProject, useDashboardsStatus, - } from "@rilldata/web-admin/features/projects/dashboards"; + } from "@rilldata/web-admin/features/dashboards/listing/dashboards"; import { invalidateDashboardsQueries } from "@rilldata/web-admin/features/projects/invalidations"; import { useProjectDeploymentStatus } from "@rilldata/web-admin/features/projects/selectors"; import CancelCircle from "@rilldata/web-common/components/icons/CancelCircle.svelte"; @@ -97,17 +97,6 @@ textClass: "text-purple-600", wrapperClass: "bg-purple-50 border-purple-300", }, - // [V1DeploymentStatus.DEPLOYMENT_STATUS_RECONCILING]: { - // icon: Spinner, - // iconProps: { - // bg: "linear-gradient(90deg, #22D3EE -0.5%, #6366F1 98.5%)", - // className: "text-purple-600 hover:text-purple-500", - // status: EntityStatus.Running, - // }, - // text: "syncing", - // textClass: "text-purple-600", - // wrapperClass: "bg-purple-50 border-purple-300", - // }, [V1DeploymentStatus.DEPLOYMENT_STATUS_ERROR]: { icon: CancelCircle, iconProps: { className: "text-red-600 hover:text-red-500" }, @@ -142,14 +131,16 @@ } -{#if deploymentStatus} +{#if $deploymentStatusFromDashboards.isFetching && !$deploymentStatusFromDashboards?.data} +
+ +
+{:else if deploymentStatus} {#if iconOnly} -
- -
+ {:else}
- Github {#if isGithubConnected} diff --git a/web-admin/src/features/projects/ProjectHero.svelte b/web-admin/src/features/projects/ProjectHero.svelte deleted file mode 100644 index dc240e785eb..00000000000 --- a/web-admin/src/features/projects/ProjectHero.svelte +++ /dev/null @@ -1,26 +0,0 @@ - - -
-

- {project} -

- - -
- - - -
- - -
diff --git a/web-admin/src/features/projects/ProjectTabs.svelte b/web-admin/src/features/projects/ProjectTabs.svelte new file mode 100644 index 00000000000..931297b00fc --- /dev/null +++ b/web-admin/src/features/projects/ProjectTabs.svelte @@ -0,0 +1,91 @@ + + +{#if tabs} +
+ + + {#each tabs as tab} + + {tab.label} + {#if tab.label === "Logs"} + + {/if} + + {/each} + + +
+{/if} diff --git a/web-admin/src/features/projects/ShareProjectButton.svelte b/web-admin/src/features/projects/ShareProjectButton.svelte new file mode 100644 index 00000000000..fd77a19329b --- /dev/null +++ b/web-admin/src/features/projects/ShareProjectButton.svelte @@ -0,0 +1,51 @@ + + + + + (open = false)} + {open} +> + + Invite a teammate to your project + + +
+ + +
Run this command in the Rill CLI:
+ + + +
+ Ask your organization's admin to invite viewers using the Rill CLI. +
+ + +
+ + +
+
+ +
+ +
 diff --git a/web-admin/src/features/projects/ShareProjectCTA.svelte b/web-admin/src/features/projects/ShareProjectCTA.svelte deleted file mode 100644 index f734ddbb0be..00000000000 --- a/web-admin/src/features/projects/ShareProjectCTA.svelte +++ /dev/null @@ -1,30 +0,0 @@ - - -
- Share - - -
- Run this command in the Rill CLI to invite a teammate to view this - project. -
- - - -
- Ask your organization’s admin to invite viewers using the Rill CLI. -
- - -
diff --git a/web-admin/src/features/projects/dashboards.ts b/web-admin/src/features/projects/dashboards.ts deleted file mode 100644 index 48a2c31f57b..00000000000 --- a/web-admin/src/features/projects/dashboards.ts +++ /dev/null @@ -1,203 +0,0 @@ -import type { V1GetProjectResponse } from "@rilldata/web-admin/client"; -import { - createAdminServiceGetProject, - V1DeploymentStatus, -} from "@rilldata/web-admin/client"; -import { - PollTimeDuringError, - PollTimeDuringReconcile, - PollTimeWhenProjectReady, -} from "@rilldata/web-admin/features/projects/selectors"; -import { refreshResource } from "@rilldata/web-common/features/entity-management/resource-invalidations"; -import { - ResourceKind, - useFilteredResources, -} from "@rilldata/web-common/features/entity-management/resource-selectors"; -import type { V1Resource } from "@rilldata/web-common/runtime-client"; -import { - V1ReconcileStatus, - createRuntimeServiceListResources, -} from "@rilldata/web-common/runtime-client"; -import { invalidateMetricsViewData } from "@rilldata/web-common/runtime-client/invalidation"; -import type { QueryClient } from "@tanstack/svelte-query"; -import Axios from "axios"; -import { derived } from "svelte/store"; - -export interface DashboardListItem { - name: string; - title?: string; - description?: string; - isValid: boolean; -} - -// TODO: use the creator pattern to get rid of the raw call to http endpoint -export async function getDashboardsForProject( - projectData: V1GetProjectResponse -): Promise { - // There may not be a prodDeployment if the project was hibernated - if (!projectData.prodDeployment) { - return []; - } - - // Hack: in development, the runtime host is actually on port 8081 - const runtimeHost = projectData.prodDeployment.runtimeHost.replace( - "localhost:9091", - "localhost:8081" - ); - - const axios = Axios.create({ - baseURL: runtimeHost, - headers: { - Authorization: `Bearer ${projectData.jwt}`, - }, - }); - - // TODO: use resource API - const catalogEntriesResponse = await axios.get( - `/v1/instances/${projectData.prodDeployment.runtimeInstanceId}/resources?kind=${ResourceKind.MetricsView}` - ); - - const catalogEntries = catalogEntriesResponse.data?.resources as V1Resource[]; - - return catalogEntries.filter((e) => !!e.metricsView); -} - -export function useDashboards(instanceId: string) { - return useFilteredResources(instanceId, ResourceKind.MetricsView, (data) => - data.resources.filter((res) => !!res.metricsView?.state?.validSpec) - ); -} - -export function useDashboardsLastUpdated( - instanceId: string, - organization: string, - project: string -) { - return derived( - [ - useDashboards(instanceId), - createAdminServiceGetProject(organization, project), - ], - ([dashboardsResp, projResp]) => { - if (!dashboardsResp.data?.length) { - if (!projResp.data?.prodDeployment?.updatedOn) return undefined; - - // return project's last updated if there are no dashboards - return new Date(projResp.data.prodDeployment.updatedOn); - } - - const max = Math.max( - ...dashboardsResp.data.map((res) => - new Date(res.meta.stateUpdatedOn).getTime() - ) - ); - return new Date(max); - } - ); -} - -export function useDashboardsStatus(instanceId: string) { - return createRuntimeServiceListResources( - instanceId, - { - kind: ResourceKind.MetricsView, - }, - { - query: { - select: (data): V1DeploymentStatus => { - let isPending = false; - let isError = false; - for (const resource of data.resources) { - if ( - resource.meta.reconcileStatus !== - V1ReconcileStatus.RECONCILE_STATUS_IDLE - ) { - isPending = true; - continue; - } - - if ( - resource.meta.reconcileError || - !resource.metricsView?.state?.validSpec - ) { - isError = true; - } - } - - if (isPending) return V1DeploymentStatus.DEPLOYMENT_STATUS_PENDING; - if (isError) return V1DeploymentStatus.DEPLOYMENT_STATUS_ERROR; - return V1DeploymentStatus.DEPLOYMENT_STATUS_OK; - }, - - refetchInterval: (data) => { - switch (data) { - case V1DeploymentStatus.DEPLOYMENT_STATUS_PENDING: - return PollTimeDuringReconcile; - - case V1DeploymentStatus.DEPLOYMENT_STATUS_ERROR: - case V1DeploymentStatus.DEPLOYMENT_STATUS_UNSPECIFIED: - return PollTimeDuringError; - - case V1DeploymentStatus.DEPLOYMENT_STATUS_OK: - return PollTimeWhenProjectReady; - - default: - return PollTimeWhenProjectReady; - } - }, - }, - } - ); -} - -export function listenAndInvalidateDashboards( - queryClient: QueryClient, - instanceId: string -) { - const store = derived( - [useDashboardsStatus(instanceId), useDashboards(instanceId)], - (state) => state - ); - - const dashboards = new Map(); - - return store.subscribe(([status, dashboardsResp]) => { - if ( - // Let through error and ok states - status.data === V1DeploymentStatus.DEPLOYMENT_STATUS_PENDING || - status.data === V1DeploymentStatus.DEPLOYMENT_STATUS_UNSPECIFIED || - !dashboardsResp.data - ) - return; - - const existingDashboards = new Set(); - for (const [name] of dashboards) { - existingDashboards.add(name); - } - - for (const dashboardResource of dashboardsResp.data) { - const stateUpdatedOn = new Date(dashboardResource.meta.stateUpdatedOn); - - if (dashboards.has(dashboardResource.meta.name.name)) { - // if the dashboard existed then check if it was updated since last seen - const prevStateUpdatedOn = dashboards.get( - dashboardResource.meta.name.name - ); - if (prevStateUpdatedOn.getTime() < stateUpdatedOn.getTime()) { - // invalidate if it was updated - refreshResource(queryClient, instanceId, dashboardResource).then(() => - invalidateMetricsViewData(queryClient, instanceId, false) - ); - } - } - - existingDashboards.delete(dashboardResource.meta.name.name); - dashboards.set(dashboardResource.meta.name.name, stateUpdatedOn); - } - - // cleanup of older dashboards - for (const oldName of existingDashboards) { - dashboards.delete(oldName); - } - }); -} diff --git a/web-admin/src/routes/[organization]/[project]/+layout.svelte b/web-admin/src/routes/[organization]/[project]/+layout.svelte index f2b90f42411..2e5853c5d62 100644 --- a/web-admin/src/routes/[organization]/[project]/+layout.svelte +++ b/web-admin/src/routes/[organization]/[project]/+layout.svelte @@ -3,6 +3,8 @@ import { page } from "$app/stores"; import ProjectDashboardsListener from "@rilldata/web-admin/features/projects/ProjectDashboardsListener.svelte"; import RuntimeProvider from "@rilldata/web-common/runtime-client/RuntimeProvider.svelte"; + import { isProjectPage } from "../../../features/navigation/nav-utils"; + import ProjectTabs from "../../../features/projects/ProjectTabs.svelte"; import { useProjectRuntime } from "../../../features/projects/selectors"; import { viewAsUserStore } from "../../../features/view-as-user/viewAsUserStore"; @@ -17,17 +19,24 @@ // Redirect any nested routes (notably dashboards) to the project page goto(`/${$page.params.organization}/${$page.params.project}`); } + + $: onProjectPage = isProjectPage($page); -{#if $projRuntime.data && !$viewAsUserStore} +{#if !$viewAsUserStore} + + {#if onProjectPage} + + {/if} diff --git a/web-admin/src/routes/[organization]/[project]/+page.svelte b/web-admin/src/routes/[organization]/[project]/+page.svelte index b7653ee962a..af36feba9d4 100644 --- a/web-admin/src/routes/[organization]/[project]/+page.svelte +++ b/web-admin/src/routes/[organization]/[project]/+page.svelte @@ -4,7 +4,6 @@ import VerticalScrollContainer from "@rilldata/web-common/layout/VerticalScrollContainer.svelte"; import { createAdminServiceGetProject } from "../../../client"; import DashboardsTable from "../../../features/dashboards/listing/DashboardsTable.svelte"; - import ProjectHero from "../../../features/projects/ProjectHero.svelte"; import RedeployProjectCta from "../../../features/projects/RedeployProjectCTA.svelte"; $: organization = $page.params.organization; @@ -20,8 +19,7 @@ -
- +
{#if isProjectDeployed} {:else if isProjectHibernating} diff --git a/web-admin/src/routes/[organization]/[project]/-/logs/+page.svelte b/web-admin/src/routes/[organization]/[project]/-/logs/+page.svelte index d16df853167..90031e1361d 100644 --- a/web-admin/src/routes/[organization]/[project]/-/logs/+page.svelte +++ b/web-admin/src/routes/[organization]/[project]/-/logs/+page.svelte @@ -1,6 +1,8 @@ - +
+
+ + +
+ +
diff --git a/web-admin/src/routes/[organization]/[project]/-/redirect/+page.svelte b/web-admin/src/routes/[organization]/[project]/-/redirect/+page.svelte deleted file mode 100644 index 70f3ed9241c..00000000000 --- a/web-admin/src/routes/[organization]/[project]/-/redirect/+page.svelte +++ /dev/null @@ -1,40 +0,0 @@ - - diff --git a/web-admin/src/routes/[organization]/[project]/[dashboard]/+page.svelte b/web-admin/src/routes/[organization]/[project]/[dashboard]/+page.svelte index d6f084ced95..5ef29c0d7c6 100644 --- a/web-admin/src/routes/[organization]/[project]/[dashboard]/+page.svelte +++ b/web-admin/src/routes/[organization]/[project]/[dashboard]/+page.svelte @@ -4,7 +4,7 @@ createAdminServiceGetProject, V1DeploymentStatus, } from "@rilldata/web-admin/client"; - import { getDashboardsForProject } from "@rilldata/web-admin/features/projects/dashboards"; + import { getDashboardsForProject } from "@rilldata/web-admin/features/dashboards/listing/dashboards"; import { invalidateDashboardsQueries } from "@rilldata/web-admin/features/projects/invalidations"; import ProjectErrored from "@rilldata/web-admin/features/projects/ProjectErrored.svelte"; import { useProjectDeploymentStatus } from "@rilldata/web-admin/features/projects/selectors"; diff --git a/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte b/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte index 130daae5ed6..8f5c025fe21 100644 --- a/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte +++ b/web-common/src/components/data-graphic/marks/MultiMetricMouseoverLabel.svelte @@ -11,7 +11,7 @@ It is probably not the most up to date code; but it works very well in practice. import { preventVerticalOverlap } from "./prevent-vertical-overlap"; import DelayedLabel from "@rilldata/web-common/components/data-graphic/marks/DelayedLabel.svelte"; - const DIMENSION_HOVER_DURATION = 250; + const DIMENSION_HOVER_DURATION = 350; interface Point { x: number; y: number; diff --git a/web-common/src/components/data-graphic/marks/sparkline.ts b/web-common/src/components/data-graphic/marks/sparkline.ts index fe07084c541..30948f10ab3 100644 --- a/web-common/src/components/data-graphic/marks/sparkline.ts +++ b/web-common/src/components/data-graphic/marks/sparkline.ts @@ -5,16 +5,17 @@ export function createSparkline( dataArr: Array, accessor: (v: unknown) => number ) { + const noDataSpark = ` + + + +`; // Check if dataArr is present and has data if (!dataArr || dataArr.length === 0) { // Return SVG with a flat line in the middle of svgHeight - return ` - - - - `; + return noDataSpark; } const data = accessor ? dataArr?.map(accessor) : (dataArr as number[]); const maxY = Math.max(...data); @@ -24,6 +25,10 @@ export function createSparkline( (y) => svgHeight - ((y - minY) / (maxY - minY)) * svgHeight ); + // Normalized data may have NaNs when data is only nulls and 0s + const hasNaN = normalizedData.every((y) => isNaN(y)); + if (hasNaN) return noDataSpark; + let d = ""; normalizedData.forEach((y, i) => { const x = (i / (data.length - 1)) * svgWidth; diff --git a/web-common/src/components/icons/CheckCircleNew.svelte b/web-common/src/components/icons/CheckCircleNew.svelte new file mode 100644 index 00000000000..ec89efcf4a5 --- /dev/null +++ b/web-common/src/components/icons/CheckCircleNew.svelte @@ -0,0 +1,30 @@ + + + + + + diff --git a/web-common/src/components/icons/Logs.svelte b/web-common/src/components/icons/Logs.svelte new file mode 100644 index 00000000000..602f1ee833c --- /dev/null +++ b/web-common/src/components/icons/Logs.svelte @@ -0,0 +1,46 @@ + + + + + + + + + + + + diff --git a/web-common/src/features/dashboards/dashboard-utils.ts b/web-common/src/features/dashboards/dashboard-utils.ts index abd4035c4b9..34854ca64a9 100644 --- a/web-common/src/features/dashboards/dashboard-utils.ts +++ b/web-common/src/features/dashboards/dashboard-utils.ts @@ -31,7 +31,8 @@ export function prepareSortedQueryBody( dimensionName: string, measureNames: string[], timeControls: TimeControlState, - sortMeasureName: string, + // Note: sortMeasureName may be null if we are sorting by dimension values + sortMeasureName: string | null, sortType: SortType, sortAscending: boolean, filterForDimension: V1MetricsViewFilter @@ -45,7 +46,7 @@ export function prepareSortedQueryBody( // Benjamin and Egor put in a patch that will allow us to use the // dimension name as the measure name. This will need to be updated // once they have stabilized the API. - if (sortType === SortType.DIMENSION) { + if (sortType === SortType.DIMENSION || sortMeasureName === null) { sortMeasureName = dimensionName; // note also that we need to remove the comparison time range // when sorting by dimension values, or the query errors diff --git a/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte b/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte index 561f03808d1..26b28ccf418 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionDisplay.svelte @@ -6,81 +6,51 @@ * to be displayed in explore */ import { cancelDashboardQueries } from "@rilldata/web-common/features/dashboards/dashboard-queries"; - import { - useMetaDimension, - useMetaMeasure, - useMetaQuery, - } from "@rilldata/web-common/features/dashboards/selectors"; + import { getStateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; import { useTimeControlStore } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; import { createQueryServiceMetricsViewComparison, createQueryServiceMetricsViewTotals, - MetricsViewDimension, - MetricsViewSpecMeasureV2, } from "@rilldata/web-common/runtime-client"; import { useQueryClient } from "@tanstack/svelte-query"; - import { runtime } from "../../../runtime-client/runtime-store"; - import { - getDimensionFilterWithSearch, - prepareDimensionTableRows, - prepareVirtualizedDimTableColumns, - } from "./dimension-table-utils"; + import { getDimensionFilterWithSearch } from "./dimension-table-utils"; import DimensionHeader from "./DimensionHeader.svelte"; import DimensionTable from "./DimensionTable.svelte"; - import { - getDimensionColumn, - isSummableMeasure, - prepareSortedQueryBody, - } from "../dashboard-utils"; import { metricsExplorerStore } from "../stores/dashboard-stores"; - export let metricViewName: string; - export let dimensionName: string; - - let searchText = ""; - - const queryClient = useQueryClient(); - - $: instanceId = $runtime.instanceId; - - $: metaQuery = useMetaQuery(instanceId, metricViewName); - - $: dimensionQuery = useMetaDimension( - instanceId, - metricViewName, - dimensionName - ); - - let dimension: MetricsViewDimension; - $: dimension = $dimensionQuery?.data as MetricsViewDimension; - $: dimensionColumn = getDimensionColumn(dimension); const stateManagers = getStateManagers(); - const timeControlsStore = useTimeControlStore(stateManagers); - const { dashboardStore, selectors: { - sorting: { sortedAscending }, + dashboardQueries: { + dimensionTableSortedQueryBody, + dimensionTableTotalQueryBody, + }, + comparison: { isBeingCompared }, + dimensions: { dimensionTableDimName }, + dimensionTable: { + virtualizedTableColumns, + selectedDimensionValueNames, + prepareDimTableRows, + }, + activeMeasure: { activeMeasureName }, }, + metricsViewName, + runtime, } = stateManagers; - $: leaderboardMeasureName = $dashboardStore?.leaderboardMeasureName; - $: isBeingCompared = - $dashboardStore?.selectedComparisonDimension === dimensionName; + // cast is safe because dimensionTableDimName must be defined + // for the dimension table to be open + $: dimensionName = $dimensionTableDimName as string; - $: leaderboardMeasureQuery = useMetaMeasure( - instanceId, - metricViewName, - leaderboardMeasureName - ); + let searchText = ""; + + const queryClient = useQueryClient(); - $: validPercentOfTotal = ( - $leaderboardMeasureQuery?.data as MetricsViewSpecMeasureV2 - )?.validPercentOfTotal; + $: instanceId = $runtime.instanceId; - $: excludeMode = - $dashboardStore?.dimensionFilterExcludeMode.get(dimensionName) ?? false; + const timeControlsStore = useTimeControlStore(stateManagers); $: filterSet = getDimensionFilterWithSearch( $dashboardStore?.filters, @@ -88,26 +58,10 @@ dimensionName ); - $: selectedValues = - (excludeMode - ? $dashboardStore?.filters?.exclude?.find((d) => d.name === dimensionName) - ?.in - : $dashboardStore?.filters?.include?.find((d) => d.name === dimensionName) - ?.in) ?? []; - - $: visibleMeasures = - $metaQuery.data?.measures?.filter((m) => - $dashboardStore?.visibleMeasureKeys.has(m.name ?? "") - ) ?? []; - $: totalsQuery = createQueryServiceMetricsViewTotals( instanceId, - metricViewName, - { - measureNames: $dashboardStore?.selectedMeasureNames, - timeStart: $timeControlsStore.timeStart, - timeEnd: $timeControlsStore.timeEnd, - }, + $metricsViewName, + $dimensionTableTotalQueryBody, { query: { enabled: $timeControlsStore.ready, @@ -115,40 +69,14 @@ } ); - $: unfilteredTotal = $totalsQuery?.data?.data?.[leaderboardMeasureName] ?? 0; - - let referenceValues: { [key: string]: number } = {}; - $: if ($totalsQuery?.data?.data) { - visibleMeasures.map((m) => { - if (m.name && isSummableMeasure(m)) { - referenceValues[m.name] = $totalsQuery.data?.data?.[m.name]; - } - }); - } - - $: columns = prepareVirtualizedDimTableColumns( - $dashboardStore, - visibleMeasures, - referenceValues, - dimension, - $timeControlsStore?.showComparison ?? false, - validPercentOfTotal ?? false - ); + $: unfilteredTotal = $totalsQuery?.data?.data?.[$activeMeasureName] ?? 0; - $: sortedQueryBody = prepareSortedQueryBody( - dimensionName, - $dashboardStore?.selectedMeasureNames, - $timeControlsStore, - leaderboardMeasureName, - $dashboardStore.dashboardSortType, - $sortedAscending, - filterSet - ); + $: columns = $virtualizedTableColumns($totalsQuery); $: sortedQuery = createQueryServiceMetricsViewComparison( $runtime.instanceId, - metricViewName, - sortedQueryBody, + $metricsViewName, + $dimensionTableSortedQueryBody, { query: { enabled: $timeControlsStore.ready && !!filterSet, @@ -156,57 +84,72 @@ } ); - $: tableRows = prepareDimensionTableRows( - $sortedQuery?.data?.rows ?? [], - $metaQuery.data?.measures ?? [], - leaderboardMeasureName, - dimensionColumn, - $timeControlsStore.showComparison ?? false, - validPercentOfTotal ?? false, - unfilteredTotal + $: tableRows = $prepareDimTableRows($sortedQuery, unfilteredTotal); + + $: areAllTableRowsSelected = tableRows.every((row) => + $selectedDimensionValueNames.includes(row[dimensionName] as string) ); function onSelectItem(event) { - const label = tableRows[event.detail][dimensionColumn] as string; - cancelDashboardQueries(queryClient, metricViewName); - metricsExplorerStore.toggleFilter(metricViewName, dimensionName, label); + const label = tableRows[event.detail][dimensionName] as string; + cancelDashboardQueries(queryClient, $metricsViewName); + metricsExplorerStore.toggleFilter($metricsViewName, dimensionName, label); } function toggleComparisonDimension(dimensionName, isBeingCompared) { metricsExplorerStore.setComparisonDimension( - metricViewName, + $metricsViewName, isBeingCompared ? undefined : dimensionName ); } + + function toggleAllSearchItems() { + const labels = tableRows.map((row) => row[dimensionName] as string); + cancelDashboardQueries(queryClient, $metricsViewName); + + if (areAllTableRowsSelected) { + metricsExplorerStore.deselectItemsInFilter( + $metricsViewName, + dimensionName, + labels + ); + return; + } else { + metricsExplorerStore.selectItemsInFilter( + $metricsViewName, + dimensionName, + labels + ); + } + } {#if sortedQuery}
{ searchText = event.detail; }} + on:toggle-all-search-items={() => toggleAllSearchItems()} />
- {#if tableRows && columns.length && dimensionColumn} + {#if tableRows && columns.length && dimensionName}
onSelectItem(event)} on:toggle-dimension-comparison={() => toggleComparisonDimension(dimensionName, isBeingCompared)} isFetching={$sortedQuery?.isFetching} - dimensionName={dimensionColumn} - {isBeingCompared} + {dimensionName} {columns} - {selectedValues} + selectedValues={$selectedDimensionValueNames} rows={tableRows} - {excludeMode} />
{/if} diff --git a/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte b/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte index 03a98d2988f..95041978c91 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionFilterGutter.svelte @@ -10,6 +10,7 @@ import { getContext } from "svelte"; import type { VirtualizedTableConfig } from "../../../components/virtualized-table/types"; import DimensionCompareMenu from "@rilldata/web-common/features/dashboards/leaderboard/DimensionCompareMenu.svelte"; + import { getStateManagers } from "../state-managers/state-managers"; export let totalHeight: number; export let virtualRowItems; @@ -18,6 +19,12 @@ export let isBeingCompared = false; export let atLeastOneActive = false; + const { + selectors: { + dimensions: { dimensionTableDimName }, + }, + } = getStateManagers(); + function getInsertIndex(arr, num) { return arr .concat(num) @@ -63,7 +70,7 @@ style:height="{config.columnHeaderHeight}px" class="sticky left-0 top-0 surface z-40 flex items-center" > - +
{#each virtualRowItems as row (`row-${row.key}`)} {@const isSelected = selectedIndex.includes(row.index)} diff --git a/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte b/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte index e588be5ef62..77d9c6ef3f1 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionHeader.svelte @@ -13,58 +13,73 @@ import { EntityStatus } from "@rilldata/web-common/features/entity-management/types"; import { slideRight } from "@rilldata/web-common/lib/transitions"; import { useQueryClient } from "@tanstack/svelte-query"; - import { createEventDispatcher } from "svelte"; import { fly } from "svelte/transition"; import Spinner from "../../entity-management/Spinner.svelte"; import { metricsExplorerStore } from "web-common/src/features/dashboards/stores/dashboard-stores"; import ExportDimensionTableDataButton from "./ExportDimensionTableDataButton.svelte"; import { getStateManagers } from "../state-managers/state-managers"; import { SortType } from "../proto-state/derived-types"; + import Button from "@rilldata/web-common/components/button/Button.svelte"; + import { createEventDispatcher } from "svelte"; - export let metricViewName: string; export let dimensionName: string; export let isFetching: boolean; - export let excludeMode = false; + export let areAllTableRowsSelected = false; + export let isRowsEmpty = true; + + const dispatch = createEventDispatcher(); const stateManagers = getStateManagers(); const { selectors: { sorting: { sortedByDimensionValue }, + dimensionTable: { dimensionTableSearchString }, + dimensionFilters: { isFilterExcludeMode }, }, actions: { sorting: { toggleSort }, + dimensionTable: { + setDimensionTableSearchString, + clearDimensionTableSearchString, + }, + dimensions: { setPrimaryDimension }, }, + metricsViewName, } = stateManagers; + $: excludeMode = $isFilterExcludeMode(dimensionName); + const queryClient = useQueryClient(); $: filterKey = excludeMode ? "exclude" : "include"; $: otherFilterKey = excludeMode ? "include" : "exclude"; - let searchToggle = false; - - const dispatch = createEventDispatcher(); + let searchBarOpen = false; - let searchText = ""; + // FIXME: this extra `searchText` variable should be eliminated, + // but there is no way to make the component a fully + // "controlled" component for now, so we have to go through the + // `value` binding it exposes. + let searchText: string | undefined = undefined; + $: searchText = $dimensionTableSearchString; function onSearch() { - dispatch("search", searchText); + setDimensionTableSearchString(searchText); } function closeSearchBar() { - searchText = ""; - searchToggle = !searchToggle; - onSearch(); + clearDimensionTableSearchString(); + searchBarOpen = false; } const goBackToLeaderboard = () => { - metricsExplorerStore.setMetricDimensionName(metricViewName, null); if ($sortedByDimensionValue) { toggleSort(SortType.VALUE); } + setPrimaryDimension(undefined); }; function toggleFilterMode() { - cancelDashboardQueries(queryClient, metricViewName); - metricsExplorerStore.toggleFilterMode(metricViewName, dimensionName); + cancelDashboardQueries(queryClient, $metricsViewName); + metricsExplorerStore.toggleFilterMode($metricsViewName, dimensionName); } @@ -84,6 +99,36 @@
+ {#if searchText && !isRowsEmpty} + + {/if} + {#if searchBarOpen || (searchText && searchText !== "")} +
+ + +
+ {:else} + + {/if} +
toggleFilterMode()}> @@ -103,27 +148,6 @@ - {#if !searchToggle} - - {:else} -
- - -
- {/if} - - +
diff --git a/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte b/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte index 64844b50d8d..380a901cc1d 100644 --- a/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte +++ b/web-common/src/features/dashboards/dimension-table/DimensionTable.svelte @@ -24,20 +24,23 @@ TableCells – the cell contents. export let rows: DimensionTableRow[]; export let columns: VirtualizedTableColumns[]; - export let selectedValues: Array = []; + export let selectedValues: string[]; export let dimensionName: string; - export let excludeMode = false; - export let isBeingCompared = false; export let isFetching: boolean; const { - actions: { dimTable }, + actions: { dimensionTable }, selectors: { sorting: { sortMeasure }, + dimensionFilters: { isFilterExcludeMode }, + comparison: { isBeingCompared: isBeingComparedReadable }, }, } = getStateManagers(); + $: excludeMode = $isFilterExcludeMode(dimensionName); + $: isBeingCompared = $isBeingComparedReadable(dimensionName); + /** the overscan values tell us how much to render off-screen. These may be set by the consumer * in certain circumstances. The tradeoff: the higher the overscan amount, the more DOM elements we have * to render on initial load. @@ -58,7 +61,7 @@ TableCells – the cell contents. const CHARACTER_LIMIT_FOR_WRAPPING = 9; const FILTER_COLUMN_WIDTH = config.indexWidth; - $: selectedIndices = selectedValues + $: selectedIndex = selectedValues .map((label) => { return rows.findIndex((row) => row[dimensionName] === label); }) @@ -87,6 +90,8 @@ TableCells – the cell contents. let estimateColumnSize: number[] = []; /* Separate out dimension column */ + // SAFETY: cast should be safe because if dimensionName is undefined, + // we should not be in a dimension table sub-component $: dimensionColumn = columns?.find( (c) => c.name == dimensionName ) as VirtualizedTableColumns; @@ -181,7 +186,7 @@ TableCells – the cell contents. async function handleColumnHeaderClick(event) { colScrollOffset = $columnVirtualizer.scrollOffset; const columnName = event.detail; - dimTable.handleMeasureColumnHeaderClick(columnName); + dimensionTable.handleMeasureColumnHeaderClick(columnName); } async function handleResizeDimensionColumn(event) { @@ -238,7 +243,7 @@ TableCells – the cell contents. 0} @@ -253,7 +258,7 @@ TableCells – the cell contents. column={dimensionColumn} {rows} {activeIndex} - selectedIndex={selectedIndices} + {selectedIndex} {excludeMode} {scrolling} {horizontalScrolling} @@ -270,7 +275,7 @@ TableCells – the cell contents. columns={measureColumns} {rows} {activeIndex} - selectedIndex={selectedIndices} + {selectedIndex} {scrolling} {excludeMode} on:select-item={(event) => onSelectItem(event)} diff --git a/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts b/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts index 8c05849606f..56c52c61932 100644 --- a/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts +++ b/web-common/src/features/dashboards/dimension-table/dimension-table-utils.ts @@ -7,6 +7,7 @@ import type { MetricsViewDimension, MetricsViewSpecMeasureV2, V1MetricsViewComparisonRow, + V1MetricsViewComparisonValue, V1MetricsViewFilter, V1MetricsViewToplistResponseDataItem, } from "../../../runtime-client"; @@ -17,7 +18,7 @@ import type { VirtualizedTableConfig } from "@rilldata/web-common/components/vir import type { SvelteComponent } from "svelte"; import { getDimensionColumn } from "../dashboard-utils"; import type { DimensionTableRow } from "./dimension-table-types"; -import { getFilterForDimension } from "../selectors"; +import { getFiltersForOtherDimensions } from "../selectors"; import { SortType } from "../proto-state/derived-types"; import type { MetricsExplorerEntity } from "../stores/metrics-explorer-entity"; import { createMeasureValueFormatter } from "@rilldata/web-common/lib/number-formatting/format-measure-value"; @@ -26,10 +27,10 @@ import { formatMeasurePercentageDifference } from "@rilldata/web-common/lib/numb /** Returns an updated filter set for a given dimension on search */ export function updateFilterOnSearch( - filterForDimension, - searchText, - dimensionName -) { + filterForDimension: V1MetricsViewFilter, + searchText: string, + dimensionName: string +): V1MetricsViewFilter { const filterSet = JSON.parse(JSON.stringify(filterForDimension)); const addNull = "null".includes(searchText); if (searchText !== "") { @@ -64,7 +65,10 @@ export function getDimensionFilterWithSearch( searchText: string, dimensionName: string ) { - const filterForDimension = getFilterForDimension(filters, dimensionName); + const filterForDimension = getFiltersForOtherDimensions( + filters, + dimensionName + ); return updateFilterOnSearch(filterForDimension, searchText, dimensionName); } @@ -231,13 +235,14 @@ export function prepareVirtualizedDimTableColumns( const selectedMeasure = allMeasures.find( (m) => m.name === leaderboardMeasureName ); + const dimensionColumn = getDimensionColumn(dimension); // copy column names so we don't mutate the original const columnNames = [...dash.visibleMeasureKeys]; // don't add context columns if sorting by dimension - if (sortType !== SortType.DIMENSION) { + if (selectedMeasure && sortType !== SortType.DIMENSION) { addContextColumnNames( columnNames, timeComparison, @@ -248,20 +253,20 @@ export function prepareVirtualizedDimTableColumns( // Make dimension the first column columnNames.unshift(dimensionColumn); - return columnNames + const columns = columnNames .map((name) => { let highlight = false; if (sortType === SortType.DIMENSION) { highlight = name === dimensionColumn; } else { highlight = - name === selectedMeasure.name || + name === selectedMeasure?.name || name.endsWith("_delta") || name.endsWith("_delta_perc") || name.endsWith("_percent_of_total"); } - let sorted = undefined; + let sorted; if (name.endsWith("_delta") && sortType === SortType.DELTA_ABSOLUTE) { sorted = sortDirection; } else if ( @@ -274,19 +279,23 @@ export function prepareVirtualizedDimTableColumns( sortType === SortType.PERCENT ) { sorted = sortDirection; - } else if (name === selectedMeasure.name && sortType === SortType.VALUE) { + } else if ( + name === selectedMeasure?.name && + sortType === SortType.VALUE + ) { sorted = sortDirection; } + let columnOut: VirtualizedTableColumns | undefined = undefined; if (measureNames.includes(name)) { // Handle all regular measures const measure = allMeasures.find((m) => m.name === name); - return { + columnOut = { name, type: "INT", label: measure?.label || measure?.expression, description: measure?.description, - total: referenceValues[measure.name] || 0, + total: referenceValues[measure?.name ?? ""] || 0, enableResize: false, format: measure?.formatPreset, highlight, @@ -294,7 +303,7 @@ export function prepareVirtualizedDimTableColumns( }; } else if (name === dimensionColumn) { // Handle dimension column - return { + columnOut = { name, type: "VARCHAR", label: dimension?.label, @@ -305,7 +314,7 @@ export function prepareVirtualizedDimTableColumns( } else if (selectedMeasure) { // Handle delta and delta_perc const comparison = getComparisonProperties(name, selectedMeasure); - return { + columnOut = { name, type: comparison.type, label: comparison.component, @@ -316,9 +325,12 @@ export function prepareVirtualizedDimTableColumns( sorted, }; } - return undefined; + return columnOut; }) - .filter((column) => !!column); + .filter((column) => column !== undefined); + + // cast is safe, because we filtered out undefined columns + return (columns as VirtualizedTableColumns[]) ?? []; } /** @@ -334,6 +346,7 @@ export function addContextColumnNames( selectedMeasure: MetricsViewSpecMeasureV2 ) { const name = selectedMeasure?.name; + if (!name) return; const sortByColumnIndex = columnNames.indexOf(name); // Add comparison columns if available @@ -379,55 +392,67 @@ export function prepareDimensionTableRows( ): DimensionTableRow[] { if (!queryRows || !queryRows.length) return []; - const formattersForMeasures = Object.fromEntries( - allMeasuresForSpec.map((m) => [m.name, createMeasureValueFormatter(m)]) - ); - - const tableRows: DimensionTableRow[] = queryRows.map((row) => { - if (!row.measureValues) return {}; - - const rawVals: [string, number][] = row.measureValues.map((m) => { - return [m.measureName ?? "", m.baseValue ? (m.baseValue as number) : 0]; - }); - - const formattedVals: [string, string | number][] = rawVals.map( - ([name, val]) => ["__formatted_" + name, formattersForMeasures[name](val)] + const formattersForMeasures: { [key: string]: (val: number) => string } = + Object.fromEntries( + allMeasuresForSpec.map((m) => [m.name, createMeasureValueFormatter(m)]) ); - const rowOut: DimensionTableRow = Object.fromEntries([ - [dimensionColumn, row.dimensionValue as string], - ...rawVals, - ...formattedVals, - ]); + const tableRows: DimensionTableRow[] = queryRows + .filter( + (row) => row.measureValues !== undefined && row.measureValues !== null + ) + .map((row) => { + // cast is safe since we filtered out rows without measureValues + const measureValues = row.measureValues as V1MetricsViewComparisonValue[]; + + const rawVals: [string, number][] = measureValues.map((m) => [ + m.measureName?.toString() ?? "", + m.baseValue ? (m.baseValue as number) : 0, + ]); + + const formattedVals: [string, string | number][] = rawVals.map( + ([name, val]) => [ + "__formatted_" + name, + formattersForMeasures[name](val), + ] + ); - const activeMeasure = row.measureValues?.find( - (m) => m.measureName === activeMeasureName - ); + const rowOut: DimensionTableRow = Object.fromEntries([ + [dimensionColumn, row.dimensionValue as string], + ...rawVals, + ...formattedVals, + ]); - if (addDeltas && activeMeasure) { - rowOut[`${activeMeasureName}_delta`] = activeMeasure.deltaAbs - ? formattersForMeasures[activeMeasureName](activeMeasure.deltaAbs) - : PERC_DIFF.PREV_VALUE_NO_DATA; + const activeMeasure = measureValues.find( + (m) => m.measureName === activeMeasureName + ); - rowOut[`${activeMeasureName}_delta_perc`] = activeMeasure.deltaRel - ? formatMeasurePercentageDifference(activeMeasure.deltaRel as number) - : PERC_DIFF.PREV_VALUE_NO_DATA; - } + if (addDeltas && activeMeasure) { + rowOut[`${activeMeasureName}_delta`] = activeMeasure.deltaAbs + ? formattersForMeasures[activeMeasureName]( + activeMeasure.deltaAbs as number + ) + : PERC_DIFF.PREV_VALUE_NO_DATA; - if (addPercentOfTotal && activeMeasure) { - const value = activeMeasure.baseValue as number; + rowOut[`${activeMeasureName}_delta_perc`] = activeMeasure.deltaRel + ? formatMeasurePercentageDifference(activeMeasure.deltaRel as number) + : PERC_DIFF.PREV_VALUE_NO_DATA; + } - if (unfilteredTotal === 0 || !unfilteredTotal) { - rowOut[activeMeasureName + "_percent_of_total"] = - PERC_DIFF.CURRENT_VALUE_NO_DATA; - } else { - rowOut[activeMeasureName + "_percent_of_total"] = - formatMeasurePercentageDifference(value / unfilteredTotal); + if (addPercentOfTotal && activeMeasure) { + const value = activeMeasure.baseValue as number; + + if (unfilteredTotal === 0 || !unfilteredTotal) { + rowOut[activeMeasureName + "_percent_of_total"] = + PERC_DIFF.CURRENT_VALUE_NO_DATA; + } else { + rowOut[activeMeasureName + "_percent_of_total"] = + formatMeasurePercentageDifference(value / unfilteredTotal); + } } - } - return rowOut; - }); + return rowOut; + }); return tableRows; } @@ -439,8 +464,8 @@ export function getSelectedRowIndicesFromFilters( ): number[] { const selectedDimValues = ((excludeMode - ? filters.exclude.find((d) => d.name === dimensionName)?.in - : filters.include.find((d) => d.name === dimensionName) + ? filters?.exclude?.find((d) => d.name === dimensionName)?.in + : filters?.include?.find((d) => d.name === dimensionName) ?.in) as string[]) ?? []; return selectedDimValues diff --git a/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte b/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte index d4f4ac60f27..3961d868981 100644 --- a/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte +++ b/web-common/src/features/dashboards/leaderboard/DimensionCompareMenu.svelte @@ -3,16 +3,30 @@ import Compare from "@rilldata/web-common/components/icons/Compare.svelte"; import Tooltip from "@rilldata/web-common/components/tooltip/Tooltip.svelte"; import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; - import { createEventDispatcher } from "svelte"; + import { getStateManagers } from "../state-managers/state-managers"; - export let isBeingCompared = false; + export let dimensionName: string | undefined; - const dispatch = createEventDispatcher(); + const { + selectors: { + comparison: { isBeingCompared: isBeingComparedReadable }, + }, + actions: { + comparison: { setComparisonDimension }, + }, + } = getStateManagers(); + + $: isBeingCompared = + dimensionName !== undefined && $isBeingComparedReadable(dimensionName); { - dispatch("toggle-dimension-comparison"); + if (isBeingCompared) { + setComparisonDimension(undefined); + } else { + setComparisonDimension(dimensionName); + } e.stopPropagation(); }} > diff --git a/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte b/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte index 86e9cf8fa76..0683f231cc7 100644 --- a/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte +++ b/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte @@ -7,24 +7,12 @@ */ import Tooltip from "@rilldata/web-common/components/tooltip/Tooltip.svelte"; import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; - import { - getFilterForDimension, - useMetaDimension, - useMetaMeasure, - } from "@rilldata/web-common/features/dashboards/selectors"; import { getStateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; - import { useTimeControlStore } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; import { createQueryServiceMetricsViewComparison, - MetricsViewDimension, - MetricsViewSpecMeasureV2, + createQueryServiceMetricsViewTotals, } from "@rilldata/web-common/runtime-client"; - import { runtime } from "../../../runtime-client/runtime-store"; - import { SortDirection } from "../proto-state/derived-types"; - import { - metricsExplorerStore, - useDashboardStore, - } from "web-common/src/features/dashboards/stores/dashboard-stores"; + import LeaderboardHeader from "./LeaderboardHeader.svelte"; import { LeaderboardItemData, @@ -32,125 +20,63 @@ prepareLeaderboardItemData, } from "./leaderboard-utils"; import LeaderboardListItem from "./LeaderboardListItem.svelte"; - import { prepareSortedQueryBody } from "../dashboard-utils"; - export let metricViewName: string; export let dimensionName: string; /** The reference value is the one that the bar in the LeaderboardListItem * gets scaled with. For a summable metric, the total is a reference value, * or for a count(*) metric, the reference value is the total number of rows. */ export let referenceValue: number; - export let unfilteredTotal: number; let slice = 7; - const stateManagers = getStateManagers(); - - $: dashboardStore = useDashboardStore(metricViewName); - - let filterExcludeMode: boolean; - $: filterExcludeMode = - $dashboardStore?.dimensionFilterExcludeMode.get(dimensionName) ?? false; - let filterKey: "exclude" | "include"; - $: filterKey = filterExcludeMode ? "exclude" : "include"; + const { + selectors: { + activeMeasure: { activeMeasureName }, + dimensionFilters: { selectedDimensionValues, isFilterExcludeMode }, + dashboardQueries: { + leaderboardSortedQueryBody, + leaderboardSortedQueryOptions, + leaderboardDimensionTotalQueryBody, + leaderboardDimensionTotalQueryOptions, + }, + }, + actions: { + dimensions: { setPrimaryDimension }, + }, + metricsViewName, + runtime, + } = getStateManagers(); - $: dimensionQuery = useMetaDimension( + $: sortedQuery = createQueryServiceMetricsViewComparison( $runtime.instanceId, - metricViewName, - dimensionName + $metricsViewName, + $leaderboardSortedQueryBody(dimensionName), + $leaderboardSortedQueryOptions(dimensionName) ); - let dimension: MetricsViewDimension; - $: dimension = $dimensionQuery?.data; - $: displayName = dimension?.label || dimension?.name || dimensionName; - $: measureQuery = useMetaMeasure( + $: totalsQuery = createQueryServiceMetricsViewTotals( $runtime.instanceId, - metricViewName, - $dashboardStore?.leaderboardMeasureName - ); - let measure: MetricsViewSpecMeasureV2; - $: measure = $measureQuery?.data; - - $: filterForDimension = getFilterForDimension( - $dashboardStore?.filters, - dimensionName - ); - - // FIXME: it is possible for this way of accessing the filters - // to return the same value twice, which would seem to indicate - // a bug in the way we're setting the filters / active values. - // Need to investigate further to determine whether this is a - // problem with the runtime or the client, but for now wrapping - // it in a set dedupes the values. - $: activeValues = new Set( - ($dashboardStore?.filters[filterKey]?.find( - (d) => d.name === dimension?.name - )?.in as (number | string)[]) ?? [] - ); - $: atLeastOneActive = activeValues?.size > 0; - - const timeControlsStore = useTimeControlStore(stateManagers); - - function selectDimension(dimensionName) { - metricsExplorerStore.setMetricDimensionName(metricViewName, dimensionName); - } - - function toggleComparisonDimension(dimensionName, isBeingCompared) { - metricsExplorerStore.setComparisonDimension( - metricViewName, - isBeingCompared ? undefined : dimensionName - ); - } - - function toggleSort(evt) { - metricsExplorerStore.toggleSort(metricViewName, evt.detail); - } - - $: isBeingCompared = - $dashboardStore?.selectedComparisonDimension === dimensionName; - - $: sortAscending = $dashboardStore.sortDirection === SortDirection.ASCENDING; - $: sortType = $dashboardStore.dashboardSortType; - - $: sortedQueryBody = prepareSortedQueryBody( - dimensionName, - [measure?.name], - $timeControlsStore, - measure?.name, - sortType, - sortAscending, - filterForDimension + $metricsViewName, + $leaderboardDimensionTotalQueryBody(dimensionName), + $leaderboardDimensionTotalQueryOptions(dimensionName) ); - $: sortedQueryEnabled = $timeControlsStore.ready && !!filterForDimension; - - $: sortedQueryOptions = { - query: { - enabled: sortedQueryEnabled, - }, - }; - - $: sortedQuery = createQueryServiceMetricsViewComparison( - $runtime.instanceId, - metricViewName, - sortedQueryBody, - sortedQueryOptions - ); + $: leaderboardTotal = $totalsQuery?.data?.data?.[$activeMeasureName]; let aboveTheFold: LeaderboardItemData[] = []; let selectedBelowTheFold: LeaderboardItemData[] = []; let noAvailableValues = true; let showExpandTable = false; - $: if (!$sortedQuery?.isFetching) { + $: if (sortedQuery && !$sortedQuery?.isFetching) { const leaderboardData = prepareLeaderboardItemData( $sortedQuery?.data?.rows?.map((r) => - getLabeledComparisonFromComparisonRow(r, measure.name) + getLabeledComparisonFromComparisonRow(r, $activeMeasureName) ) ?? [], slice, - [...activeValues], - unfilteredTotal, - filterExcludeMode + $selectedDimensionValues(dimensionName), + leaderboardTotal, + $isFilterExcludeMode(dimensionName) ); aboveTheFold = leaderboardData.aboveTheFold; @@ -162,7 +88,7 @@ let hovered: boolean; -{#if sortedQuery} +{#if $sortedQuery !== undefined}
(hovered = true)} @@ -170,28 +96,19 @@ > - toggleComparisonDimension(dimensionName, isBeingCompared)} - {isBeingCompared} + {dimensionName} {hovered} - dimensionDescription={dimension?.description || ""} - on:open-dimension-details={() => selectDimension(dimensionName)} - on:toggle-sort={toggleSort} /> {#if aboveTheFold || selectedBelowTheFold}
{#each aboveTheFold as itemData (itemData.dimensionValue)} {/each} @@ -199,14 +116,11 @@
{#each selectedBelowTheFold as itemData (itemData.dimensionValue)} {/each} @@ -224,7 +138,7 @@ {#if showExpandTable}
diff --git a/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte b/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte index 8f05f99c4f1..20a869d4d59 100644 --- a/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte +++ b/web-common/src/features/dashboards/workspace/DashboardCTAs.svelte @@ -30,7 +30,7 @@ $: dashboardQuery = useDashboard($runtime.instanceId, metricViewName); $: dashboardIsIdle = - $dashboardQuery.data.meta.reconcileStatus === + $dashboardQuery.data?.meta?.reconcileStatus === V1ReconcileStatus.RECONCILE_STATUS_IDLE; function viewMetrics(metricViewName: string) { diff --git a/web-common/src/features/entity-management/resource-invalidations.ts b/web-common/src/features/entity-management/resource-invalidations.ts index e17b7631aba..0c19ddd5079 100644 --- a/web-common/src/features/entity-management/resource-invalidations.ts +++ b/web-common/src/features/entity-management/resource-invalidations.ts @@ -1,5 +1,8 @@ import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors"; -import { resourcesStore } from "@rilldata/web-common/features/entity-management/resources-store"; +import { + getLastStateUpdatedOn, + resourcesStore, +} from "@rilldata/web-common/features/entity-management/resources-store"; import type { V1WatchResourcesResponse } from "@rilldata/web-common/runtime-client"; import { getRuntimeServiceGetResourceQueryKey, @@ -78,12 +81,19 @@ async function invalidateResource( ) { refreshResource(queryClient, instanceId, resource); - if (resource.meta.reconcileStatus !== V1ReconcileStatus.RECONCILE_STATUS_IDLE) + const lastStateUpdatedOn = getLastStateUpdatedOn(resource); + if ( + resource.meta.reconcileStatus !== V1ReconcileStatus.RECONCILE_STATUS_IDLE || + lastStateUpdatedOn === resource.meta.stateUpdatedOn + ) return; + + resourcesStore.setVersion(resource); const failed = !!resource.meta.reconcileError; switch (resource.meta.name.kind) { case ResourceKind.Source: + // eslint-disable-next-line no-fallthrough case ResourceKind.Model: return invalidateProfilingQueries( queryClient, @@ -111,6 +121,7 @@ async function invalidateRemovedResource( "name.kind": resource.meta.name.kind, }) ); + resourcesStore.deleteResource(resource); switch (resource.meta.name.kind) { case ResourceKind.Source: case ResourceKind.Model: diff --git a/web-common/src/features/entity-management/resource-status-utils.ts b/web-common/src/features/entity-management/resource-status-utils.ts index ce4efad7ec5..6fc1f3f8fd5 100644 --- a/web-common/src/features/entity-management/resource-status-utils.ts +++ b/web-common/src/features/entity-management/resource-status-utils.ts @@ -1,6 +1,10 @@ -import { useProjectParser } from "@rilldata/web-common/features/entity-management/resource-selectors"; +import { + ResourceKind, + useProjectParser, +} from "@rilldata/web-common/features/entity-management/resource-selectors"; import { getAllErrorsForFile, + getLastStateUpdatedOnByKindAndName, getResourceNameForFile, useResourceForFile, } from "@rilldata/web-common/features/entity-management/resources-store"; @@ -71,13 +75,95 @@ export function waitForResource( }); } +/** + * Used while saving to wait until either a resource is created or parse has errored. + */ +export function resourceStatusStore( + queryClient: QueryClient, + instanceId: string, + filePath: string, + kind: ResourceKind, + name: string +) { + const lastUpdatedOn = getLastStateUpdatedOnByKindAndName(kind, name); + return derived( + [ + useResourceForFile(queryClient, instanceId, filePath), + getAllErrorsForFile(queryClient, instanceId, filePath), + ], + ([res, errors]) => { + if (res.isFetching) return { status: ResourceStatus.Busy }; + if ( + (res.isError && (res.error as any).response.status !== 404) || + errors.length > 0 + ) + return { status: ResourceStatus.Errored, changed: false }; + + if ( + res.data?.meta?.reconcileStatus !== + V1ReconcileStatus.RECONCILE_STATUS_IDLE + ) + return { status: ResourceStatus.Busy }; + + return { + status: !res.data?.meta?.reconcileError + ? ResourceStatus.Idle + : ResourceStatus.Errored, + changed: + !lastUpdatedOn || res.data?.meta?.stateUpdatedOn > lastUpdatedOn, + }; + } + ); +} + +export function waitForResourceUpdate( + queryClient: QueryClient, + instanceId: string, + filePath: string, + kind: ResourceKind, + name: string +) { + return new Promise((resolve) => { + let timer; + let idled = false; + + const end = (changed: boolean) => { + unsub?.(); + resolve(changed); + }; + + // eslint-disable-next-line prefer-const + const unsub = resourceStatusStore( + queryClient, + instanceId, + filePath, + kind, + name + ).subscribe((status) => { + if (status.status === ResourceStatus.Busy) return; + if (timer) clearTimeout(timer); + + if (idled) { + end(status.status === ResourceStatus.Idle && status.changed); + return; + } else { + idled = true; + timer = setTimeout(() => { + end(status.status === ResourceStatus.Idle && status.changed); + }, 500); + } + }); + }); +} + /** * Assumes the initial resource has been created after a new entity creation. */ export function getResourceStatusStore( queryClient: QueryClient, instanceId: string, - filePath: string + filePath: string, + validator?: (res: V1Resource) => boolean ): Readable { return derived( [ @@ -104,10 +190,15 @@ export function getResourceStatusStore( }; } - const isBusy = - resourceResp.isFetching || - resourceResp.data?.meta?.reconcileStatus !== - V1ReconcileStatus.RECONCILE_STATUS_IDLE; + let isBusy: boolean; + if (validator && resourceResp.data) { + isBusy = !validator(resourceResp.data); + } else { + isBusy = + resourceResp.isFetching || + resourceResp.data?.meta?.reconcileStatus !== + V1ReconcileStatus.RECONCILE_STATUS_IDLE; + } return { status: isBusy ? ResourceStatus.Busy : ResourceStatus.Idle, diff --git a/web-common/src/features/entity-management/resources-store.ts b/web-common/src/features/entity-management/resources-store.ts index d1ea91e5bbe..e423e636fe4 100644 --- a/web-common/src/features/entity-management/resources-store.ts +++ b/web-common/src/features/entity-management/resources-store.ts @@ -14,7 +14,7 @@ import type { } from "@rilldata/web-common/runtime-client"; import type { QueryClient } from "@tanstack/svelte-query"; import type { CreateQueryResult } from "@tanstack/svelte-query"; -import { derived, Readable, writable } from "svelte/store"; +import { derived, get, Readable, writable } from "svelte/store"; /** * Global resources store that maps file name to a resource. @@ -28,11 +28,15 @@ export type ResourcesState = { // array of paths currently reconciling // we use path since parse error will only give us paths from ProjectParser currentlyReconciling: Record; + // last time the state of the resource `kind/name` was updated + // used to make sure we do not have unnecessary refreshes + lastStateUpdatedOn: Record; }; const { update, subscribe } = writable({ resources: {}, currentlyReconciling: {}, + lastStateUpdatedOn: {}, }); const resourcesStoreReducers = { @@ -50,6 +54,7 @@ const resourcesStoreReducers = { ) { this.reconciling(resource); } + this.setVersion(resource); break; } } @@ -95,6 +100,21 @@ const resourcesStoreReducers = { return state; }); }, + + setVersion(resource: V1Resource) { + update((state) => { + state.lastStateUpdatedOn[getKeyForResource(resource)] = + resource.meta.stateUpdatedOn; + return state; + }); + }, + + deleteResource(resource: V1Resource) { + update((state) => { + delete state.lastStateUpdatedOn[getKeyForResource(resource)]; + return state; + }); + }, }; export type ResourcesStore = Readable & @@ -182,3 +202,18 @@ export function getReconcilingItems() { return currentlyReconciling; }); } + +export function getLastStateUpdatedOn(resource: V1Resource) { + return get(resourcesStore).lastStateUpdatedOn[getKeyForResource(resource)]; +} + +export function getLastStateUpdatedOnByKindAndName( + kind: ResourceKind, + name: string +) { + return get(resourcesStore).lastStateUpdatedOn[`${kind}/${name}`]; +} + +function getKeyForResource(resource: V1Resource) { + return `${resource.meta.name.kind}/${resource.meta.name.name}`; +} diff --git a/web-common/src/features/sources/createDashboard.ts b/web-common/src/features/sources/createDashboard.ts index 5b61320a504..4f2982f998d 100644 --- a/web-common/src/features/sources/createDashboard.ts +++ b/web-common/src/features/sources/createDashboard.ts @@ -1,9 +1,23 @@ +import { goto } from "$app/navigation"; +import { useDashboardFileNames } from "@rilldata/web-common/features/dashboards/selectors"; import { getFileAPIPathFromNameAndType, getFilePathFromNameAndType, } from "@rilldata/web-common/features/entity-management/entity-mappers"; +import { getName } from "@rilldata/web-common/features/entity-management/name-utils"; import { waitForResource } from "@rilldata/web-common/features/entity-management/resource-status-utils"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; +import { useModelFileNames } from "@rilldata/web-common/features/models/selectors"; +import { useSource } from "@rilldata/web-common/features/sources/selectors"; +import { appScreen } from "@rilldata/web-common/layout/app-store"; +import { overlay } from "@rilldata/web-common/layout/overlay-store"; +import { waitUntil } from "@rilldata/web-common/lib/waitUtils"; +import { behaviourEvent } from "@rilldata/web-common/metrics/initMetrics"; +import { BehaviourEventMedium } from "@rilldata/web-common/metrics/service/BehaviourEventTypes"; +import { + MetricsEventScreenName, + MetricsEventSpace, +} from "@rilldata/web-common/metrics/service/MetricsTypes"; import { connectorServiceOLAPGetTable, RpcStatus, @@ -16,6 +30,7 @@ import { MutationFunction, useQueryClient, } from "@tanstack/svelte-query"; +import { get } from "svelte/store"; import { generateDashboardYAMLForModel } from "../metrics-views/metrics-internal-store"; export interface CreateDashboardFromSourceRequest { @@ -45,12 +60,13 @@ export const useCreateDashboardFromSource = < > = async (props) => { const { data } = props ?? {}; const sourceName = data.sourceResource?.meta?.name?.name; - if (!sourceName) throw new Error("Source name is missing"); + if (!sourceName) + throw new Error("Failed to create dashboard: Source name is missing"); if ( !data.sourceResource.source.state.connector || !data.sourceResource.source.state.table ) - throw new Error("Source is not ready"); + throw new Error("Failed to create dashboard: Source is not ready"); // first, create model from source @@ -106,3 +122,56 @@ export const useCreateDashboardFromSource = < TContext >(mutationFn, mutationOptions); }; + +/** + * Wrapper function that takes care of UI side effects on top of creating a dashboard from source. + * TODO: where would this go? + */ +export function useCreateDashboardFromSourceUIAction( + instanceId: string, + sourceName: string +) { + const createDashboardFromSourceMutation = useCreateDashboardFromSource(); + const modelNames = useModelFileNames(instanceId); + const dashboardNames = useDashboardFileNames(instanceId); + const sourceQuery = useSource(instanceId, sourceName); + + return async () => { + overlay.set({ + title: "Creating a dashboard for " + sourceName, + }); + const newModelName = getName(`${sourceName}_model`, get(modelNames).data); + const newDashboardName = getName( + `${sourceName}_dashboard`, + get(dashboardNames).data + ); + + // Wait for source query to have data + await waitUntil(() => !!get(sourceQuery).data); + + try { + await get(createDashboardFromSourceMutation).mutateAsync({ + data: { + instanceId, + sourceResource: get(sourceQuery).data, + newModelName, + newDashboardName, + }, + }); + goto(`/dashboard/${newDashboardName}`); + behaviourEvent.fireNavigationEvent( + newDashboardName, + BehaviourEventMedium.Menu, + MetricsEventSpace.LeftPanel, + get(appScreen)?.type, + MetricsEventScreenName.Dashboard + ); + overlay.set(null); + } catch (err) { + overlay.set({ + title: "Failed to create a dashboard for " + sourceName, + message: err.response?.data?.message ?? err.message, + }); + } + }; +} diff --git a/web-common/src/features/sources/editor/SourceEditor.svelte b/web-common/src/features/sources/editor/SourceEditor.svelte index 944c13eb746..0ecc55f4b59 100644 --- a/web-common/src/features/sources/editor/SourceEditor.svelte +++ b/web-common/src/features/sources/editor/SourceEditor.svelte @@ -1,17 +1,18 @@ diff --git a/web-common/src/features/sources/modal/FileDrop.svelte b/web-common/src/features/sources/modal/FileDrop.svelte index 2aebef0460b..7ee4797a363 100644 --- a/web-common/src/features/sources/modal/FileDrop.svelte +++ b/web-common/src/features/sources/modal/FileDrop.svelte @@ -1,9 +1,12 @@ diff --git a/web-common/src/features/sources/modal/LocalSourceUpload.svelte b/web-common/src/features/sources/modal/LocalSourceUpload.svelte index d12cc142414..e0ed3fcf09e 100644 --- a/web-common/src/features/sources/modal/LocalSourceUpload.svelte +++ b/web-common/src/features/sources/modal/LocalSourceUpload.svelte @@ -1,14 +1,17 @@ diff --git a/web-common/src/features/sources/modal/SourceImportedModal.svelte b/web-common/src/features/sources/modal/SourceImportedModal.svelte new file mode 100644 index 00000000000..5bbcfb7a896 --- /dev/null +++ b/web-common/src/features/sources/modal/SourceImportedModal.svelte @@ -0,0 +1,49 @@ + + + +
+
+ +
+
Source "{$sourceImportedName}" imported successfully.
+
+
+
+
What would you like to do next?
+
+
+ + +
+
 diff --git a/web-common/src/features/sources/modal/submitRemoteSourceForm.ts b/web-common/src/features/sources/modal/submitRemoteSourceForm.ts index d632f229d1d..86d7b536e8b 100644 --- a/web-common/src/features/sources/modal/submitRemoteSourceForm.ts +++ b/web-common/src/features/sources/modal/submitRemoteSourceForm.ts @@ -1,3 +1,4 @@ +import { checkSourceImported } from "@rilldata/web-common/features/sources/source-imported-utils"; import type { QueryClient } from "@tanstack/query-core"; import { get } from "svelte/store"; import { appScreen } from "../../../layout/app-store"; @@ -12,7 +13,10 @@ import { runtimeServiceUnpackEmpty, } from "../../../runtime-client"; import { runtime } from "../../../runtime-client/runtime-store"; -import { getFileAPIPathFromNameAndType } from "../../entity-management/entity-mappers"; +import { + getFileAPIPathFromNameAndType, + getFilePathFromNameAndType, +} from "../../entity-management/entity-mappers"; import { EntityType } from "../../entity-management/types"; import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { isProjectInitializedV2 } from "../../welcome/is-project-initialized"; @@ -59,7 +63,6 @@ export async function submitRemoteSourceForm( case "account": case "output_location": case "workgroup": - return [key, value]; case "database_url": return [key, value]; default: @@ -79,6 +82,11 @@ export async function submitRemoteSourceForm( createOnly: false, // The modal might be opened from a YAML file with placeholder text, so the file might already exist } ); + checkSourceImported( + queryClient, + values.sourceName, + getFilePathFromNameAndType(values.sourceName, EntityType.Table) + ); // TODO: telemetry // Emit telemetry diff --git a/web-common/src/features/sources/source-imported-utils.ts b/web-common/src/features/sources/source-imported-utils.ts new file mode 100644 index 00000000000..5562b320fcf --- /dev/null +++ b/web-common/src/features/sources/source-imported-utils.ts @@ -0,0 +1,30 @@ +import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors"; +import { waitForResourceUpdate } from "@rilldata/web-common/features/entity-management/resource-status-utils"; +import { getLastStateUpdatedOnByKindAndName } from "@rilldata/web-common/features/entity-management/resources-store"; +import { sourceImportedName } from "@rilldata/web-common/features/sources/sources-store"; +import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; +import type { QueryClient } from "@tanstack/svelte-query"; +import { get } from "svelte/store"; + +export function checkSourceImported( + queryClient: QueryClient, + sourceName: string, + filePath: string +) { + const lastUpdatedOn = getLastStateUpdatedOnByKindAndName( + ResourceKind.Source, + sourceName + ); + if (lastUpdatedOn) return; // For now only show for fresh sources + waitForResourceUpdate( + queryClient, + get(runtime).instanceId, + filePath, + ResourceKind.Source, + sourceName + ).then((success) => { + if (!success) return; + sourceImportedName.set(sourceName); + // TODO: telemetry + }); +} diff --git a/web-common/src/features/sources/sources-store.ts b/web-common/src/features/sources/sources-store.ts index 6b686858824..d406142ab18 100644 --- a/web-common/src/features/sources/sources-store.ts +++ b/web-common/src/features/sources/sources-store.ts @@ -31,3 +31,5 @@ export function useSourceStore(sourceName: string): Writable { return sourceStores[sourceName]; } + +export const sourceImportedName = writable(null); diff --git a/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte b/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte index 8caf3ce9e87..ba97769286b 100644 --- a/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte +++ b/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte @@ -14,6 +14,7 @@ } from "@rilldata/web-common/features/entity-management/resource-selectors"; import { getFileHasErrors } from "@rilldata/web-common/features/entity-management/resources-store"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; + import { checkSourceImported } from "@rilldata/web-common/features/sources/source-imported-utils"; import { overlay } from "@rilldata/web-common/layout/overlay-store"; import { slideRight } from "@rilldata/web-common/lib/transitions"; import { @@ -117,6 +118,7 @@ const onSaveAndRefreshClick = async (tableName: string) => { overlay.set({ title: `Importing ${tableName}.yaml` }); await saveAndRefresh(tableName, $sourceStore.clientYAML); + checkSourceImported(queryClient, sourceName, filePath); overlay.set(null); }; diff --git a/web-common/src/features/welcome/ProjectCards.svelte b/web-common/src/features/welcome/ProjectCards.svelte index ea0bb525344..c0ef3187ba6 100644 --- a/web-common/src/features/welcome/ProjectCards.svelte +++ b/web-common/src/features/welcome/ProjectCards.svelte @@ -31,11 +31,12 @@ firstPage: "dashboard/auction", }, { - name: "rill-311-ops", - title: "311 Call Center Operations", - description: "Citizen reports across US cities", - image: "bg-[url('$img/welcome-bg-311.png')] bg-no-repeat bg-cover", - firstPage: "dashboard/call_center_metrics", + name: "rill-github-analytics", + title: "Github Analytics", + description: "A Git project's commit activity", + image: + "bg-[url('$img/welcome-bg-github-analytics.png')] bg-no-repeat bg-cover", + firstPage: "dashboard/duckdb_commits", }, ]; diff --git a/web-common/src/layout/RillDeveloperLayout.svelte b/web-common/src/layout/RillDeveloperLayout.svelte index 5a16f48c244..171c48f4c41 100644 --- a/web-common/src/layout/RillDeveloperLayout.svelte +++ b/web-common/src/layout/RillDeveloperLayout.svelte @@ -4,7 +4,11 @@ import { featureFlags } from "@rilldata/web-common/features/feature-flags"; import DuplicateSource from "@rilldata/web-common/features/sources/modal/DuplicateSource.svelte"; import FileDrop from "@rilldata/web-common/features/sources/modal/FileDrop.svelte"; - import { duplicateSourceName } from "@rilldata/web-common/features/sources/sources-store"; + import SourceImportedModal from "@rilldata/web-common/features/sources/modal/SourceImportedModal.svelte"; + import { + duplicateSourceName, + sourceImportedName, + } from "@rilldata/web-common/features/sources/sources-store"; import BlockingOverlayContainer from "@rilldata/web-common/layout/BlockingOverlayContainer.svelte"; import { initMetrics } from "@rilldata/web-common/metrics/initMetrics"; import type { ApplicationBuildMetadata } from "@rilldata/web-local/lib/application-state-stores/build-metadata"; @@ -77,6 +81,7 @@ {#if $duplicateSourceName !== null} {/if} +
{ */ firstMonthOfYear = 0; + /** + * Selected default comparison mode. + * + * @generated from field: rill.runtime.v1.MetricsViewSpec.ComparisonMode default_comparison_mode = 14; + */ + defaultComparisonMode = MetricsViewSpec_ComparisonMode.UNSPECIFIED; + + /** + * If comparison mode is dimension then this determines which is the default dimension + * + * @generated from field: string default_comparison_dimension = 15; + */ + defaultComparisonDimension = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -963,6 +977,8 @@ export class MetricsViewSpec extends Message { { no: 11, name: "security", kind: "message", T: MetricsViewSpec_SecurityV2 }, { no: 12, name: "first_day_of_week", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, { no: 13, name: "first_month_of_year", kind: "scalar", T: 13 /* ScalarType.UINT32 */ }, + { no: 14, name: "default_comparison_mode", kind: "enum", T: proto3.getEnumType(MetricsViewSpec_ComparisonMode) }, + { no: 15, name: "default_comparison_dimension", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): MetricsViewSpec { @@ -982,6 +998,38 @@ export class MetricsViewSpec extends Message { } } +/** + * @generated from enum rill.runtime.v1.MetricsViewSpec.ComparisonMode + */ +export enum MetricsViewSpec_ComparisonMode { + /** + * @generated from enum value: COMPARISON_MODE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: COMPARISON_MODE_NONE = 1; + */ + NONE = 1, + + /** + * @generated from enum value: COMPARISON_MODE_TIME = 2; + */ + TIME = 2, + + /** + * @generated from enum value: COMPARISON_MODE_DIMENSION = 3; + */ + DIMENSION = 3, +} +// Retrieve enum metadata with: proto3.getEnumType(MetricsViewSpec_ComparisonMode) +proto3.util.setEnumType(MetricsViewSpec_ComparisonMode, "rill.runtime.v1.MetricsViewSpec.ComparisonMode", [ + { no: 0, name: "COMPARISON_MODE_UNSPECIFIED" }, + { no: 1, name: "COMPARISON_MODE_NONE" }, + { no: 2, name: "COMPARISON_MODE_TIME" }, + { no: 3, name: "COMPARISON_MODE_DIMENSION" }, +]); + /** * Dimensions are columns to filter and group by * @@ -1008,6 +1056,11 @@ export class MetricsViewSpec_DimensionV2 extends Message) { super(); proto3.util.initPartial(data, this); @@ -1020,6 +1073,7 @@ export class MetricsViewSpec_DimensionV2 extends Message): MetricsViewSpec_DimensionV2 { diff --git a/web-common/src/proto/gen/rill/ui/v1/dashboard_pb.ts b/web-common/src/proto/gen/rill/ui/v1/dashboard_pb.ts index ad72ec70747..ed5b297f5df 100644 --- a/web-common/src/proto/gen/rill/ui/v1/dashboard_pb.ts +++ b/web-common/src/proto/gen/rill/ui/v1/dashboard_pb.ts @@ -232,7 +232,7 @@ proto3.util.setEnumType(DashboardState_LeaderboardSortDirection, "rill.ui.v1.Das ]); /** - * * + * * SortType is used to determine how to sort the leaderboard * and dimension detail table, as well as where to place the * sort arrow. diff --git a/web-common/src/runtime-client/gen/index.schemas.ts b/web-common/src/runtime-client/gen/index.schemas.ts index 9a539ef66cc..e0eaa110673 100644 --- a/web-common/src/runtime-client/gen/index.schemas.ts +++ b/web-common/src/runtime-client/gen/index.schemas.ts @@ -1158,6 +1158,8 @@ export interface V1MetricsViewSpec { firstDayOfWeek?: number; /** Month number to use as the base for time aggregations by year. Defaults to 1 (January). */ firstMonthOfYear?: number; + defaultComparisonMode?: MetricsViewSpecComparisonMode; + defaultComparisonDimension?: string; } export interface V1MetricsViewState { @@ -1865,8 +1867,20 @@ export interface MetricsViewSpecDimensionV2 { column?: string; label?: string; description?: string; + unnest?: boolean; } +export type MetricsViewSpecComparisonMode = + (typeof MetricsViewSpecComparisonMode)[keyof typeof MetricsViewSpecComparisonMode]; + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const MetricsViewSpecComparisonMode = { + COMPARISON_MODE_UNSPECIFIED: "COMPARISON_MODE_UNSPECIFIED", + COMPARISON_MODE_NONE: "COMPARISON_MODE_NONE", + COMPARISON_MODE_TIME: "COMPARISON_MODE_TIME", + COMPARISON_MODE_DIMENSION: "COMPARISON_MODE_DIMENSION", +} as const; + export interface MetricsViewSecurity { access?: string; rowFilter?: string; diff --git a/web-common/static/img/welcome-bg-311.png b/web-common/static/img/welcome-bg-311.png deleted file mode 100644 index f2caf2b0def..00000000000 Binary files a/web-common/static/img/welcome-bg-311.png and /dev/null differ diff --git a/web-common/static/img/welcome-bg-github-analytics.png b/web-common/static/img/welcome-bg-github-analytics.png new file mode 100644 index 00000000000..ff1c91034b1 Binary files /dev/null and b/web-common/static/img/welcome-bg-github-analytics.png differ diff --git a/web-local/src/lib/types.ts b/web-local/src/lib/types.ts index 0a4680dedd9..a2e4c3e1060 100644 --- a/web-local/src/lib/types.ts +++ b/web-local/src/lib/types.ts @@ -15,6 +15,7 @@ export interface VirtualizedTableColumns { // Is this the table sorted by this column, and if so, in what direction? // Leave undefined if the table is not sorted by this column. sorted?: SortDirection; + format?: string; } export type ProfileColumnSummary = diff --git a/web-local/src/routes/(application)/dashboard/[name]/+page.svelte b/web-local/src/routes/(application)/dashboard/[name]/+page.svelte index 829acb0fd2a..58c5346a72d 100644 --- a/web-local/src/routes/(application)/dashboard/[name]/+page.svelte +++ b/web-local/src/routes/(application)/dashboard/[name]/+page.svelte @@ -45,7 +45,8 @@ $: resourceStatusStore = getResourceStatusStore( queryClient, $runtime.instanceId, - filePath + filePath, + (res) => !!res?.metricsView?.state?.validSpec ); let showErrorPage = false; $: if (metricViewName) { diff --git a/web-local/test/ui/dashboards.spec.ts b/web-local/test/ui/dashboards.spec.ts index eb211dd6efa..168637a19b5 100644 --- a/web-local/test/ui/dashboards.spec.ts +++ b/web-local/test/ui/dashboards.spec.ts @@ -518,6 +518,8 @@ dimensions: await runThroughDefaultTimeRanges(page); + await runThroughDefaultComparisons(page); + // go back to the dashboard // TODO @@ -952,6 +954,115 @@ dimensions: await page.getByRole("button", { name: "Edit metrics" }).click(); } +async function runThroughDefaultComparisons(page: Page) { + /** + * SUBFLOW: Change default time comparison and assert it updates the selections. + */ + + // Set comparison to time + const docWithPresetDefaultTimeComparison = `# Visit https://docs.rilldata.com/reference/project-files to learn more about Rill project files. + +title: "AdBids_model_dashboard_rename" +model: "AdBids_model" +default_time_range: "P4W" +smallest_time_grain: "week" +timeseries: "timestamp" +default_comparison: + mode: time +measures: + - label: Total rows + expression: count(*) + name: total_rows + description: Total number of records present +dimensions: + - name: publisher + label: Publisher + column: publisher + description: "" + - name: domain + label: Domain Name + column: domain + description: "" + `; + await updateCodeEditor(page, docWithPresetDefaultTimeComparison); + await waitForDashboard(page); + // Go to dashboard + await page.getByRole("button", { name: "Go to dashboard" }).click(); + // Comparison is selected + await expect(page.getByText("Comparing by Time")).toBeVisible(); + // Go back to metrics editor + await page.getByRole("button", { name: "Edit metrics" }).click(); + + // Set comparison to dimension + const docWithPresetDefaultDimensionComparison = `# Visit https://docs.rilldata.com/reference/project-files to learn more about Rill project files. + +title: "AdBids_model_dashboard_rename" +model: "AdBids_model" +default_time_range: "P4W" +smallest_time_grain: "week" +timeseries: "timestamp" +default_comparison: + mode: dimension + dimension: publisher +measures: + - label: Total rows + expression: count(*) + name: total_rows + description: Total number of records present +dimensions: + - name: publisher + label: Publisher + column: publisher + description: "" + - name: domain + label: Domain Name + column: domain + description: "" + `; + await updateCodeEditor(page, docWithPresetDefaultDimensionComparison); + await waitForDashboard(page); + // Go to dashboard + await page.getByRole("button", { name: "Go to dashboard" }).click(); + // Comparison is selected + await expect(page.getByText("Comparing by Publisher")).toBeVisible(); + // Go back to metrics editor + await page.getByRole("button", { name: "Edit metrics" }).click(); + + // Set comparison to none + const docWithPresetDefaultNoComparison = `# Visit https://docs.rilldata.com/reference/project-files to learn more about Rill project files. + +title: "AdBids_model_dashboard_rename" +model: "AdBids_model" +default_time_range: "P4W" +smallest_time_grain: "week" +timeseries: "timestamp" +default_comparison: + mode: none +measures: + - label: Total rows + expression: count(*) + name: total_rows + description: Total number of records present +dimensions: + - name: publisher + label: Publisher + column: publisher + description: "" + - name: domain + label: Domain Name + column: domain + description: "" + `; + await updateCodeEditor(page, docWithPresetDefaultNoComparison); + await waitForDashboard(page); + // Go to dashboard + await page.getByRole("button", { name: "Go to dashboard" }).click(); + // No Comparison + await expect(page.getByText("No Comparison")).toBeVisible(); + // Go back to metrics editor + await page.getByRole("button", { name: "Edit metrics" }).click(); +} + // Helper that opens the time range menu, calls your interactions, and then waits until the menu closes async function interactWithTimeRangeMenu( page: Page, diff --git a/web-local/test/ui/utils/sourceHelpers.ts b/web-local/test/ui/utils/sourceHelpers.ts index 227ace232b6..d1ee99487ab 100644 --- a/web-local/test/ui/utils/sourceHelpers.ts +++ b/web-local/test/ui/utils/sourceHelpers.ts @@ -75,7 +75,10 @@ export async function createOrReplaceSource( } catch (err) { await uploadFile(page, file); } - await waitForEntity(page, TestEntityType.Source, name, true); + await Promise.all([ + page.getByText("View this source").click(), + waitForEntity(page, TestEntityType.Source, name, true), + ]); } export async function waitForSource( @@ -83,7 +86,8 @@ export async function waitForSource( name: string, columns: Array ) { - return Promise.all([ + await Promise.all([ + page.getByText("View this source").click(), waitForEntity(page, TestEntityType.Source, name, true), waitForProfiling(page, name, columns), ]);