Merge remote-tracking branch 'origin/main' into pivot-on

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve.
 title: ''
-labels: 'bug'
+labels: 'Type:Bug'
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project.
 title: ''
-labels: 'request'
+labels: 'Type:Feature'
 assignees: ''
 
 ---

.github/workflows/cloud-admin.yml

@@ -1,4 +1,4 @@
-name: Build and Deploy Cloud Admin to Netlify
+name: Build and Deploy Cloud UI to Netlify
 on:
   push:
     tags:
@@ -10,10 +10,17 @@ on:
       - "web-admin/**"
       - "web-common/**"
   workflow_dispatch:
+    inputs:
+      env:
+        description: "Environment"
+        required: true
+        default: "stage"
+        type: choice
+        options:
+        - stage
+        - prod
 env:
   NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
-  NETLIFY_SITE_ID: ${{ fromJSON('["e73ac785-882e-425a-8fc8-5432528bb374","afb26917-7a60-47d9-81dc-c0babbb0d9a9"]')[ startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch') ] }}
-  DOMAIN: ${{ fromJSON('["rilldata.io","rilldata.com"]')[ startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch') ] }}
 
 jobs:
   build:
@@ -27,14 +34,27 @@ jobs:
         with:
           node-version: 16
 
-      - name: Build Cloud Admin
+      - name: Setup Env variables from Inputs
+        env:
+          ENV: ${{ inputs.env }}
+          RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
+        run: |-
+          if [[ ( $GITHUB_EVENT_NAME == "push" && $RELEASE == "true" ) || ( $GITHUB_EVENT_NAME == "workflow_dispatch" && $ENV == "prod" ) ]]; then
+            echo "NETLIFY_SITE_ID=afb26917-7a60-47d9-81dc-c0babbb0d9a9" >> $GITHUB_ENV
+            echo "DOMAIN=rilldata.com" >> $GITHUB_ENV
+          else
+            echo "NETLIFY_SITE_ID=e73ac785-882e-425a-8fc8-5432528bb374" >> $GITHUB_ENV
+            echo "DOMAIN=rilldata.io" >> $GITHUB_ENV
+          fi
+
+      - name: Build Cloud UI
         run: |-
           npm install
           npm run build -w web-admin
         env:
           VITE_RILL_ADMIN_URL: https://admin.${{ env.DOMAIN }}
 
-      - name: Deploy cloud-admin to Netlify
+      - name: Deploy Cloud UI to Netlify
         uses: nwtgck/[email protected]
         with:
           publish-dir: ./web-admin/build

.github/workflows/rill-cloud.yml

@@ -45,9 +45,7 @@ jobs:
           gcloud auth configure-docker
 
           docker build -t gcr.io/rilldata/rill-headless:${GITHUB_SHA} .
-          docker tag gcr.io/rilldata/rill-headless:${GITHUB_SHA} gcr.io/rilldata/rill-headless
           docker push gcr.io/rilldata/rill-headless:${GITHUB_SHA}
-          docker push gcr.io/rilldata/rill-headless
 
           if [ ${RELEASE} == "true" ]; then
             docker tag gcr.io/rilldata/rill-headless:${GITHUB_SHA} gcr.io/rilldata/rill-headless:${GITHUB_REF_NAME}

.github/workflows/web-test.yml

@@ -75,7 +75,7 @@ jobs:
         run: |-
           npx prettier --check "web-common/**/*"
           npx eslint web-common --quiet
-          npx svelte-check --workspace web-common --no-tsconfig --ignore "src/components/(data-graphic|notifications|column-profile|virtualized-table|menu|modal|chip|overlay|floating-element|data-types|BarAndLabel.svelte|button|preview-table/ConnectedPreviewTable.svelte),src/features/models/workspace/inspector,src/**/workspace,src/features/dashboards/(time-series|leaderboard|time-controls),src/layout/navigation/NavigationEntry.svelte"
+          npx svelte-check --threshold error --workspace web-common --no-tsconfig --ignore "src/components/(data-graphic|notifications|column-profile|virtualized-table|menu),src/features/models/workspace/inspector,src/**/workspace,src/features/dashboards/(time-series|time-controls),src/layout/navigation/NavigationEntry.svelte"
 
       - name: Prettier checks and lint for web local
         if: steps.filter.outputs.local == 'true'

.gitignore

@@ -11,7 +11,6 @@ web-local/src/server/duckdb
 web-local/src/server/scratch/
 server/venv/
 tsc-tmp/
-export/
 .vscode
 /temp
 /tmp

admin/deployments.go

@@ -64,7 +64,7 @@ func (s *Service) createDeployment(ctx context.Context, opts *createDeploymentOp
 		olapConfig["cpu"] = strconv.Itoa(alloc.CPU)
 		olapConfig["memory_limit_gb"] = strconv.Itoa(alloc.MemoryGB)
 		olapConfig["storage_limit_bytes"] = strconv.FormatInt(alloc.StorageBytes, 10)
-		embedCatalog = true
+		embedCatalog = false
 	case "duckdb-ext-storage": // duckdb driver having capability to store table as view
 		if opts.ProdOLAPDSN != "" {
 			return nil, fmt.Errorf("passing a DSN is not allowed for driver 'duckdb-ext-storage'")
@@ -79,7 +79,7 @@ func (s *Service) createDeployment(ctx context.Context, opts *createDeploymentOp
 		olapConfig["memory_limit_gb"] = strconv.Itoa(alloc.MemoryGB)
 		olapConfig["storage_limit_bytes"] = strconv.FormatInt(alloc.StorageBytes, 10)
 		olapConfig["external_table_storage"] = strconv.FormatBool(true)
-		embedCatalog = true
+		embedCatalog = false
 	default:
 		olapConfig["dsn"] = opts.ProdOLAPDSN
 		embedCatalog = false

admin/server/reports.go

@@ -429,6 +429,7 @@ func (s *Server) yamlForManagedReport(opts *adminv1.ReportOptions, reportName, o
 	res.Kind = "report"
 	res.Title = opts.Title
 	res.Refresh.Cron = opts.RefreshCron
+	res.Refresh.TimeZone = opts.RefreshTimeZone
 	res.Query.Name = opts.QueryName
 	res.Query.ArgsJSON = opts.QueryArgsJson
 	res.Export.Format = opts.ExportFormat.String()
@@ -468,6 +469,7 @@ func (s *Server) yamlForCommittedReport(opts *adminv1.ReportOptions) ([]byte, er
 	res.Kind = "report"
 	res.Title = opts.Title
 	res.Refresh.Cron = opts.RefreshCron
+	res.Refresh.TimeZone = opts.RefreshTimeZone
 	res.Query.Name = opts.QueryName
 	res.Query.Args = args
 	res.Export.Format = exportFormat
@@ -522,6 +524,7 @@ func recreateReportOptionsFromSpec(spec *runtimev1.ReportSpec) (*adminv1.ReportO
 	opts.Title = spec.Title
 	if spec.RefreshSchedule != nil && spec.RefreshSchedule.Cron != "" {
 		opts.RefreshCron = spec.RefreshSchedule.Cron
+		opts.RefreshTimeZone = spec.RefreshSchedule.TimeZone
 	}
 	opts.QueryName = spec.QueryName
 	opts.QueryArgsJson = spec.QueryArgsJson
@@ -537,7 +540,8 @@ type reportYAML struct {
 	Kind    string `yaml:"kind"`
 	Title   string `yaml:"title"`
 	Refresh struct {
-		Cron string `yaml:"cron"`
+		Cron     string `yaml:"cron"`
+		TimeZone string `yaml:"time_zone"`
 	} `yaml:"refresh"`
 	Query struct {
 		Name     string         `yaml:"name"`

admin/server/server.go

@@ -20,13 +20,15 @@ import (
 	"github.com/rilldata/rill/admin/server/auth"
 	"github.com/rilldata/rill/admin/server/cookies"
 	adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1"
+	"github.com/rilldata/rill/runtime/pkg/activity"
 	"github.com/rilldata/rill/runtime/pkg/graceful"
 	"github.com/rilldata/rill/runtime/pkg/middleware"
 	"github.com/rilldata/rill/runtime/pkg/observability"
 	"github.com/rilldata/rill/runtime/pkg/ratelimit"
 	runtimeauth "github.com/rilldata/rill/runtime/server/auth"
 	"github.com/rs/cors"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -70,11 +72,13 @@ type Server struct {
 	issuer        *runtimeauth.Issuer
 	urls          *externalURLs
 	limiter       ratelimit.Limiter
+	// Activity specifically for events from UI
+	uiActivity activity.Client
 }
 
 var _ adminv1.AdminServiceServer = (*Server)(nil)
 
-func New(logger *zap.Logger, adm *admin.Service, issuer *runtimeauth.Issuer, limiter ratelimit.Limiter, opts *Options) (*Server, error) {
+func New(logger *zap.Logger, adm *admin.Service, issuer *runtimeauth.Issuer, limiter ratelimit.Limiter, uiActivity activity.Client, opts *Options) (*Server, error) {
 	externalURL, err := url.Parse(opts.ExternalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse external URL: %w", err)
@@ -109,6 +113,7 @@ func New(logger *zap.Logger, adm *admin.Service, issuer *runtimeauth.Issuer, lim
 		issuer:        issuer,
 		urls:          newURLRegistry(opts),
 		limiter:       limiter,
+		uiActivity:    uiActivity,
 	}, nil
 }
 
@@ -295,6 +300,15 @@ func (s *Server) Ping(ctx context.Context, req *adminv1.PingRequest) (*adminv1.P
 	return resp, nil
 }
 
+func (s *Server) Telemetry(ctx context.Context, req *adminv1.TelemetryRequest) (*adminv1.TelemetryResponse, error) {
+	dims := make([]attribute.KeyValue, 0)
+	for k, v := range req.Event {
+		dims = append(dims, attribute.String(k, v))
+	}
+	s.uiActivity.Emit(ctx, "cloud-ui-telemetry", 1, dims...)
+	return &adminv1.TelemetryResponse{}, nil
+}
+
 func timeoutSelector(fullMethodName string) time.Duration {
 	return time.Minute
 }
@@ -395,7 +409,9 @@ func newURLRegistry(opts *Options) *externalURLs {
 }
 
 func (u *externalURLs) reportOpen(org, project, projectSubpath string) string {
-	return urlutil.MustJoinURL(u.frontend, org, project, projectSubpath)
+	res := urlutil.MustJoinURL(u.frontend, org, project)
+	res += projectSubpath // Need to do an unsafe concat to provide flexibility, e.g. to avoid escaping '?'
+	return res
 }
 
 func (u *externalURLs) reportExport(org, project, report string) string {

cli/cmd/admin/start.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rilldata/rill/admin/server"
 	"github.com/rilldata/rill/admin/worker"
 	"github.com/rilldata/rill/cli/pkg/config"
+	"github.com/rilldata/rill/runtime/pkg/activity"
 	"github.com/rilldata/rill/runtime/pkg/email"
 	"github.com/rilldata/rill/runtime/pkg/graceful"
 	"github.com/rilldata/rill/runtime/pkg/observability"
@@ -33,39 +34,44 @@ import (
 // Env var keys must be prefixed with RILL_ADMIN_ and are converted from snake_case to CamelCase.
 // For example RILL_ADMIN_HTTP_PORT is mapped to Config.HTTPPort.
 type Config struct {
-	DatabaseDriver         string                 `default:"postgres" split_words:"true"`
-	DatabaseURL            string                 `split_words:"true"`
-	Jobs                   []string               `split_words:"true"`
-	HTTPPort               int                    `default:"8080" split_words:"true"`
-	GRPCPort               int                    `default:"9090" split_words:"true"`
-	LogLevel               zapcore.Level          `default:"info" split_words:"true"`
-	MetricsExporter        observability.Exporter `default:"prometheus" split_words:"true"`
-	TracesExporter         observability.Exporter `default:"" split_words:"true"`
-	ExternalURL            string                 `default:"http://localhost:8080" split_words:"true"`
-	ExternalGRPCURL        string                 `envconfig:"external_grpc_url"`
-	FrontendURL            string                 `default:"http://localhost:3000" split_words:"true"`
-	SessionKeyPairs        []string               `split_words:"true"`
-	AllowedOrigins         []string               `default:"*" split_words:"true"`
-	AuthDomain             string                 `split_words:"true"`
-	AuthClientID           string                 `split_words:"true"`
-	AuthClientSecret       string                 `split_words:"true"`
-	GithubAppID            int64                  `split_words:"true"`
-	GithubAppName          string                 `split_words:"true"`
-	GithubAppPrivateKey    string                 `split_words:"true"`
-	GithubAppWebhookSecret string                 `split_words:"true"`
-	GithubClientID         string                 `split_words:"true"`
-	GithubClientSecret     string                 `split_words:"true"`
-	ProvisionerSpec        string                 `split_words:"true"`
-	SigningJWKS            string                 `split_words:"true"`
-	SigningKeyID           string                 `split_words:"true"`
-	EmailSMTPHost          string                 `split_words:"true"`
-	EmailSMTPPort          int                    `split_words:"true"`
-	EmailSMTPUsername      string                 `split_words:"true"`
-	EmailSMTPPassword      string                 `split_words:"true"`
-	EmailSenderEmail       string                 `split_words:"true"`
-	EmailSenderName        string                 `split_words:"true"`
-	EmailBCC               string                 `split_words:"true"`
-	RedisURL               string                 `default:"" split_words:"true"`
+	DatabaseDriver           string                 `default:"postgres" split_words:"true"`
+	DatabaseURL              string                 `split_words:"true"`
+	Jobs                     []string               `split_words:"true"`
+	HTTPPort                 int                    `default:"8080" split_words:"true"`
+	GRPCPort                 int                    `default:"9090" split_words:"true"`
+	LogLevel                 zapcore.Level          `default:"info" split_words:"true"`
+	MetricsExporter          observability.Exporter `default:"prometheus" split_words:"true"`
+	TracesExporter           observability.Exporter `default:"" split_words:"true"`
+	ExternalURL              string                 `default:"http://localhost:8080" split_words:"true"`
+	ExternalGRPCURL          string                 `envconfig:"external_grpc_url"`
+	FrontendURL              string                 `default:"http://localhost:3000" split_words:"true"`
+	SessionKeyPairs          []string               `split_words:"true"`
+	AllowedOrigins           []string               `default:"*" split_words:"true"`
+	AuthDomain               string                 `split_words:"true"`
+	AuthClientID             string                 `split_words:"true"`
+	AuthClientSecret         string                 `split_words:"true"`
+	GithubAppID              int64                  `split_words:"true"`
+	GithubAppName            string                 `split_words:"true"`
+	GithubAppPrivateKey      string                 `split_words:"true"`
+	GithubAppWebhookSecret   string                 `split_words:"true"`
+	GithubClientID           string                 `split_words:"true"`
+	GithubClientSecret       string                 `split_words:"true"`
+	ProvisionerSpec          string                 `split_words:"true"`
+	SigningJWKS              string                 `split_words:"true"`
+	SigningKeyID             string                 `split_words:"true"`
+	EmailSMTPHost            string                 `split_words:"true"`
+	EmailSMTPPort            int                    `split_words:"true"`
+	EmailSMTPUsername        string                 `split_words:"true"`
+	EmailSMTPPassword        string                 `split_words:"true"`
+	EmailSenderEmail         string                 `split_words:"true"`
+	EmailSenderName          string                 `split_words:"true"`
+	EmailBCC                 string                 `split_words:"true"`
+	RedisURL                 string                 `default:"" split_words:"true"`
+	ActivitySinkType         string                 `default:"" split_words:"true"`
+	ActivitySinkPeriodMs     int                    `default:"1000" split_words:"true"`
+	ActivityMaxBufferSize    int                    `default:"1000" split_words:"true"`
+	ActivitySinkKafkaBrokers string                 `default:"" split_words:"true"`
+	ActivityUISinkKafkaTopic string                 `default:"" split_words:"true"`
 }
 
 // StartCmd starts an admin server. It only allows configuration using environment variables.
@@ -207,6 +213,15 @@ func StartCmd(cliCfg *config.Config) *cobra.Command {
 			runWorker := len(args) == 0 || args[0] == "worker"
 			runJobs := len(args) == 0 || args[0] == "jobs"
 
+			uiActivityClient := activity.NewClientFromConf(
+				conf.ActivitySinkType,
+				conf.ActivitySinkPeriodMs,
+				conf.ActivityMaxBufferSize,
+				conf.ActivitySinkKafkaBrokers,
+				conf.ActivityUISinkKafkaTopic,
+				logger,
+			)
+
 			// Init and run server
 			if runServer {
 				var limiter ratelimit.Limiter
@@ -219,7 +234,7 @@ func StartCmd(cliCfg *config.Config) *cobra.Command {
 					}
 					limiter = ratelimit.NewRedis(redis.NewClient(opts))
 				}
-				srv, err := server.New(logger, adm, issuer, limiter, &server.Options{
+				srv, err := server.New(logger, adm, issuer, limiter, uiActivityClient, &server.Options{
 					HTTPPort:               conf.HTTPPort,
 					GRPCPort:               conf.GRPCPort,
 					ExternalURL:            conf.ExternalURL,

cli/cmd/deploy/deploy.go

@@ -492,7 +492,7 @@ func variablesFlow(ctx context.Context, projectPath, projectName string) {
 	if err != nil {
 		return
 	}
-	parser, err := rillv1.Parse(ctx, repo, instanceID, "", nil)
+	parser, err := rillv1.Parse(ctx, repo, instanceID, "duckdb", []string{"duckdb"})
 	if err != nil {
 		return
 	}

cli/cmd/env/configure.go

@@ -14,11 +14,8 @@ import (
 	adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1"
 	"github.com/rilldata/rill/runtime/compilers/rillv1"
 	"github.com/rilldata/rill/runtime/compilers/rillv1beta"
-	"github.com/rilldata/rill/runtime/drivers"
-	"github.com/rilldata/rill/runtime/pkg/activity"
 	"github.com/rilldata/rill/runtime/pkg/fileutil"
 	"github.com/spf13/cobra"
-	"go.uber.org/zap"
 )
 
 func ConfigureCmd(cfg *config.Config) *cobra.Command {
@@ -143,15 +140,11 @@ func ConfigureCmd(cfg *config.Config) *cobra.Command {
 }
 
 func VariablesFlow(ctx context.Context, projectPath string, tel *telemetry.Telemetry) (map[string]string, error) {
-	// Create an ad-hoc file repo for projectPath
-	instanceID := "default"
-	repoHandle, err := drivers.Open("file", map[string]any{"dsn": projectPath}, false, activity.NewNoopClient(), zap.NewNop())
+	// Parse the project's connectors
+	repo, instanceID, err := cmdutil.RepoForProjectPath(projectPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open project repo: %w", err)
+		return nil, err
 	}
-	repo, _ := repoHandle.AsRepoStore(instanceID)
-
-	// Parse the project's connectors
 	parser, err := rillv1.Parse(ctx, repo, instanceID, "duckdb", []string{"duckdb"})
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse project: %w", err)