Merge branch 'main' into feat/canvas-2.0

.gitignore

@@ -25,6 +25,7 @@ tsc-tmp/
 !/runtime/drivers/duckdb/extensions/embed/**/.gitkeep
 /dev-project*
 /dev-cloud-state*
+/e2e-cloud-state*
 /playwright-report
 
 # data files

admin/billing.go

@@ -215,7 +215,7 @@ func (s *Service) StartTrial(ctx context.Context, org *database.Organization) (*
 	if err != nil {
 		if !errors.Is(err, billing.ErrNotFound) {
 			if errors.Is(err, billing.ErrCustomerIDRequired) {
-				return nil, nil, fmt.Errorf("org billing not initialized yet, retry")
+				return nil, nil, fmt.Errorf("org billing not initialized yet")
 			}
 			return nil, nil, fmt.Errorf("failed to get subscriptions for customer: %w", err)
 		}

admin/jobs/river/org_jobs.go

@@ -105,26 +105,27 @@ func (w *StartTrialWorker) Work(ctx context.Context, job *river.Job[StartTrialAr
 			// org got deleted, ignore
 			return nil
 		}
-		w.logger.Error("failed to find organization", zap.String("org_id", job.Args.OrgID), zap.Error(err))
 		return err
 	}
 
-	org, sub, err := w.admin.StartTrial(ctx, org)
+	trialOrg, sub, err := w.admin.StartTrial(ctx, org)
 	if err != nil {
-		w.logger.Error("failed to start trial for organization", zap.String("org_id", job.Args.OrgID), zap.Error(err))
+		if job.Attempt >= job.MaxAttempts {
+			w.logger.Error("failed to start trial for organization", zap.String("org_name", org.Name), zap.String("org_id", org.ID), zap.Error(err))
+		}
 		return err
 	}
 
 	// send trial started email
 	err = w.admin.Email.SendTrialStarted(&email.TrialStarted{
-		ToEmail:      org.BillingEmail,
-		ToName:       org.Name,
-		OrgName:      org.Name,
+		ToEmail:      trialOrg.BillingEmail,
+		ToName:       trialOrg.Name,
+		OrgName:      trialOrg.Name,
 		FrontendURL:  w.admin.URLs.Frontend(),
 		TrialEndDate: sub.TrialEndDate,
 	})
 	if err != nil {
-		w.logger.Error("failed to send trial started email", zap.String("org_name", org.Name), zap.String("org_id", org.ID), zap.String("billing_email", org.BillingEmail), zap.Error(err))
+		w.logger.Error("failed to send trial started email", zap.String("org_name", trialOrg.Name), zap.String("org_id", trialOrg.ID), zap.String("billing_email", trialOrg.BillingEmail), zap.Error(err))
 	}
 
 	return nil

admin/jobs/river/river.go

@@ -378,9 +378,11 @@ type ErrorHandler struct {
 }
 
 func (h *ErrorHandler) HandleError(ctx context.Context, job *rivertype.JobRow, err error) *river.ErrorHandlerResult {
-	var args string
-	_ = json.Unmarshal(job.EncodedArgs, &args) // ignore parse errors
-	h.logger.Error("Job errored", zap.Int64("job_id", job.ID), zap.Int("num_attempt", job.Attempt), zap.String("kind", job.Kind), zap.String("args", args), zap.Error(err))
+	if job.Attempt >= job.MaxAttempts {
+		var args string
+		_ = json.Unmarshal(job.EncodedArgs, &args) // ignore parse errors
+		h.logger.Error("Job failed, max attempts reached", zap.Int64("job_id", job.ID), zap.Int("num_attempt", job.Attempt), zap.Int("max_attempts", job.MaxAttempts), zap.String("kind", job.Kind), zap.String("args", args), zap.Error(err))
+	}
 	return nil
 }
 

admin/server/billing.go

@@ -203,7 +203,7 @@ func (s *Server) UpdateBillingSubscription(ctx context.Context, req *adminv1.Upd
 				OrgName:          org.Name,
 				FrontendURL:      s.admin.URLs.Frontend(),
 				PlanName:         plan.DisplayName,
-				BillingStartDate: sub.CurrentBillingCycleEndDate.AddDate(0, 0, 1),
+				BillingStartDate: sub.CurrentBillingCycleEndDate,
 			})
 		} else {
 			err = s.admin.Email.SendPlanUpdate(&email.PlanUpdate{
@@ -405,7 +405,7 @@ func (s *Server) RenewBillingSubscription(ctx context.Context, req *adminv1.Rene
 			OrgName:          org.Name,
 			FrontendURL:      s.admin.URLs.Frontend(),
 			PlanName:         sub.Plan.DisplayName,
-			BillingStartDate: sub.CurrentBillingCycleEndDate.AddDate(0, 0, 1),
+			BillingStartDate: sub.CurrentBillingCycleEndDate,
 		})
 	} else {
 		err = s.admin.Email.SendSubscriptionRenewed(&email.SubscriptionRenewed{

admin/server/projects.go

@@ -1539,18 +1539,24 @@ func (s *Server) RedeployProject(ctx context.Context, req *adminv1.RedeployProje
 		attribute.String("args.project", req.Project),
 	)
 
-	org, err := s.admin.DB.FindOrganizationByName(ctx, req.Organization)
+	proj, err := s.admin.DB.FindProjectByName(ctx, req.Organization, req.Project)
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
 
-	// check if org has blocking billing errors
-	err = s.admin.CheckBlockingBillingErrors(ctx, org.ID)
+	claims := auth.GetClaims(ctx)
+	forceAccess := req.SuperuserForceAccess && claims.Superuser(ctx)
+	if !claims.ProjectPermissions(ctx, proj.OrganizationID, proj.ID).ManageProd && !forceAccess {
+		return nil, status.Error(codes.PermissionDenied, "does not have permission to manage deployment")
+	}
+
+	org, err := s.admin.DB.FindOrganizationByName(ctx, req.Organization)
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
 
-	proj, err := s.admin.DB.FindProjectByName(ctx, req.Organization, req.Project)
+	// check if org has blocking billing errors
+	err = s.admin.CheckBlockingBillingErrors(ctx, org.ID)
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
@@ -1563,11 +1569,6 @@ func (s *Server) RedeployProject(ctx context.Context, req *adminv1.RedeployProje
 		}
 	}
 
-	claims := auth.GetClaims(ctx)
-	if !claims.ProjectPermissions(ctx, proj.OrganizationID, proj.ID).ManageProd {
-		return nil, status.Error(codes.PermissionDenied, "does not have permission to manage deployment")
-	}
-
 	_, err = s.admin.RedeployProject(ctx, proj, depl)
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())

admin/worker/billing_reporter.go

@@ -127,7 +127,7 @@ func (w *Worker) reportUsage(ctx context.Context) error {
 	}
 
 	if len(reportedOrgs) == 0 {
-		w.logger.Warn("skipping usage reporting: no usage data available", zap.Time("start_time", startTime), zap.Time("end_time", endTime))
+		w.logger.Named("billing").Warn("skipping usage reporting: no usage data available", zap.Time("start_time", startTime), zap.Time("end_time", endTime))
 		return nil
 	}
 

cli/cmd/devtool/data/clickhouse-config.xml

@@ -0,0 +1,13 @@
+<clickhouse replace="true">
+    <logger>
+        <level>information</level>
+        <console>true</console>
+    </logger>
+    <display_name>rill devtool clickhouse</display_name>
+    <tcp_port>9000</tcp_port>
+    <http_port>8123</http_port>
+    <timezone>UTC</timezone>
+    <access_control_improvements>
+        <select_from_information_schema_requires_grant>true</select_from_information_schema_requires_grant>
+    </access_control_improvements>
+</clickhouse>

cli/cmd/devtool/data/clickhouse-users.xml

@@ -0,0 +1,37 @@
+<clickhouse replace="true">
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>in_order</load_balancing>
+            <log_queries>1</log_queries>
+            <date_time_input_format>best_effort</date_time_input_format>
+        </default>
+    </profiles>
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+    <users>
+        <default>
+            <password>default</password>
+            <profile>default</profile>
+            <quota>default</quota>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <access_management>1</access_management>
+            <named_collection_control>1</named_collection_control>
+            <show_named_collections>1</show_named_collections>
+            <show_named_collections_secrets>1</show_named_collections_secrets>
+        </default>
+    </users>
+</clickhouse>

cli/cmd/devtool/data/cloud-deps.docker-compose.yml

@@ -8,7 +8,7 @@ services:
     ports:
       - '5432:5432'
     volumes:
-      - ../../../../dev-cloud-state/postgres:/var/lib/postgresql/data
+      - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/postgres:/var/lib/postgresql/data
   e2e-postgres:
     image: postgres:17
     restart: always
@@ -63,4 +63,13 @@ services:
     image: stripe/stripe-cli:v1.21.11
     command: listen --forward-to http://host.docker.internal:8080/payment/webhook --config /etc/stripe-config.toml
     volumes:
-      - ../../../../dev-cloud-state/stripe-config.toml:/etc/stripe-config.toml
+      - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/stripe-config.toml:/etc/stripe-config.toml
+  clickhouse:
+    image: 'clickhouse/clickhouse-server:24.11.1.2557'
+    volumes:
+      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/config.xml
+      - ./clickhouse-users.xml:/etc/clickhouse-server/users.d/users.xml
+      - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/clickhouse:/var/lib/clickhouse
+    ports:
+      - '9000:9000' # Native port
+      - '8123:8123' # HTTP port

cli/cmd/devtool/start.go

@@ -30,7 +30,6 @@ import (
 const (
 	minGoVersion   = "1.23"
 	minNodeVersion = "18"
-	stateDirCloud  = "dev-cloud-state"
 	stateDirLocal  = "dev-project"
 	rillGithubURL  = "https://github.com/rilldata/rill"
 )
@@ -225,19 +224,6 @@ func (s *servicesCfg) parse() error {
 type cloud struct{}
 
 func (s cloud) start(ctx context.Context, ch *cmdutil.Helper, verbose, reset, refreshDotenv bool, preset string, services *servicesCfg) error {
-	if reset {
-		err := s.resetState(ctx)
-		if err != nil {
-			return fmt.Errorf("reset cloud deps: %w", err)
-		}
-		logInfo.Printf("Reset cloud dependencies\n")
-	}
-
-	err := os.MkdirAll(stateDirCloud, os.ModePerm)
-	if err != nil {
-		return fmt.Errorf("failed to create state dir %q: %w", stateDirCloud, err)
-	}
-
 	if refreshDotenv {
 		err := downloadDotenv(ctx, preset)
 		if err != nil {
@@ -247,7 +233,7 @@ func (s cloud) start(ctx context.Context, ch *cmdutil.Helper, verbose, reset, re
 	}
 
 	// Validate the .env file is well-formed.
-	err = checkDotenv()
+	err := checkDotenv()
 	if err != nil {
 		return err
 	}
@@ -256,8 +242,22 @@ func (s cloud) start(ctx context.Context, ch *cmdutil.Helper, verbose, reset, re
 		return fmt.Errorf("error parsing .env: %w", err)
 	}
 
+	if reset {
+		err := s.resetState(ctx)
+		if err != nil {
+			return fmt.Errorf("reset cloud deps: %w", err)
+		}
+		logInfo.Printf("Reset cloud dependencies\n")
+	}
+
 	g, ctx := errgroup.WithContext(ctx)
 
+	err = os.MkdirAll(stateDirectory(), os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("failed to create state dir %q: %w", stateDirectory(), err)
+	}
+	logInfo.Printf("State directory is %q\n", stateDirectory())
+
 	if services.deps {
 		g.Go(func() error { return s.runDeps(ctx, verbose) })
 	}
@@ -384,21 +384,22 @@ func (s cloud) resetState(ctx context.Context) (err error) {
 		}
 	}()
 
-	_ = os.RemoveAll(stateDirCloud)
-	return newCmd(ctx, "docker", "compose", "-f", "cli/cmd/devtool/data/cloud-deps.docker-compose.yml", "down", "--volumes").Run()
+	_ = os.RemoveAll(stateDirectory())
+
+	return newCmd(ctx, "docker", "compose", "--env-file", ".env", "-f", "cli/cmd/devtool/data/cloud-deps.docker-compose.yml", "down", "--volumes").Run()
 }
 
 func (s cloud) runDeps(ctx context.Context, verbose bool) error {
 	composeFile := "cli/cmd/devtool/data/cloud-deps.docker-compose.yml"
-	logInfo.Printf("Starting dependencies: docker compose -f %s up\n", composeFile)
+	logInfo.Printf("Starting dependencies: docker compose --env-file .env -f %s up\n", composeFile)
 	defer logInfo.Printf("Stopped dependencies\n")
 
 	err := prepareStripeConfig()
 	if err != nil {
 		return fmt.Errorf("failed to prepare stripe config: %w", err)
 	}
 
-	cmd := newCmd(ctx, "docker", "compose", "-f", composeFile, "up")
+	cmd := newCmd(ctx, "docker", "compose", "--env-file", ".env", "-f", composeFile, "up")
 	if verbose {
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stdout
@@ -722,7 +723,7 @@ func (s local) awaitUI(ctx context.Context) error {
 
 func prepareStripeConfig() error {
 	templateFile := "cli/cmd/devtool/data/stripe-config.template"
-	outputFile := filepath.Join(stateDirCloud, "stripe-config.toml")
+	outputFile := filepath.Join(stateDirectory(), "stripe-config.toml")
 
 	apiKey := lookupDotenv("RILL_DEVTOOL_STRIPE_CLI_API_KEY")
 	if apiKey == "" {
@@ -782,3 +783,13 @@ func lookupDotenv(key string) string {
 	}
 	return env[key]
 }
+
+// stateDirectory returns the directory where the devtool's state is stored.
+// Deleting this directory will reset the state of the local development environment.
+func stateDirectory() string {
+	dir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
+	if dir == "" {
+		dir = "dev-cloud-state"
+	}
+	return dir
+}

cli/cmd/project/connect_github.go

@@ -91,13 +91,18 @@ func ConnectGithubFlow(ctx context.Context, ch *cmdutil.Helper, opts *DeployOpts
 		}
 	}
 
-	// If the Git path is local, we'll do some extra steps to infer the githubURL.
-	localGitPath, localProjectPath, err := ValidateLocalProject(ch, opts.GitPath, opts.SubPath)
-	if err != nil {
-		if errors.Is(err, ErrInvalidProject) {
-			return nil
+	var localGitPath string
+	var localProjectPath string
+	var err error
+	if isLocalGitPath {
+		// If the Git path is local, we'll do some extra steps to infer the githubURL.
+		localGitPath, localProjectPath, err = ValidateLocalProject(ch, opts.GitPath, opts.SubPath)
+		if err != nil {
+			if errors.Is(err, ErrInvalidProject) {
+				return nil
+			}
+			return err
 		}
-		return err
 	}
 
 	if ch.Org != "" {

cli/cmd/sudo/project/reset.go

@@ -35,8 +35,9 @@ func ResetCmd(ch *cmdutil.Helper) *cobra.Command {
 			}
 
 			_, err = client.RedeployProject(cmd.Context(), &adminv1.RedeployProjectRequest{
-				Organization: org,
-				Project:      project,
+				Organization:         org,
+				Project:              project,
+				SuperuserForceAccess: true,
 			})
 			if err != nil {
 				return err

docs/docs/contact.md

@@ -32,4 +32,6 @@ For individuals that prefer a more traditional email medium, you can open a supp
 
 ### In-app chat
 
-For logged-in users, Rill provides an ability to ask a question and/or report an issue to the Rill Support team from within the application directly via an embedded in-app chat widget. More details to come soon!
+For logged-in users, Rill provides an ability to ask a question and/or report an issue to the Rill Support team from within the application directly via an embedded in-app chat widget. Select your user icon in the top right corner and select "Contact Rill support" and a chat widget will appear on your screen!
+
+![rill-chat](/img/contact/rill-developer-chat.png)