Merge remote-tracking branch 'origin/main' into 3299-runtime-deprecat…

…e-runtimeserverexportgo-migrate-frontend-to-use-the-new-export-api

.github/workflows/rill-cloud.yml

@@ -45,9 +45,7 @@ jobs:
           gcloud auth configure-docker
 
           docker build -t gcr.io/rilldata/rill-headless:${GITHUB_SHA} .
-          docker tag gcr.io/rilldata/rill-headless:${GITHUB_SHA} gcr.io/rilldata/rill-headless
           docker push gcr.io/rilldata/rill-headless:${GITHUB_SHA}
-          docker push gcr.io/rilldata/rill-headless
 
           if [ ${RELEASE} == "true" ]; then
             docker tag gcr.io/rilldata/rill-headless:${GITHUB_SHA} gcr.io/rilldata/rill-headless:${GITHUB_REF_NAME}

admin/deployments.go

@@ -64,7 +64,7 @@ func (s *Service) createDeployment(ctx context.Context, opts *createDeploymentOp
 		olapConfig["cpu"] = strconv.Itoa(alloc.CPU)
 		olapConfig["memory_limit_gb"] = strconv.Itoa(alloc.MemoryGB)
 		olapConfig["storage_limit_bytes"] = strconv.FormatInt(alloc.StorageBytes, 10)
-		embedCatalog = true
+		embedCatalog = false
 	case "duckdb-ext-storage": // duckdb driver having capability to store table as view
 		if opts.ProdOLAPDSN != "" {
 			return nil, fmt.Errorf("passing a DSN is not allowed for driver 'duckdb-ext-storage'")
@@ -79,7 +79,7 @@ func (s *Service) createDeployment(ctx context.Context, opts *createDeploymentOp
 		olapConfig["memory_limit_gb"] = strconv.Itoa(alloc.MemoryGB)
 		olapConfig["storage_limit_bytes"] = strconv.FormatInt(alloc.StorageBytes, 10)
 		olapConfig["external_table_storage"] = strconv.FormatBool(true)
-		embedCatalog = true
+		embedCatalog = false
 	default:
 		olapConfig["dsn"] = opts.ProdOLAPDSN
 		embedCatalog = false

cli/cmd/project/project.go

@@ -27,6 +27,8 @@ func ProjectCmd(cfg *config.Config) *cobra.Command {
 	projectCmd.AddCommand(DeleteCmd(cfg))
 	projectCmd.AddCommand(ListCmd(cfg))
 	projectCmd.AddCommand(ReconcileCmd(cfg))
+	projectCmd.AddCommand(RefreshCmd(cfg))
+	projectCmd.AddCommand(ResetCmd(cfg))
 	projectCmd.AddCommand(JwtCmd(cfg))
 	projectCmd.AddCommand(RenameCmd(cfg))
 	projectCmd.AddCommand(LogsCmd(cfg))

cli/cmd/project/reconcile.go

@@ -11,13 +11,14 @@ import (
 
 func ReconcileCmd(cfg *config.Config) *cobra.Command {
 	var project, path string
-	var refresh, reset bool
+	var refresh, reset, force bool
 	var refreshSources []string
 
 	reconcileCmd := &cobra.Command{
 		Use:               "reconcile [<project-name>]",
 		Args:              cobra.MaximumNArgs(1),
 		Short:             "Send trigger to deployment",
+		Hidden:            true,
 		PersistentPreRunE: cmdutil.CheckChain(cmdutil.CheckAuth(cfg), cmdutil.CheckOrganization(cfg)),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
@@ -49,6 +50,13 @@ func ReconcileCmd(cfg *config.Config) *cobra.Command {
 			}
 
 			if reset || resp.ProdDeployment == nil {
+				if !force {
+					msg := "This will create a new deployment, causing downtime as data sources are reloaded from scratch. If you just need to refresh data, use `rill project refresh`. Do you want to continue?"
+					if !cmdutil.ConfirmPrompt(msg, "", false) {
+						return nil
+					}
+				}
+
 				_, err = client.TriggerRedeploy(ctx, &adminv1.TriggerRedeployRequest{Organization: cfg.Org, Project: project})
 				if err != nil {
 					return err
@@ -85,6 +93,7 @@ func ReconcileCmd(cfg *config.Config) *cobra.Command {
 	reconcileCmd.Flags().BoolVar(&refresh, "refresh", false, "Refresh all sources")
 	reconcileCmd.Flags().StringSliceVar(&refreshSources, "refresh-source", nil, "Refresh specific source(s)")
 	reconcileCmd.Flags().BoolVar(&reset, "reset", false, "Reset and redeploy the project from scratch")
+	reconcileCmd.Flags().BoolVar(&force, "force", false, "Force the operation")
 
 	reconcileCmd.MarkFlagsMutuallyExclusive("reset", "refresh")
 	reconcileCmd.MarkFlagsMutuallyExclusive("reset", "refresh-source")

cli/cmd/project/refresh.go

@@ -0,0 +1,67 @@
+package project
+
+import (
+	"fmt"
+
+	"github.com/rilldata/rill/cli/pkg/cmdutil"
+	"github.com/rilldata/rill/cli/pkg/config"
+	adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1"
+	"github.com/spf13/cobra"
+)
+
+func RefreshCmd(cfg *config.Config) *cobra.Command {
+	var project, path string
+	var source []string
+
+	refreshCmd := &cobra.Command{
+		Use:               "refresh [<project-name>]",
+		Args:              cobra.MaximumNArgs(1),
+		Short:             "Refresh project",
+		PersistentPreRunE: cmdutil.CheckChain(cmdutil.CheckAuth(cfg), cmdutil.CheckOrganization(cfg)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+
+			client, err := cmdutil.Client(cfg)
+			if err != nil {
+				return err
+			}
+			defer client.Close()
+
+			if len(args) > 0 {
+				project = args[0]
+			}
+
+			if !cmd.Flags().Changed("project") && len(args) == 0 && cfg.Interactive {
+				var err error
+				project, err = inferProjectName(ctx, client, cfg.Org, path)
+				if err != nil {
+					return err
+				}
+			}
+
+			resp, err := client.GetProject(ctx, &adminv1.GetProjectRequest{
+				OrganizationName: cfg.Org,
+				Name:             project,
+			})
+			if err != nil {
+				return err
+			}
+
+			_, err = client.TriggerRefreshSources(ctx, &adminv1.TriggerRefreshSourcesRequest{DeploymentId: resp.ProdDeployment.Id, Sources: source})
+			if err != nil {
+				return fmt.Errorf("failed to trigger refresh: %w", err)
+			}
+
+			fmt.Printf("Triggered refresh. To see status, run `rill project status --project %s`.\n", project)
+
+			return nil
+		},
+	}
+
+	refreshCmd.Flags().SortFlags = false
+	refreshCmd.Flags().StringVar(&project, "project", "", "Project name")
+	refreshCmd.Flags().StringVar(&path, "path", ".", "Project directory")
+	refreshCmd.Flags().StringSliceVar(&source, "source", nil, "Refresh specific source(s)")
+
+	return refreshCmd
+}

cli/cmd/project/reset.go

@@ -0,0 +1,65 @@
+package project
+
+import (
+	"fmt"
+
+	"github.com/rilldata/rill/cli/pkg/cmdutil"
+	"github.com/rilldata/rill/cli/pkg/config"
+	adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1"
+	"github.com/spf13/cobra"
+)
+
+func ResetCmd(cfg *config.Config) *cobra.Command {
+	var project, path string
+	var force bool
+
+	resetCmd := &cobra.Command{
+		Use:               "reset [<project-name>]",
+		Args:              cobra.MaximumNArgs(1),
+		Short:             "Reset project",
+		PersistentPreRunE: cmdutil.CheckChain(cmdutil.CheckAuth(cfg), cmdutil.CheckOrganization(cfg)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+
+			client, err := cmdutil.Client(cfg)
+			if err != nil {
+				return err
+			}
+			defer client.Close()
+
+			if len(args) > 0 {
+				project = args[0]
+			}
+
+			if !cmd.Flags().Changed("project") && len(args) == 0 && cfg.Interactive {
+				var err error
+				project, err = inferProjectName(ctx, client, cfg.Org, path)
+				if err != nil {
+					return err
+				}
+			}
+
+			if !force {
+				msg := "This will create a new deployment, causing downtime as data sources are reloaded from scratch. If you just need to refresh data, use `rill project refresh`. Do you want to continue?"
+				if !cmdutil.ConfirmPrompt(msg, "", false) {
+					return nil
+				}
+			}
+
+			_, err = client.TriggerRedeploy(ctx, &adminv1.TriggerRedeployRequest{Organization: cfg.Org, Project: project})
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("Triggered project reset. To see status, run `rill project status --project %s`.\n", project)
+
+			return nil
+		},
+	}
+
+	resetCmd.Flags().SortFlags = false
+	resetCmd.Flags().StringVar(&project, "project", "", "Project name")
+	resetCmd.Flags().StringVar(&path, "path", ".", "Project directory")
+	resetCmd.Flags().BoolVar(&force, "force", false, "Force reset even if project is already deployed")
+	return resetCmd
+}

cli/pkg/local/app.go

@@ -194,15 +194,6 @@ func NewApp(ctx context.Context, ver config.Version, verbose, strict, reset bool
 		activity:              client,
 	}
 
-	// Wait for the initial reconcile
-	if isInit {
-		err = app.AwaitInitialReconcile(strict)
-		if err != nil {
-			app.Close()
-			return nil, fmt.Errorf("reconcile project: %w", err)
-		}
-	}
-
 	return app, nil
 }
 
@@ -226,80 +217,6 @@ func (a *App) Close() error {
 	return nil
 }
 
-func (a *App) AwaitInitialReconcile(strict bool) (err error) {
-	defer func() {
-		if a.Context.Err() != nil {
-			a.Logger.Errorf("Hydration canceled")
-			err = nil
-		}
-	}()
-
-	controller, err := a.Runtime.Controller(a.Context, a.Instance.ID)
-	if err != nil {
-		return err
-	}
-
-	// We need to do some extra work to ensure we don't return until all resources have been reconciled.
-	// We can't call WaitUntilIdle until the parser has initially parsed and created the resources for the project.
-	// We know the global project parser is created immediately, and should only be IDLE initially or if a fatal error occurs with the watcher.
-	// So we poll for it's state to transition to Watching.
-	start := time.Now()
-	for {
-		if a.Context.Err() != nil {
-			return nil
-		}
-
-		if time.Since(start) >= 5*time.Second {
-			// Starting the watcher should take just a few ms. This is just meant to serve as an extra safety net in case something goes wrong.
-			return fmt.Errorf("timed out waiting for project parser to start watching")
-		}
-
-		r, err := controller.Get(a.Context, runtime.GlobalProjectParserName, false)
-		if err != nil {
-			return fmt.Errorf("could not find project parser: %w", err)
-		}
-
-		if r.Meta.ReconcileStatus == runtimev1.ReconcileStatus_RECONCILE_STATUS_IDLE && r.Meta.ReconcileError != "" {
-			return fmt.Errorf("parser failed: %s", r.Meta.ReconcileError)
-		}
-
-		if r.GetProjectParser().State.Watching {
-			break
-		}
-
-		time.Sleep(100 * time.Millisecond)
-	}
-
-	err = a.Runtime.WaitUntilIdle(a.Context, a.Instance.ID, true)
-	if err != nil {
-		return err
-	}
-
-	rs, err := controller.List(a.Context, "", false)
-	if err != nil {
-		return err
-	}
-
-	hasError := false
-	for _, r := range rs {
-		if r.Meta.ReconcileError != "" {
-			hasError = true
-			break
-		}
-	}
-
-	if hasError {
-		a.Logger.Named("console").Errorf("Hydration failed")
-		if strict {
-			return fmt.Errorf("strict mode exit")
-		}
-	} else {
-		a.Logger.Named("console").Infof("Hydration completed!")
-	}
-
-	return nil
-}
-
 func (a *App) Serve(httpPort, grpcPort int, enableUI, openBrowser, readonly bool, userID string) error {
 	// Get analytics info
 	installID, enabled, err := dotrill.AnalyticsInfo()