Merge branch 'main' into adityahegde/rill-time-syntax-backend

admin/provisioner/clickhousestatic/provisioner.go

@@ -137,6 +137,14 @@ func (p *Provisioner) Provision(ctx context.Context, r *provisioner.Resource, op
 		return nil, fmt.Errorf("failed to grant privileges to clickhouse user: %w", err)
 	}
 
+	// Grant access to system.parts for reporting disk usage.
+	// NOTE 1: ClickHouse automatically adds row filters to restrict result to tables the user has access to.
+	// NOTE 2: We do not need to explicitly grant access to system.tables and system.columns because ClickHouse adds those implicitly.
+	_, err = p.ch.ExecContext(ctx, fmt.Sprintf("GRANT SELECT ON system.parts TO %s", user))
+	if err != nil {
+		return nil, fmt.Errorf("failed to grant system privileges to clickhouse user: %w", err)
+	}
+
 	// Grant some additional global privileges to the user
 	_, err = p.ch.ExecContext(ctx, fmt.Sprintf(`
 		GRANT

admin/provisioner/clickhousestatic/provisioner_test.go

@@ -66,7 +66,7 @@ func TestClickHouseStatic(t *testing.T) {
 	require.NoError(t, err)
 	_, err = db1.Exec("INSERT INTO test VALUES (1)")
 	require.NoError(t, err)
-	rows, err := db1.Query("SELECT COUNT(*) FROM system.tables")
+	rows, err := db1.Query("SELECT COUNT(*) FROM system.tables WHERE database <> 'system'")
 	require.NoError(t, err)
 	for rows.Next() {
 		var count int
@@ -88,7 +88,7 @@ func TestClickHouseStatic(t *testing.T) {
 	require.Error(t, err)
 
 	// Check the second connection can't see the other connection's tables in the information schema
-	rows, err = db2.Query("SELECT name FROM system.tables")
+	rows, err = db2.Query("SELECT name FROM system.tables WHERE database <> 'system'")
 	require.NoError(t, err)
 	for rows.Next() {
 		require.Fail(t, "unexpected visible table in information schema")

cli/cmd/devtool/data/clickhouse-config.xml

@@ -0,0 +1,13 @@
+<clickhouse replace="true">
+    <logger>
+        <level>information</level>
+        <console>true</console>
+    </logger>
+    <display_name>rill devtool clickhouse</display_name>
+    <tcp_port>9000</tcp_port>
+    <http_port>8123</http_port>
+    <timezone>UTC</timezone>
+    <access_control_improvements>
+        <select_from_information_schema_requires_grant>true</select_from_information_schema_requires_grant>
+    </access_control_improvements>
+</clickhouse>

cli/cmd/devtool/data/clickhouse-users.xml

@@ -0,0 +1,37 @@
+<clickhouse replace="true">
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>in_order</load_balancing>
+            <log_queries>1</log_queries>
+            <date_time_input_format>best_effort</date_time_input_format>
+        </default>
+    </profiles>
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+    <users>
+        <default>
+            <password>default</password>
+            <profile>default</profile>
+            <quota>default</quota>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <access_management>1</access_management>
+            <named_collection_control>1</named_collection_control>
+            <show_named_collections>1</show_named_collections>
+            <show_named_collections_secrets>1</show_named_collections_secrets>
+        </default>
+    </users>
+</clickhouse>

cli/cmd/devtool/data/cloud-deps.docker-compose.yml

@@ -64,3 +64,12 @@ services:
     command: listen --forward-to http://host.docker.internal:8080/payment/webhook --config /etc/stripe-config.toml
     volumes:
       - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/stripe-config.toml:/etc/stripe-config.toml
+  clickhouse:
+    image: 'clickhouse/clickhouse-server:24.11.1.2557'
+    volumes:
+      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/config.xml
+      - ./clickhouse-users.xml:/etc/clickhouse-server/users.d/users.xml
+      - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/clickhouse:/var/lib/clickhouse
+    ports:
+      - '9000:9000' # Native port
+      - '8123:8123' # HTTP port

cli/cmd/devtool/data/template.env

@@ -1,11 +1,19 @@
+# devtool
+RILL_DEVTOOL_ENV=dev
+RILL_DEVTOOL_STATE_DIRECTORY=dev-cloud-state
+RILL_DEVTOOL_STRIPE_CLI_API_KEY=
+
 # admin service
 RILL_ADMIN_DATABASE_DRIVER=postgres
 RILL_ADMIN_DATABASE_URL=postgres://postgres:postgres@localhost:5432/postgres
+RILL_ADMIN_RIVER_DATABASE_URL=postgres://postgres:postgres@localhost:5432/postgres
 RILL_ADMIN_REDIS_URL=redis://localhost:6379
-RILL_ADMIN_PROVISIONER_SPEC='{"runtimes":[{"host":"http://localhost:9091","slots":50,"data_dir":"dev-cloud-state","audience_url":"http://localhost:8081"}]}'
+RILL_ADMIN_PROVISIONER_SET_JSON='{"static":{"type":"static","spec":{"runtimes":[{"host":"http://localhost:9091","slots":50,"data_dir":"dev-cloud-state","audience_url":"http://localhost:8081"}]}}, "clickhouse": {"type": "clickhouse-static", "spec": {"dsn": "clickhouse://default:default@localhost:9000"}}}'
+RILL_ADMIN_DEFAULT_PROVISIONER=static
 RILL_ADMIN_HTTP_PORT=8080
 RILL_ADMIN_GRPC_PORT=9090
-RILL_ADMIN_DEBUG_PORT=6060
+RILL_ADMIN_DEBUG_PORT=6062
+RILL_ADMIN_PSQL_PORT=25432
 RILL_ADMIN_EXTERNAL_URL=http://localhost:8080
 RILL_ADMIN_EXTERNAL_GRPC_URL=http://localhost:9090
 RILL_ADMIN_FRONTEND_URL=http://localhost:3000
@@ -27,17 +35,20 @@ RILL_ADMIN_EMAIL_SMTP_USERNAME=
 RILL_ADMIN_EMAIL_SMTP_PASSWORD=
 RILL_ADMIN_EMAIL_SENDER_EMAIL=
 RILL_ADMIN_EMAIL_SENDER_NAME=
-RILL_ADMIN_ORB_API_KEY=
+RILL_ADMIN_OPENAI_API_KEY=
+RILL_ADMIN_ASSETS_BUCKET=
+RILL_ADMIN_ASSETS_BUCKET_GOOGLE_CREDENTIALS_JSON=
 RILL_ADMIN_STRIPE_API_KEY=
 RILL_ADMIN_STRIPE_WEBHOOK_SECRET=
+RILL_ADMIN_ORB_API_KEY=
 RILL_ADMIN_ORB_WEBHOOK_SECRET=
-RILL_DEVTOOL_STRIPE_CLI_API_KEY=
 
 # runtime service
 RILL_RUNTIME_METASTORE_URL=dev-cloud-state/meta.db
 RILL_RUNTIME_REDIS_URL=redis://localhost:6379
 RILL_RUNTIME_HTTP_PORT=8081
 RILL_RUNTIME_GRPC_PORT=9091
+RILL_RUNTIME_PSQL_PORT=15432
 RILL_RUNTIME_DEBUG_PORT=6061
 RILL_RUNTIME_SESSION_KEY_PAIRS=1234567890abcdef1234567890abcdef,1234567890abcdef1234567890abcdef
 RILL_RUNTIME_AUTH_ENABLE=true
@@ -49,6 +60,10 @@ RILL_RUNTIME_EMAIL_SMTP_USERNAME=
 RILL_RUNTIME_EMAIL_SMTP_PASSWORD=
 RILL_RUNTIME_EMAIL_SENDER_EMAIL=
 RILL_RUNTIME_EMAIL_SENDER_NAME=
+RILL_RUNTIME_DATA_DIR=dev-cloud-state
+RILL_RUNTIME_DATA_BUCKET=
+RILL_RUNTIME_DATA_BUCKET_CREDENTIALS_JSON=
+RILL_RUNTIME_ACTIVITY_SINK_TYPE=
 
 # observability-related config
 RILL_ADMIN_METRICS_EXPORTER="otel"
@@ -57,3 +72,10 @@ RILL_RUNTIME_METRICS_EXPORTER="otel"
 RILL_RUNTIME_TRACES_EXPORTER="otel"
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 # For details about all `OTEL_` env vars, see: https://github.com/open-telemetry/opentelemetry-go/tree/main/exporters/otlp/otlptrace
+
+# ui
+RILL_UI_PUBLIC_POSTHOG_API_KEY=
+RILL_UI_PUBLIC_PYLON_APP_ID=
+
+# runtime test connectors
+RILL_RUNTIME_DRUID_TEST_DSN=

cli/cmd/devtool/start.go

@@ -252,12 +252,11 @@ func (s cloud) start(ctx context.Context, ch *cmdutil.Helper, verbose, reset, re
 
 	g, ctx := errgroup.WithContext(ctx)
 
-	stateDir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
-	err = os.MkdirAll(stateDir, os.ModePerm)
+	err = os.MkdirAll(stateDirectory(), os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("failed to create state dir %q: %w", stateDir, err)
+		return fmt.Errorf("failed to create state dir %q: %w", stateDirectory(), err)
 	}
-	logInfo.Printf("State dir set to %s\n", stateDir)
+	logInfo.Printf("State directory is %q\n", stateDirectory())
 
 	if services.deps {
 		g.Go(func() error { return s.runDeps(ctx, verbose) })
@@ -385,8 +384,8 @@ func (s cloud) resetState(ctx context.Context) (err error) {
 		}
 	}()
 
-	stateDir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
-	_ = os.RemoveAll(stateDir)
+	_ = os.RemoveAll(stateDirectory())
+
 	return newCmd(ctx, "docker", "compose", "--env-file", ".env", "-f", "cli/cmd/devtool/data/cloud-deps.docker-compose.yml", "down", "--volumes").Run()
 }
 
@@ -724,8 +723,7 @@ func (s local) awaitUI(ctx context.Context) error {
 
 func prepareStripeConfig() error {
 	templateFile := "cli/cmd/devtool/data/stripe-config.template"
-	stateDir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
-	outputFile := filepath.Join(stateDir, "stripe-config.toml")
+	outputFile := filepath.Join(stateDirectory(), "stripe-config.toml")
 
 	apiKey := lookupDotenv("RILL_DEVTOOL_STRIPE_CLI_API_KEY")
 	if apiKey == "" {
@@ -785,3 +783,13 @@ func lookupDotenv(key string) string {
 	}
 	return env[key]
 }
+
+// stateDirectory returns the directory where the devtool's state is stored.
+// Deleting this directory will reset the state of the local development environment.
+func stateDirectory() string {
+	dir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
+	if dir == "" {
+		dir = "dev-cloud-state"
+	}
+	return dir
+}

cli/cmd/runtime/start.go

@@ -185,6 +185,9 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command {
 					logger.Fatal("error creating kafka sink", zap.Error(err))
 				}
 				activityClient = activity.NewClient(sink, logger)
+			case "console":
+				sink := activity.NewLoggerSink(logger, zapcore.InfoLevel)
+				activityClient = activity.NewClient(sink, logger)
 			default:
 				logger.Fatal("unknown activity sink type", zap.String("type", conf.ActivitySinkType))
 			}

docs/docs/home/FAQ.md

@@ -95,3 +95,37 @@ You can follow the same steps as above. The button will have changed from `deplo
 ### How do I share my dashboard to other users?
 
 You will need to [invite users to your organization/project](https://docs.rilldata.com/manage/user-management#option-1---admin-invites-user) or send them a URL for them to [request access to your dashboard](https://docs.rilldata.com/manage/user-management#option-2---user-requests-access). If you just want them to see the contents of your dashboard, you can look into using [public URLs](https://docs.rilldata.com/explore/share-url).
+
+
+## Rill Cloud Trial
+### What is Rill Cloud Trial?
+We offer a free 30 day trial to any one interested in testing out our online platform. Simply create an account and deploy your project from Rill Developer. If you haven't already created and account and logged in, you will be prompted during the deployment process. 
+
+There are no feature limitations in a free trial but we have set the limit for imported data to 10GB per project with two projects per deployment. You can check the data usage in the settings page. 
+
+:::note 
+The banner will show you the remaining days for your trial and will update as the expiration gets closer! Upgrade to a Teams plan and input your payment method to continue using Rill!
+:::
+![img](/img/FAQ/rill-trial-banner.png)
+
+### When does my trial start?
+Your trial will start when you deploy a project to Rill Cloud from Rill Developer. An Organization will be autoamatically created during this process using your email and the project will be the folder that your Rill project exists in. You can change the name using [CLI commands](https://docs.rilldata.com/reference/cli/project/rename). 
+
+### How long does my Rill Cloud Trial last?
+A Rill Cloud trial lasts for 30 days. If you have any comments or concerns, please reach out to us on our [various platforms](../contact.md)! 
+
+### What is included in the free trial? 
+The free trial is locked at 2 projects and up to 10GB of data each. You can invite as many users as required and there are no locked features. 
+
+### What happens to my project if I do not upgrade to a Team plan?
+Your projects will hibernate. Your project metadata will still be available once you've activated your team plan. If you'd like to delete your deployment from Rill Cloud, you can do so via the [CLI commands.](https://docs.rilldata.com/reference/cli/org/delete)
+
+![expired](/img/FAQ/expired-project.png)
+
+### What is project hibernation?
+When a project is inactive for a specific number of days or your trial has expired, we automatically hibernate the project. What this means is that all of your information and metadata is saved and resource consumption will be zero. You will need to unhibernate the project to gain access to the dashboard again. 
+
+If the project is hibernated due to payment issues, the project will stay in this state until payment is confirmed. Once the payment is confirmed, you can reaccess the project with the following CLI command.
+```
+rill project hibernate <project_id> --redeploy
+```

go.mod

@@ -109,6 +109,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.31.0
 	go.opentelemetry.io/otel/sdk/metric v1.31.0
 	go.opentelemetry.io/otel/trace v1.33.0
+	go.uber.org/atomic v1.11.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
 	go.uber.org/zap/exp v0.2.0
@@ -388,7 +389,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/mock v0.4.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect