Merge branch 'master' into stateless_dag

PostHog · Jan 25, 2025 · 8578d3f · 8578d3f
2 parents 72bde2c + b6f14aa
commit 8578d3f
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 0 deletions.
diff --git a/dags/definitions.py b/dags/definitions.py
@@ -13,6 +13,25 @@
     execution_timezone="UTC",
     name="deletes_schedule",
 )
+env = "local" if settings.DEBUG else "prod"
+
+# Define resources for different environments
+resources_by_env = {
+    "prod": {
+        "cluster": ClickhouseClusterResource.configure_at_launch(),
+        "io_manager": s3_pickle_io_manager.configured(
+            {"s3_bucket": settings.DAGSTER_S3_BUCKET, "s3_prefix": "dag-storage"}
+        ),
+        "s3": s3_resource,
+    },
+    "local": {
+        "cluster": ClickhouseClusterResource.configure_at_launch(),
+        "io_manager": fs_io_manager,
+    },
+}
+
+# Get resources for current environment, fallback to local if env not found
+resources = resources_by_env.get(env, resources_by_env["local"])
 
 defs = Definitions(
     assets=all_assets,

diff --git a/posthog/settings/__init__.py b/posthog/settings/__init__.py
@@ -24,6 +24,7 @@
 from posthog.settings.async_migrations import *
 from posthog.settings.celery import *
 from posthog.settings.data_stores import *
+from posthog.settings.dagster import *
 from posthog.settings.demo import *
 from posthog.settings.dynamic_settings import *
 from posthog.settings.ee import *

diff --git a/posthog/settings/dagster.py b/posthog/settings/dagster.py
@@ -0,0 +1,3 @@
+import os
+
+DAGSTER_S3_BUCKET = os.getenv("DAGSTER_S3_BUCKET", "posthog-dags")
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		import os

		DAGSTER_S3_BUCKET = os.getenv("DAGSTER_S3_BUCKET", "posthog-dags")