Add max_worker option

ooni · Jan 25, 2025 · 9fe1122 · 9fe1122
1 parent 0574fc0
commit 9fe1122
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/oonipipeline/src/oonipipeline/settings.py b/oonipipeline/src/oonipipeline/settings.py
@@ -18,6 +18,8 @@ class Settings(BaseSettings):
     clickhouse_url: str = "clickhouse://localhost"
     clickhouse_write_batch_size: int = 200_000
 
+    max_workers = int(os.cpu_count() * 0.7)
+
     telemetry_endpoint: Optional[str] = None
     prometheus_bind_address: Optional[str] = None
 

diff --git a/oonipipeline/src/oonipipeline/tasks/observations.py b/oonipipeline/src/oonipipeline/tasks/observations.py
@@ -224,7 +224,7 @@ def make_observations(params: MakeObservationsParams) -> MakeObservationsResult:
         bucket_date=params.bucket_date,
     )
 
-    with concurrent.futures.ProcessPoolExecutor() as executor:
+    with concurrent.futures.ProcessPoolExecutor(max_workers=config.max_workers) as executor:
         futures = [
             executor.submit(
                 make_observations_for_file_entry_batch,