updating ray actor options

guardrails-ai · Sep 10, 2024 · 38ad66e · 38ad66e
1 parent 8a9a446
commit 38ad66e
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 5 deletions.
diff --git a/app_ray_serve.py b/app_ray_serve.py
@@ -8,7 +8,7 @@
 
 app = FastAPI()
 
-@serve.deployment
+@serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 1})
 class InferenceDeployment:
     def __init__(self):
         self.spec = InferenceSpec()

diff --git a/app_ray_serve_config.yaml b/app_ray_serve_config.yaml
@@ -12,17 +12,27 @@ logging_config:
 
 applications:
 - name: guardrails--toxic_language
-  route_prefix: /guardrails/toxic_language
+  route_prefix: /validator/guardrails/toxic_language
   import_path: app_ray_serve:app
   runtime_env: {
     "pip": ["fastapi", "transformers", "torch", "torchvision", "detoxify"],
-    "working_dir": "https://github.com/guardrails-ai/toxic_language/archive/f208ad6aad6c4011a0e519de354e4a0f57b0c169.zip"
+    "working_dir": "https://github.com/guardrails-ai/toxic_language/archive/f208ad6aad6c4011a0e519de354e4a0f57b0c169.zip",
+    "env_vars": {
+      "env": "prod"
+    }
   }
   deployments:
   - name: InferenceDeployment
     num_replicas: 1
+    ray_actor_options:
+      num_cpus: 1.0
+      num_gpus: 1.0
     user_config:
-      version: 14
+      version: 15
   - name: Ingress
+    num_replicas: 1
+    ray_actor_options:
+      num_cpus: 2.0
+      num_gpus: 0.0
     user_config:
-      version: 14
+      version: 15