Add env variable to locate the project folder and resolve new pydanti…

…c version issues (#971)

Co-authored-by: jeffrey <[email protected]>

api/app.py

@@ -87,9 +87,12 @@
 current_task_id = None  # ID of the currently running task
 lock = asyncio.Lock()  # To manage access to shared variables
 
-ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
-WORK_DIR = os.path.join(ROOT_DIR, "projects")
-
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+WORK_DIR = os.getenv("AUTORAG_WORK_DIR", None)
+if WORK_DIR is None:
+    WORK_DIR = os.path.join(ROOT_DIR, "projects")
+if not os.path.exists(WORK_DIR):
+    os.makedirs(WORK_DIR)
 
 # Function to create a task
 # async def create_task(task_id: str, task: Task, func, *args):

api/requirements.txt

@@ -1,5 +1,5 @@
-AutoRAG[parse,api]>=0.3.8
-quart-schema
+AutoRAG[parse]>=0.3.10
+pydantic<2.10.0
 jwt
 quart-cors
 Werkzeug

autorag/autorag/__init__.py

@@ -5,7 +5,7 @@
 from random import random
 from typing import List, Any
 
-from llama_index.core import MockEmbedding
+from llama_index.core.embeddings.mock_embed_model import MockEmbedding
 from llama_index.core.base.llms.types import CompletionResponse
 from llama_index.core.llms.mock import MockLLM
 from llama_index.llms.bedrock import Bedrock

autorag/requirements.txt

@@ -1,3 +1,4 @@
+pydantic<2.10.0 # incompatible with llama index
 numpy<2.0.0 # temporal not using numpy 2.0.0
 pandas>=2.1.0
 tqdm

docker-compose.yml

@@ -29,6 +29,7 @@ services:
       - PYTHONUNBUFFERED=1
       - PORT=5001
       - HOST=0.0.0.0
+      - AUTORAG_WORK_DIR=/app/projects
     command: uvicorn app:app --host 0.0.0.0 --port 5001 --reload --loop asyncio
     networks:
       - app-network

projects/tutorial_1/trial_config.csv

@@ -1,2 +1,3 @@
 id,project_id,config,name,status,created_at,report_task_id,chat_task_id
-8c42b9e6-490d-4971-bb9e-705b36b7a3a2,tutorial_1,"{""id"": ""8c42b9e6-490d-4971-bb9e-705b36b7a3a2"", ""project_id"": ""tutorial_1"", ""config"": {""trial_id"": ""8c42b9e6-490d-4971-bb9e-705b36b7a3a2"", ""project_id"": ""tutorial_1"", ""raw_path"": null, ""corpus_path"": null, ""qa_path"": null, ""config_path"": null, ""metadata"": {}}, ""name"": ""Trial_20241117_1111"", ""status"": ""not_started"", ""created_at"": ""2024-11-17T11:11:15.348589"", ""report_task_id"": null, ""chat_task_id"": null, ""corpus_path"": null, ""qa_path"": ""/Users/martin/Development/org_autorag/SaaS-AutoRAG-API/projects/tutorial_1/qa/qa_8c42b9e6-490d-4971-bb9e-705b36b7a3a2.parquet""}",Trial_20241117_1111,not_started,2024-11-17T11:11:15.348589,,
+8c42b9e6-490d-4971-bb9e-705b36b7a3a2,tutorial_1,"{""id"": ""8c42b9e6-490d-4971-bb9e-705b36b7a3a2"", ""project_id"": ""tutorial_1"", ""config"": {""trial_id"": ""8c42b9e6-490d-4971-bb9e-705b36b7a3a2"", ""project_id"": ""tutorial_1"", ""raw_path"": null, ""corpus_path"": null, ""qa_path"": null, ""config_path"": null, ""metadata"": {}}, ""name"": ""Trial_20241117_1111"", ""status"": ""not_started"", ""created_at"": ""2024-11-17T11:11:15.348589"", ""report_task_id"": null, ""chat_task_id"": null, ""corpus_path"": null, ""qa_path"": ""/Users/martin/Development/org_autorag/SaaS-AutoRAG-API/projects/tutorial_1/qa/qa_8c42b9e6-490d-4971-bb9e-705b36b7a3a2.parquet""}",Trial_20241117_1111,not_started,2024-11-17T11:11:15.348589,,
+119f5ac8-a39d-47f7-aa87-e98dbed16931,tutorial_1,"{""id"": ""119f5ac8-a39d-47f7-aa87-e98dbed16931"", ""project_id"": ""tutorial_1"", ""config"": {""trial_id"": ""119f5ac8-a39d-47f7-aa87-e98dbed16931"", ""project_id"": ""tutorial_1"", ""raw_path"": null, ""corpus_path"": null, ""qa_path"": null, ""config_path"": null, ""metadata"": {}}, ""name"": ""Trial_20241121_1618"", ""status"": ""not_started"", ""created_at"": ""2024-11-21T08:18:09.443575"", ""report_task_id"": null, ""chat_task_id"": null, ""corpus_path"": null, ""qa_path"": ""/app/projects/tutorial_1/qa/qa_119f5ac8-a39d-47f7-aa87-e98dbed16931.parquet""}",Trial_20241121_1618,not_started,2024-11-21T08:18:09.443575,,