fix(api): incorrect rag endpoint config (#13)

This PR fixes the incorrect RAG endpoint setting for the API service.

Close #12

nesis/api/__init__.py

@@ -6,5 +6,7 @@
 PRETTY_LOG_FORMAT = (
     "%(asctime)s.%(msecs)03d [%(levelname)-8s] %(name)+25s - %(message)s"
 )
-logging.basicConfig(level=ROOT_LOG_LEVEL, format=PRETTY_LOG_FORMAT, datefmt="%H:%M:%S")
+logging.basicConfig(
+    level=ROOT_LOG_LEVEL, format=PRETTY_LOG_FORMAT, datefmt="%Y-%m-%d %H:%M:%S"
+)
 logging.captureWarnings(True)

nesis/api/config.yml

@@ -2,7 +2,7 @@ database:
     url: postgresql://postgres:password@localhost:65432/nesis
     debug: false
 
-llm:
+rag:
     endpoint: "http://localhost:9080"
 
 service:

nesis/api/core/document_loaders/minio.py

@@ -27,7 +27,7 @@
 
 def fetch_documents(
     connection: Dict[str, str],
-    pgpt_endpoint: str,
+    rag_endpoint: str,
     http_client: http.HttpClient,
     metadata: Dict[str, Any],
 ) -> None:
@@ -47,14 +47,14 @@ def fetch_documents(
         _sync_s3_documents(
             client=_minio_client,
             connection=connection,
-            pgpt_endpoint=pgpt_endpoint,
+            rag_endpoint=rag_endpoint,
             http_client=http_client,
             metadata=metadata,
         )
         _unsync_s3_documents(
             client=_minio_client,
             connection=connection,
-            pgpt_endpoint=pgpt_endpoint,
+            rag_endpoint=rag_endpoint,
             http_client=http_client,
         )
     except:
@@ -64,7 +64,7 @@ def fetch_documents(
 def _sync_s3_documents(
     client: Minio,
     connection: dict,
-    pgpt_endpoint: str,
+    rag_endpoint: str,
     http_client: http.HttpClient,
     metadata: dict,
 ) -> None:
@@ -81,7 +81,7 @@ def _sync_s3_documents(
         work_dir = f"/tmp/{uuid.uuid4()}"
         pathlib.Path(work_dir).mkdir(parents=True)
 
-        _LOG.info(f"Initializing syncing to endpoint {pgpt_endpoint}")
+        _LOG.info(f"Initializing syncing to endpoint {rag_endpoint}")
 
         for bucket_name in bucket_names_parts:
             try:
@@ -93,15 +93,15 @@ def _sync_s3_documents(
                 _sync_document(
                     client=client,
                     connection=connection,
-                    pgpt_endpoint=pgpt_endpoint,
+                    rag_endpoint=rag_endpoint,
                     http_client=http_client,
                     metadata=metadata,
                     bucket_name=bucket_name,
                     item=item,
                     work_dir=work_dir,
                 )
 
-        _LOG.info(f"Completed syncing to endpoint {pgpt_endpoint}")
+        _LOG.info(f"Completed syncing to endpoint {rag_endpoint}")
 
     except:
         _LOG.warn("Error fetching and updating documents", exc_info=True)
@@ -110,7 +110,7 @@ def _sync_s3_documents(
 def _sync_document(
     client: Minio,
     connection: dict,
-    pgpt_endpoint: str,
+    rag_endpoint: str,
     http_client: http.HttpClient,
     metadata: dict,
     bucket_name: str,
@@ -155,7 +155,7 @@ def _sync_document(
                     try:
                         util.un_ingest_file(
                             http_client=http_client,
-                            endpoint=pgpt_endpoint,
+                            endpoint=rag_endpoint,
                             doc_id=document_data["doc_id"],
                         )
                     except:
@@ -173,7 +173,7 @@ def _sync_document(
         try:
             response = ingest_file(
                 http_client=http_client,
-                endpoint=pgpt_endpoint,
+                endpoint=rag_endpoint,
                 metadata=_metadata,
                 file_path=file_path,
             )
@@ -206,7 +206,7 @@ def _sync_document(
 
 
 def _unsync_s3_documents(
-    client: Minio, connection: dict, pgpt_endpoint: str, http_client: http.HttpClient
+    client: Minio, connection: dict, rag_endpoint: str, http_client: http.HttpClient
 ) -> None:
 
     try:
@@ -226,7 +226,7 @@ def _unsync_s3_documents(
                     for document_data in rag_metadata.get("data") or []:
                         try:
                             http_client.delete(
-                                url=f"{pgpt_endpoint}/v1/ingest/documents/{document_data['doc_id']}"
+                                url=f"{rag_endpoint}/v1/ingest/documents/{document_data['doc_id']}"
                             )
                         except:
                             _LOG.warn(

nesis/api/core/document_loaders/samba.py

@@ -81,7 +81,7 @@ def _connect_samba_server(connection):
         raise
 
 
-def _sync_samba_documents(connection, pgpt_endpoint, http_client, metadata):
+def _sync_samba_documents(connection, rag_endpoint, http_client, metadata):
 
     username = connection["user"]
     password = connection["password"]
@@ -110,7 +110,7 @@ def _sync_samba_documents(connection, pgpt_endpoint, http_client, metadata):
             continue
         try:
             _process_file(
-                connection, file_share, work_dir, http_client, pgpt_endpoint, metadata
+                connection, file_share, work_dir, http_client, rag_endpoint, metadata
             )
         except:
             _LOG.warn(
@@ -119,12 +119,12 @@ def _sync_samba_documents(connection, pgpt_endpoint, http_client, metadata):
             )
     _LOG.info(
         f"Completed syncing files from samba server {endpoint} "
-        f"to endpoint {pgpt_endpoint}"
+        f"to endpoint {rag_endpoint}"
     )
 
 
 def _process_file(
-    connection, file_share, work_dir, http_client, pgpt_endpoint, metadata
+    connection, file_share, work_dir, http_client, rag_endpoint, metadata
 ):
     username = connection["user"]
     password = connection["password"]
@@ -138,7 +138,7 @@ def _process_file(
             )
             for dir_file in dir_files:
                 _process_file(
-                    connection, dir_file, work_dir, http_client, pgpt_endpoint, metadata
+                    connection, dir_file, work_dir, http_client, rag_endpoint, metadata
                 )
         return
 
@@ -195,7 +195,7 @@ def _process_file(
                     try:
                         util.un_ingest_file(
                             http_client=http_client,
-                            endpoint=pgpt_endpoint,
+                            endpoint=rag_endpoint,
                             doc_id=document_data["doc_id"],
                         )
                     except:
@@ -223,7 +223,7 @@ def _process_file(
         try:
             response = ingest_file(
                 http_client=http_client,
-                endpoint=pgpt_endpoint,
+                endpoint=rag_endpoint,
                 metadata=_metadata,
                 file_path=file_path,
             )
@@ -247,11 +247,11 @@ def _process_file(
             exc_info=True,
         )
     _LOG.info(
-        f"Completed syncing files from shared_file share {file_share.path} to endpoint {pgpt_endpoint}"
+        f"Completed syncing files from shared_file share {file_share.path} to endpoint {rag_endpoint}"
     )
 
 
-def _unsync_samba_documents(connection, pgpt_endpoint, http_client):
+def _unsync_samba_documents(connection, rag_endpoint, http_client):
     try:
         username = connection["user"]
         password = connection["password"]
@@ -276,7 +276,7 @@ def _unsync_samba_documents(connection, pgpt_endpoint, http_client):
                     try:
                         util.un_ingest_file(
                             http_client=http_client,
-                            endpoint=pgpt_endpoint,
+                            endpoint=rag_endpoint,
                             doc_id=document_data["doc_id"],
                         )
                     except:
@@ -285,6 +285,6 @@ def _unsync_samba_documents(connection, pgpt_endpoint, http_client):
                         )
                 _LOG.info(f"Deleting document {document.filename}")
                 delete_document(document_id=document.id)
-        _LOG.info(f"Completed unsyncing files from endpoint {pgpt_endpoint}")
+        _LOG.info(f"Completed unsyncing files from endpoint {rag_endpoint}")
     except:
         _LOG.warn("Error fetching and updating documents", exc_info=True)

nesis/api/core/document_loaders/sharepoint.py

@@ -25,14 +25,14 @@
 
 def fetch_documents(**kwargs) -> None:
     try:
-        pgpt_endpoint = kwargs["pgpt_endpoint"]
+        rag_endpoint = kwargs["rag_endpoint"]
         connection = kwargs["connection"]
 
         _sync_sharepoint_documents(
-            **kwargs, connection=connection, pgpt_endpoint=pgpt_endpoint
+            **kwargs, connection=connection, rag_endpoint=rag_endpoint
         )
         _unsync_sharepoint_documents(
-            **kwargs, connection=connection, pgpt_endpoint=pgpt_endpoint
+            **kwargs, connection=connection, rag_endpoint=rag_endpoint
         )
     except:
         _LOG.exception("Error fetching sharepoint documents")
@@ -55,13 +55,13 @@ def _sync_sharepoint_documents(**kwargs):
 
         file_locations = doc_store_connection.get("file_locations")
 
-        pgpt_endpoint = kwargs["pgpt_endpoint"]
+        rag_endpoint = kwargs["rag_endpoint"]
 
         location_names = file_locations.split(",")
         work_dir = f"/tmp/{uuid.uuid4()}"
         pathlib.Path(work_dir).mkdir(parents=True)
 
-        _LOG.info(f"Initializing sharepoint syncing to endpoint {pgpt_endpoint}")
+        _LOG.info(f"Initializing sharepoint syncing to endpoint {rag_endpoint}")
 
         # Sharepoint stores documents in different ways, in folders and there is a concept of lists Most document
         # libraries are stores as lists, thus we can safely assume that the locations provided in the settings are
@@ -108,7 +108,7 @@ def _sync_sharepoint_documents(**kwargs):
                             for document_data in rag_metadata.get("data") or []:
                                 try:
                                     http_client.delete(
-                                        url=f"{pgpt_endpoint}/v1/ingest/{document_data['doc_id']}"
+                                        url=f"{rag_endpoint}/v1/ingest/{document_data['doc_id']}"
                                     )
                                 except:
                                     _LOG.warn(
@@ -123,7 +123,7 @@ def _sync_sharepoint_documents(**kwargs):
                                 )
 
                     response = http_client.upload(
-                        url=f"{pgpt_endpoint}/v1/ingest",
+                        url=f"{rag_endpoint}/v1/ingest",
                         filepath=downloaded_file_name,
                         field="file",
                     )
@@ -155,7 +155,7 @@ def _sync_sharepoint_documents(**kwargs):
                         f"Error when getting and ingesting file {file.name} - {ex}"
                     )
 
-        _LOG.info(f"Completed syncing to endpoint {pgpt_endpoint}")
+        _LOG.info(f"Completed syncing to endpoint {rag_endpoint}")
 
     except:
         _LOG.warn("Error fetching and updating documents", exc_info=True)
@@ -165,7 +165,7 @@ def _unsync_sharepoint_documents(**kwargs):
     global _sharepoint_context
     http_client: http.HttpClient = kwargs["http_client"]
     doc_store_connection = kwargs["connection"]
-    pgpt_endpoint = kwargs["pgpt_endpoint"]
+    rag_endpoint = kwargs["rag_endpoint"]
 
     try:
         site_url = doc_store_connection.get("site_url")
@@ -193,7 +193,7 @@ def _unsync_sharepoint_documents(**kwargs):
                     for document_data in rag_metadata.get("data") or []:
                         try:
                             http_client.delete(
-                                url=f"{pgpt_endpoint}/v1/ingest/{document_data['doc_id']}"
+                                url=f"{rag_endpoint}/v1/ingest/{document_data['doc_id']}"
                             )
                         except:
                             _LOG.warn(

nesis/api/core/tasks/document_management.py

@@ -17,36 +17,36 @@
 def fetch_documents(**kwargs) -> None:
     try:
         config = kwargs["config"] or {}
-        pgpt_endpoint = (config.get("llm") or {}).get("endpoint")
+        rag_endpoint = (config.get("rag") or {}).get("endpoint")
         http_client = kwargs["http_client"]
         datasource_list: List[Datasource] = DatasourceService.get_datasources()
 
         for datasource in datasource_list:
             if datasource.type == DatasourceType.MINIO:
                 s3_documents.fetch_documents(
                     connection=datasource.connection,
-                    pgpt_endpoint=pgpt_endpoint,
+                    rag_endpoint=rag_endpoint,
                     http_client=http_client,
                     metadata={"datasource": datasource.name},
                 )
             if datasource.type == DatasourceType.SHAREPOINT:
                 sharepoint_documents.fetch_documents(
                     **kwargs,
                     connection=datasource.connection,
-                    pgpt_endpoint=pgpt_endpoint,
+                    rag_endpoint=rag_endpoint,
                     metadata={"datasource": datasource.name}
                 )
             if datasource.type == DatasourceType.GOOGLE_DRIVE:
                 google_drive.fetch_documents(
                     connection=datasource.connection,
-                    llm_endpoint=pgpt_endpoint,
+                    llm_endpoint=rag_endpoint,
                     http_client=http_client,
                     metadata={"datasource": datasource.name},
                 )
             if datasource.type == DatasourceType.WINDOWS_SHARE:
                 samba.fetch_documents(
                     connection=datasource.connection,
-                    llm_endpoint=pgpt_endpoint,
+                    llm_endpoint=rag_endpoint,
                     http_client=http_client,
                     metadata={"datasource": datasource.name},
                 )

nesis/api/tests/core/document_loaders/test_minio.py

@@ -83,7 +83,7 @@ def test_fetch_documents(minio_instance: mock.MagicMock, session: Session) -> No
         connection=data["connection"],
         http_client=http_client,
         metadata={"datasource": "documents"},
-        pgpt_endpoint="http://localhost:8080",
+        rag_endpoint="http://localhost:8080",
     )
 
     _, upload_kwargs = http_client.upload.call_args_list[0]

nesis/rag/__init__.py

@@ -9,5 +9,7 @@
 PRETTY_LOG_FORMAT = (
     "%(asctime)s.%(msecs)03d [%(levelname)-8s] %(name)+25s - %(message)s"
 )
-logging.basicConfig(level=ROOT_LOG_LEVEL, format=PRETTY_LOG_FORMAT, datefmt="%H:%M:%S")
+logging.basicConfig(
+    level=ROOT_LOG_LEVEL, format=PRETTY_LOG_FORMAT, datefmt="%Y-%m-%d %H:%M:%S"
+)
 logging.captureWarnings(True)

package.sh

@@ -1,3 +1,5 @@
-while read v; do
-  docker build -t ametnes/nesis:"$v"-frontend . -f nesis/frontend/Dockerfile
-done <version.txt
+while read -r v; do
+  docker build -t ametnes/nesis:"$v"-rag . -f nesis/rag/Dockerfile
+  docker build -t ametnes/nesis:"$v"-api . -f nesis/api/Dockerfile
+  docker build --build-arg PUBLIC_URL=/ --build-arg PROFILE=PROD -t ametnes/nesis:"$v"-frontend . -f nesis/frontend/Dockerfile
+done <version.txt