FEAT: Unified Notifications

backend/account/authentication_controller.py

@@ -39,6 +39,7 @@
 from django.middleware import csrf
 from django.shortcuts import redirect
 from django_tenants.utils import tenant_context
+from logs_helper.log_service import LogService
 from rest_framework import status
 from rest_framework.request import Request
 from rest_framework.response import Response
@@ -268,6 +269,8 @@ def make_user_organization_display_name(self, user_name: str) -> str:
         return self.auth_service.make_user_organization_display_name(user_name)
 
     def user_logout(self, request: Request) -> Response:
+        session_id: str = UserSessionUtils.get_session_id(request=request)
+        LogService.remove_logs_on_logout(session_id=session_id)
         response = self.auth_service.user_logout(request=request)
         organization_id = UserSessionUtils.get_organization_id(request)
         user_id = UserSessionUtils.get_user_id(request)

backend/backend/settings/base.py

@@ -164,6 +164,9 @@ def get_required_setting(
     get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60")
 )
 LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
+LOGS_EXPIRATION_TIME_IN_SECOND = int(
+    get_required_setting("LOGS_EXPIRATION_TIME_IN_SECOND")
+)
 CELERY_BROKER_URL = get_required_setting(
     "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"
 )

backend/backend/urls.py

@@ -27,6 +27,7 @@
     path("", include("connector_processor.urls")),
     path("", include("adapter_processor.urls")),
     path("", include("file_management.urls")),
+    path("", include("logs_helper.urls")),
     path("", include("tool_instance.urls")),
     path("", include("pipeline.urls")),
     path("", include("feature_flag.urls")),

backend/logs_helper/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class LogsHelperConfig(AppConfig):
+    name = "logs_helper"

backend/logs_helper/constants.py

@@ -0,0 +1,3 @@
+class LogsHelperKeys:
+    LOG = "LOG"
+    LOG_EVENTS_ID = "log_events_id"

backend/logs_helper/log_service.py

@@ -0,0 +1,24 @@
+from utils.cache_service import CacheService
+
+
+class LogService:
+    @staticmethod
+    def remove_logs_on_logout(session_id: str) -> None:
+
+        if session_id:
+            key_pattern = f"{LogService.generate_redis_key(session_id=session_id)}*"
+
+            # Delete keys matching the pattern
+            CacheService.clear_cache(key_pattern=key_pattern)
+
+    @staticmethod
+    def generate_redis_key(session_id):
+        """Generate a Redis key for logs based on the provided session_id.
+
+        Parameters:
+        session_id (str): The session identifier to include in the Redis key.
+
+        Returns:
+        str: The constructed Redis key.
+        """
+        return f"logs:{session_id}"

backend/logs_helper/serializers.py

@@ -0,0 +1,5 @@
+from rest_framework import serializers
+
+
+class StoreLogMessagesSerializer(serializers.Serializer):
+    log = serializers.CharField()

backend/logs_helper/urls.py

@@ -0,0 +1,16 @@
+from django.urls import path
+from rest_framework.urlpatterns import format_suffix_patterns
+
+from .views import LogsHelperViewSet
+
+logs_helper = LogsHelperViewSet.as_view({"get": "get_logs", "post": "store_log"})
+
+urlpatterns = format_suffix_patterns(
+    [
+        path(
+            "logs/",
+            logs_helper,
+            name="logs-helper",
+        ),
+    ]
+)

backend/logs_helper/views.py

@@ -0,0 +1,66 @@
+import json
+import logging
+from datetime import datetime, timezone
+
+from django.conf import settings
+from django.http import HttpRequest
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from utils.cache_service import CacheService
+from utils.user_session import UserSessionUtils
+
+from .log_service import LogService
+from .serializers import StoreLogMessagesSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class LogsHelperViewSet(viewsets.ModelViewSet):
+    """Viewset to handle all Tool Studio prompt related API logics."""
+
+    @action(detail=False, methods=["get"])
+    def get_logs(self, request: HttpRequest) -> Response:
+        # Extract the session ID
+        session_id: str = UserSessionUtils.get_session_id(request=request)
+
+        # Construct the Redis key pattern to match keys
+        # associated with the session ID
+        redis_key = LogService.generate_redis_key(session_id=session_id)
+
+        # Retrieve keys matching the pattern
+        keys = CacheService.get_all_keys(f"{redis_key}*")
+
+        # Retrieve values corresponding to the keys and sort them by timestamp
+        logs = []
+        for key in keys:
+            log_data = CacheService.get_key(key)
+            logs.append(log_data)
+
+        # Sort logs based on timestamp
+        sorted_logs = sorted(logs, key=lambda x: x["timestamp"])
+
+        return Response({"data": sorted_logs}, status=status.HTTP_200_OK)
+
+    @action(detail=False, methods=["post"])
+    def store_log(self, request: HttpRequest) -> Response:
+        """Store log message in Redis."""
+        # Extract the session ID
+        logs_expiry = settings.LOGS_EXPIRATION_TIME_IN_SECOND
+        session_id: str = UserSessionUtils.get_session_id(request=request)
+
+        serializer = StoreLogMessagesSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+
+        # Extract the log message from the validated data
+        log: str = serializer.validated_data.get("log")
+        log_data = json.loads(log)
+        timestamp = datetime.now(timezone.utc).timestamp()
+
+        redis_key = (
+            f"{LogService.generate_redis_key(session_id=session_id)}:{timestamp}"
+        )
+
+        CacheService.set_key(redis_key, log_data, logs_expiry)
+
+        return Response({"message": "Successfully stored the message in redis"})

backend/sample.env

@@ -137,8 +137,9 @@ ENABLE_LOG_HISTORY=True
 LOG_HISTORY_CONSUMER_INTERVAL=30
 # Maximum number of logs to insert in a single batch.
 LOGS_BATCH_LIMIT=30
+# Logs Expiry
+LOGS_EXPIRATION_TIME_IN_SECOND=3600
 
-# Celery Configuration
 CELERY_BROKER_URL = "redis://unstract-redis:6379"
 
 # Indexing flag to prevent re-index

backend/utils/cache_service.py

@@ -28,6 +28,12 @@ def set_key(
             expire,
         )
 
+    @staticmethod
+    def get_all_keys(key_pattern: str) -> Any:
+        keys = redis_cache.keys(key_pattern)
+        # Ensure all keys are strings
+        return [key.decode("utf-8") if isinstance(key, bytes) else key for key in keys]
+
     @staticmethod
     def clear_cache(key_pattern: str) -> Any:
         """Delete keys in bulk based on the key pattern."""

backend/utils/user_session.py

@@ -25,6 +25,10 @@ def set_organization_id(request: HttpRequest, organization_id: str) -> None:
     def get_user_id(request: HttpRequest) -> Optional[str]:
         return request.session.get("user_id")
 
+    @staticmethod
+    def get_session_id(request: HttpRequest) -> Optional[str]:
+        return request.session.session_key
+
     @staticmethod
     def set_organization_member_role(
         request: HttpRequest, member: OrganizationMember

frontend/src/components/agency/agency/Agency.css

@@ -4,6 +4,7 @@
   height: 100%;
   display: flex;
   flex-direction: column;
+  padding-bottom: 10px;
 }
 
 .agency-sider-layout {
@@ -40,20 +41,16 @@
   padding: 12px;
 }
 
-.agency-ide-logs {
-  height: 10vh;
-}
-
 .agency-ide-collapse-panel {
   background-color: var(--white);
   border: none;
   border-radius: 0px;
 }
 
-.agency-ide-log-modal .ant-modal-content{
+.agency-ide-log-modal .ant-modal-content {
   height: 80vh;
 }
 
-.agency-ide-log-modal .agency-ide-logs{
+.agency-ide-log-modal .agency-ide-logs {
   height: 70vh !important;
 }

frontend/src/components/agency/agency/Agency.jsx

@@ -1,78 +1,32 @@
-import { Button, Collapse, Layout, Modal } from "antd";
-import {
-  FullscreenExitOutlined,
-  FullscreenOutlined,
-  LeftOutlined,
-  RightOutlined,
-} from "@ant-design/icons";
+import { Button, Layout } from "antd";
+import { LeftOutlined, RightOutlined } from "@ant-design/icons";
 import Sider from "antd/es/layout/Sider";
 import { useEffect, useState } from "react";
 
 import { IslandLayout } from "../../../layouts/island-layout/IslandLayout";
 import { Actions } from "../actions/Actions";
 import { WorkflowExecution } from "../workflow-execution/WorkflowExecution";
 import "./Agency.css";
-import { useSocketLogsStore } from "../../../store/socket-logs-store";
 import { useSocketMessagesStore } from "../../../store/socket-messages-store";
 import { useWorkflowStore } from "../../../store/workflow-store";
-import { LogsLabel } from "../logs-label/LogsLabel";
 import { SidePanel } from "../side-panel/SidePanel";
-import { DisplayLogs } from "../display-logs/DisplayLogs";
 
 function Agency() {
   const [isCollapsed, setIsCollapsed] = useState(false);
-  const [activeKey, setActiveKey] = useState([]);
   const [steps, setSteps] = useState([]);
   const [inputMd, setInputMd] = useState("");
   const [outputMd, setOutputMd] = useState("");
   const [statusBarMsg, setStatusBarMsg] = useState("");
   const [sourceMsg, setSourceMsg] = useState("");
   const [destinationMsg, setDestinationMsg] = useState("");
   const { message, setDefault } = useSocketMessagesStore();
-  const { emptyLogs } = useSocketLogsStore();
   const workflowStore = useWorkflowStore();
   const { details, loadingType } = workflowStore;
   const prompt = details?.prompt_text;
   const [activeToolId, setActiveToolId] = useState("");
   const [prevLoadingType, setPrevLoadingType] = useState("");
   const [isUpdateSteps, setIsUpdateSteps] = useState(false);
   const [stepLoader, setStepLoader] = useState(false);
-  const [showLogsModal, setShowLogsModal] = useState(false);
-
-  const openLogsModal = () => {
-    setShowLogsModal(true);
-  };
-
-  const closeLogsModal = () => {
-    setShowLogsModal(false);
-  };
-
-  const genExtra = () => (
-    <FullscreenOutlined
-      onClick={(event) => {
-        // If you don't want click extra trigger collapse, you can prevent this:
-        openLogsModal();
-        event.stopPropagation();
-      }}
-    />
-  );
-
-  const getItems = () => [
-    {
-      key: "1",
-      label: activeKey?.length > 0 ? <LogsLabel /> : "Logs",
-      children: (
-        <div className="agency-ide-logs">
-          <DisplayLogs />
-        </div>
-      ),
-      extra: genExtra(),
-    },
-  ];
-
-  const handleCollapse = (keys) => {
-    setActiveKey(keys);
-  };
 
   useEffect(() => {
     if (prevLoadingType !== "EXECUTE") {
@@ -102,7 +56,6 @@ function Agency() {
     setOutputMd("");
     setStatusBarMsg("");
     setDefault();
-    emptyLogs();
     setSourceMsg("");
     setDestinationMsg("");
   };
@@ -111,7 +64,6 @@ function Agency() {
     // Clean up function to clear all the socket messages
     return () => {
       setDefault();
-      emptyLogs();
     };
   }, []);
 
@@ -138,7 +90,6 @@ function Agency() {
     }
 
     if (msgComp === "SOURCE" && state === "RUNNING") {
-      setActiveKey("");
       setSourceMsg("");
       setDestinationMsg("");
       const newSteps = [...steps].map((step) => {
@@ -225,31 +176,6 @@ function Agency() {
           stepLoader={stepLoader}
         />
       </div>
-      <div className="agency-footer">
-        <Collapse
-          className="agency-ide-collapse-panel"
-          size="small"
-          activeKey={activeKey}
-          items={getItems()}
-          expandIconPosition="end"
-          onChange={handleCollapse}
-          bordered={false}
-        />
-        <Modal
-          title="Logs"
-          open={showLogsModal}
-          onCancel={closeLogsModal}
-          className="agency-ide-log-modal"
-          footer={null}
-          width={1000}
-          closeIcon={<FullscreenExitOutlined />}
-        >
-          <LogsLabel />
-          <div className="agency-ide-logs">
-            <DisplayLogs />
-          </div>
-        </Modal>
-      </div>
     </div>
   );
 }

frontend/src/components/agency/display-logs/DisplayLogs.css

@@ -1,15 +1,25 @@
 /* Styles for DisplayLogs */
 
 .tool-logs {
-    height: 100%;
-    overflow-y: auto;
+  height: 100%;
+  overflow-y: auto;
 }
 
 .display-logs-col-first {
-    font-size: 12px;
+  font-size: 12px;
 }
 
 .display-logs-col {
-    font-size: 12px;
-    padding-left: 5px;
+  font-size: 12px;
+  padding-left: 5px;
+}
+
+.display-logs-container {
+  padding-right: 25px;
+  padding-top: 2px;
+  border-top: 1px rgba(0, 0, 0, 0.15) solid;
+}
+
+.display-logs-error-bg {
+  background-color: #fad4d4;
 }