From 727b128ce38e831e0eeb6f8025c83f6258c7a1e6 Mon Sep 17 00:00:00 2001 From: vishnuszipstack Date: Fri, 23 Aug 2024 11:42:55 +0530 Subject: [PATCH 1/5] mrq deployments tab in progress --- .../0004_alter_pipeline_cron_string.py | 20 +++++++++++++ backend/pipeline/views.py | 28 ++++++++++++++++++- .../api-deployment/ApiDeployment.jsx | 9 ++++-- .../navigations/side-nav-bar/SideNavBar.jsx | 11 +++++++- .../etl-task-deploy/EtlTaskDeploy.jsx | 12 ++------ frontend/src/helpers/GetStaticData.js | 5 ++++ frontend/src/pages/DeploymentsPage.jsx | 2 +- frontend/src/routes/Router.jsx | 9 ++++++ 8 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 backend/pipeline/migrations/0004_alter_pipeline_cron_string.py diff --git a/backend/pipeline/migrations/0004_alter_pipeline_cron_string.py b/backend/pipeline/migrations/0004_alter_pipeline_cron_string.py new file mode 100644 index 000000000..305fc0125 --- /dev/null +++ b/backend/pipeline/migrations/0004_alter_pipeline_cron_string.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.1 on 2024-08-13 06:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pipeline", "0003_alter_pipeline_active_alter_pipeline_cron_string_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="pipeline", + name="cron_string", + field=models.TextField( + blank=True, db_comment="UNIX cron string", max_length=256, null=True + ), + ), + ] diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py index 09e9bd204..ecd374386 100644 --- a/backend/pipeline/views.py +++ b/backend/pipeline/views.py @@ -1,6 +1,6 @@ import json import logging -from typing import Optional +from typing import Any, Optional from account.custom_exceptions import DuplicateData from api.exceptions import NoActiveAPIKeyError @@ -36,6 +36,8 @@ class PipelineViewSet(viewsets.ModelViewSet): def get_queryset(self) -> Optional[QuerySet]: type = self.request.query_params.get(PipelineConstants.TYPE) if type is not None: + if type == "MRQ": + type = "ETL" queryset = Pipeline.objects.filter( created_by=self.request.user, pipeline_type=type ) @@ -50,6 +52,30 @@ def get_serializer_class(self) -> serializers.Serializer: else: return PipelineSerializer + def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: + queryset = self.get_queryset() + serializer = self.get_serializer(queryset, many=True) + data = serializer.data + + # Check if the request is for 'MRQ' + type = request.query_params.get(PipelineConstants.TYPE) + if type == "MRQ": + # Filter the data based on 'destination_name' + filtered_data = [ + item for item in data if item["destination_name"] == "Manual Review" + ] + return Response(filtered_data) + + if type == "ETL": + # Filter the data to exclude those with + # 'destination_name' == "Manual Review" + filtered_data = [ + item for item in data if item["destination_name"] != "Manual Review" + ] + return Response(filtered_data) + # If 'type' is not 'MRQ', return the default list + return super().list(request, *args, **kwargs) + # TODO: Refactor to perform an action with explicit arguments # For eg, passing pipeline ID and with_log=False -> executes pipeline # For FE however we call the same API twice diff --git a/frontend/src/components/deployments/api-deployment/ApiDeployment.jsx b/frontend/src/components/deployments/api-deployment/ApiDeployment.jsx index d9b174a1a..1424fd753 100644 --- a/frontend/src/components/deployments/api-deployment/ApiDeployment.jsx +++ b/frontend/src/components/deployments/api-deployment/ApiDeployment.jsx @@ -1,3 +1,4 @@ +import PropTypes from "prop-types"; import { CodeOutlined, CopyOutlined, @@ -30,7 +31,7 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js"; import usePipelineHelper from "../../../hooks/usePipelineHelper.js"; import { NotificationModal } from "../../pipelines-or-deployments/notification-modal/NotificationModal.jsx"; -function ApiDeployment() { +function ApiDeployment({ type }) { const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); const navigate = useNavigate(); @@ -378,7 +379,7 @@ function ApiDeployment() { return ( <> { const navigate = useNavigate(); const { sessionDetails } = useSessionStore(); @@ -147,7 +154,9 @@ const SideNavBar = ({ collapsed }) => { if (getMenuItem && flags.app_deployment) { data[0].subMenu.splice(1, 0, getMenuItem.default(orgName)); } - + if (getMRQMenuItem) { + data[0].subMenu.splice(3, 0, getMRQMenuItem.default(orgName)); + } return ( { - const connectorType = type === "task" ? "FILESYSTEM" : "DATABASE"; + let connectorType = type === "task" ? "FILESYSTEM" : "DATABASE"; + connectorType = type === "mrq" ? "MANUALREVIEW" : connectorType; setWorkflowList([]); fetchWorkflows(connectorType).then((data) => { - if (connectorType === "DATABASE") { - fetchWorkflows("MANUALREVIEW").then((manualReviewData) => { - const combinedData = [...data, ...manualReviewData]; - setWorkflowList(combinedData); - }); - } else { - setWorkflowList(data); - } + setWorkflowList(data); }); }; diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.js index 6ad179ee8..bb0778a68 100644 --- a/frontend/src/helpers/GetStaticData.js +++ b/frontend/src/helpers/GetStaticData.js @@ -233,6 +233,11 @@ const deploymentsStaticContent = { addBtn: "App Deployment", isLogsRequired: false, }, + mrq: { + title: "MRQ Deployments", + addBtn: "MRQ Deployment", + isLogsRequired: false, + }, }; const endpointType = { diff --git a/frontend/src/pages/DeploymentsPage.jsx b/frontend/src/pages/DeploymentsPage.jsx index f139c2ab3..e5f2ef5aa 100644 --- a/frontend/src/pages/DeploymentsPage.jsx +++ b/frontend/src/pages/DeploymentsPage.jsx @@ -5,7 +5,7 @@ import { Pipelines } from "../components/pipelines-or-deployments/pipelines/Pipe function DeploymentsPage({ type }) { if (type === "api") { - return ; + return ; } else { return ; } diff --git a/frontend/src/routes/Router.jsx b/frontend/src/routes/Router.jsx index b7ad85c4b..656f13d63 100644 --- a/frontend/src/routes/Router.jsx +++ b/frontend/src/routes/Router.jsx @@ -39,6 +39,7 @@ let ManualReviewPage; let ReviewLayout; let PublicPromptStudioHelper; let ManualReviewSettings; +let ManualReviewDeployment; try { TrialRoutes = @@ -69,6 +70,8 @@ try { require("../plugins/manual-review/review-layout/ReviewLayout.jsx").ReviewLayout; ManualReviewSettings = require("../plugins/manual-review/settings/Settings.jsx").ManualReviewSettings; + ManualReviewDeployment = + require("../plugins/manual-review/components/deployment/ManualReviewDeployment.jsx").ManualReviewDeployment; } catch (err) { // Do nothing, Not-found Page will be triggered. } @@ -163,6 +166,12 @@ function Router() { {AppDeployments && ( } /> )} + {ManualReviewDeployment && ( + } + /> + )} } /> }> } /> From 4a15d246a87fe15f9811da8c7b880a222078d783 Mon Sep 17 00:00:00 2001 From: vishnuszipstack Date: Thu, 29 Aug 2024 10:24:21 +0530 Subject: [PATCH 2/5] removed un related migration --- .../0004_alter_pipeline_cron_string.py | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 backend/pipeline/migrations/0004_alter_pipeline_cron_string.py diff --git a/backend/pipeline/migrations/0004_alter_pipeline_cron_string.py b/backend/pipeline/migrations/0004_alter_pipeline_cron_string.py deleted file mode 100644 index 305fc0125..000000000 --- a/backend/pipeline/migrations/0004_alter_pipeline_cron_string.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 4.2.1 on 2024-08-13 06:28 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("pipeline", "0003_alter_pipeline_active_alter_pipeline_cron_string_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="pipeline", - name="cron_string", - field=models.TextField( - blank=True, db_comment="UNIX cron string", max_length=256, null=True - ), - ), - ] From 998d391af854a135fc1e838b8c96e2afe9af6697 Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Fri, 30 Aug 2024 01:50:41 +0530 Subject: [PATCH 3/5] etl endpoint fix for manual review --- backend/pipeline/models.py | 10 ++++++++++ frontend/src/components/agency/actions/Actions.jsx | 13 ++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/pipeline/models.py b/backend/pipeline/models.py index 7532885f4..3d8595c8f 100644 --- a/backend/pipeline/models.py +++ b/backend/pipeline/models.py @@ -4,6 +4,7 @@ from django.conf import settings from django.db import connection, models from utils.models.base_model import BaseModel +from workflow_manager.endpoint.models import WorkflowEndpoint from workflow_manager.workflow.models.workflow import Workflow from backend.constants import FieldLengthConstants as FieldLength @@ -93,6 +94,15 @@ def api_key_data(self): def api_endpoint(self): org_schema = connection.get_tenant().schema_name deployment_endpoint = settings.API_DEPLOYMENT_PATH_PREFIX + "/pipeline/api" + + # Check if the WorkflowEndpoint has a connection_type of MANUALREVIEW + workflow_endpoint = WorkflowEndpoint.objects.filter( + workflow=self.workflow, + connection_type=WorkflowEndpoint.ConnectionType.MANUALREVIEW, + ).first() + if workflow_endpoint: + deployment_endpoint = "mr/api" + api_endpoint = f"{deployment_endpoint}/{org_schema}/{self.id}/" return api_endpoint diff --git a/frontend/src/components/agency/actions/Actions.jsx b/frontend/src/components/agency/actions/Actions.jsx index bfde51b8d..fc12ca31a 100644 --- a/frontend/src/components/agency/actions/Actions.jsx +++ b/frontend/src/components/agency/actions/Actions.jsx @@ -63,7 +63,7 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) { const axiosPrivate = useAxiosPrivate(); const { posthogWfDeploymentEventText, setPostHogCustomEvent } = usePostHogEvents(); - + const isManualReview = destination?.connection_type === "MANUALREVIEW"; useEffect(() => { // Enable Deploy as API only when // Source & Destination connection_type are selected as API @@ -84,7 +84,7 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) { source?.connector_instance && ((destination?.connection_type === "DATABASE" && destination.connector_instance) || - destination.connection_type === "MANUALREVIEW") + isManualReview) ); }, [source, destination]); useEffect(() => { @@ -307,10 +307,7 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) { ) { return false; } - if ( - source?.connection_type === "FILESYSTEM" && - destination?.connection_type === "MANUALREVIEW" - ) { + if (source?.connection_type === "FILESYSTEM" && isManualReview) { return false; } return !source?.connector_instance || !destination?.connector_instance; @@ -497,7 +494,9 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) { - +