From 31bce18fadc0b60ea4268258f8065bc5caef651e Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 14 Feb 2025 18:58:33 +0530 Subject: [PATCH] testing using source code --- .../com/akto/action/CodeAnalysisAction.java | 5 +- .../com/akto/action/DependencyAction.java | 65 ++++++-- .../DependencyTable/DependencyTable.js | 62 +++++-- .../testing/DependencyTable/EditModal.js | 1 + .../testing/DependencyTable/EditTextField.js | 2 +- .../src/apps/dashboard/pages/testing/api.js | 5 +- .../pages/testing/user_config/UserConfig.jsx | 2 + .../com/akto/test_editor/execution/Build.java | 58 +++++-- .../src/main/java/com/akto/testing/Main.java | 2 +- .../dao/CodeAnalysisSingleTypeInfoDao.java | 47 +++++- .../java/com/akto/dao/SingleTypeInfoDao.java | 14 +- .../src/main/java/com/akto/dto/RawApi.java | 3 +- .../com/akto/dto/dependency_flow/KVPair.java | 2 +- .../akto/util/HttpRequestResponseUtils.java | 6 + .../akto/util/grpc/ParameterTransformer.java | 153 ++++++++++++++++++ .../com/akto/util/grpc/ProtoBufUtils.java | 49 +++++- .../util/grpc/TestParameterTransformer.java | 69 ++++++++ .../akto/utils/grpc/TestProtobufUtils.java | 13 ++ 18 files changed, 501 insertions(+), 57 deletions(-) create mode 100644 libs/dao/src/main/java/com/akto/util/grpc/ParameterTransformer.java create mode 100644 libs/dao/src/test/java/com/akto/util/grpc/TestParameterTransformer.java diff --git a/apps/dashboard/src/main/java/com/akto/action/CodeAnalysisAction.java b/apps/dashboard/src/main/java/com/akto/action/CodeAnalysisAction.java index e9a26acaf9..0d94e0eaef 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CodeAnalysisAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CodeAnalysisAction.java @@ -3,6 +3,7 @@ import java.net.URI; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -305,7 +306,9 @@ public String syncExtractedAPIs() { singleTypeInfos.addAll(generateSTIsFromPayload(apiCollection.getId(), codeAnalysisApi.getEndpoint(), codeAnalysisApi.getMethod(), requestBody, -1)); singleTypeInfos.addAll(generateSTIsFromPayload(apiCollection.getId(), codeAnalysisApi.getEndpoint(), codeAnalysisApi.getMethod(), responseBody, 200)); - Bson update = Updates.combine(Updates.max(SingleTypeInfo.LAST_SEEN, now), Updates.setOnInsert("timestamp", now)); + Bson update = Updates.combine(Updates.max(SingleTypeInfo.LAST_SEEN, now), + Updates.setOnInsert("timestamp", now), + Updates.set(SingleTypeInfo._COLLECTION_IDS, Arrays.asList(apiCollection.getId()))); for (SingleTypeInfo singleTypeInfo: singleTypeInfos) { bulkUpdatesSTI.add( diff --git a/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java b/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java index c9efb08fc1..00ae28c95a 100644 --- a/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/DependencyAction.java @@ -1,25 +1,20 @@ package com.akto.action; -import com.akto.DaoInit; import com.akto.dao.*; import com.akto.dao.context.Context; import com.akto.dto.ApiCollection; import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; -import com.akto.dto.OriginalHttpResponse; import com.akto.dto.dependency_flow.*; import com.akto.dto.traffic.SampleData; -import com.akto.dto.type.APICatalog; import com.akto.dto.type.URLMethods; -import com.akto.dto.type.URLMethods.Method; import com.akto.log.LoggerMaker; +import com.akto.log.LoggerMaker.LogDb; import com.akto.runtime.RelationshipSync; import com.akto.test_editor.execution.Build; import com.akto.utils.Utils; import com.mongodb.BasicDBObject; -import com.mongodb.ConnectionString; import com.mongodb.client.model.*; -import org.apache.logging.log4j.util.Strings; import org.bson.conversions.Bson; import java.util.*; @@ -35,7 +30,7 @@ public class DependencyAction extends UserAction { private Collection result; - private static final LoggerMaker loggerMaker = new LoggerMaker(DependencyAction.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(DependencyAction.class,LogDb.DASHBOARD); private boolean dependencyGraphExists = false; public String checkIfDependencyGraphAvailable() { @@ -69,7 +64,6 @@ public String execute() { private int total; private int skip; - public String buildDependencyTable() { List nodes = DependencyFlowNodesDao.instance.findNodesForCollectionIds(apiCollectionIds,false, skip, 50); dependencyTableList = new ArrayList<>(); @@ -79,6 +73,12 @@ public String buildDependencyTable() { apiInfoKeys.add(new ApiInfo.ApiInfoKey(Integer.parseInt(node.getApiCollectionId()), node.getUrl(), URLMethods.Method.fromString(node.getMethod()))); } Map> parametersMap = SingleTypeInfoDao.instance.fetchRequestParameters(apiInfoKeys); + Map> sourceCodeParametersMap = CodeAnalysisSingleTypeInfoDao.instance.fetchRequestParameters(apiInfoKeys); + + // Add parameters from source code, if any. + if (sourceCodeParametersMap != null && !sourceCodeParametersMap.isEmpty()) { + parametersMap.putAll(sourceCodeParametersMap); + } replaceDetails = ReplaceDetailsDao.instance.findAll(Filters.in(ReplaceDetail._API_COLLECTION_ID, apiCollectionIds)); @@ -97,17 +97,36 @@ public String buildDependencyTable() { private int newCollectionId; private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + + private boolean sourceCodeApis; + public String invokeDependencyTable() { - ApiCollectionsAction apiCollectionsAction = new ApiCollectionsAction(); - apiCollectionsAction.setCollectionName("temp " + Context.now()); - apiCollectionsAction.createCollection(); - List apiCollections = apiCollectionsAction.getApiCollections();; - if (apiCollections == null || apiCollections.size() == 0) { - addActionError("Couldn't create collection"); + if(apiCollectionIds== null || apiCollectionIds.isEmpty()){ + addActionError("No API collections to invoke dependency graph"); return ERROR.toUpperCase(); } - newCollectionId = apiCollections.get(0).getId(); + + if (sourceCodeApis && apiCollectionIds.size() > 1) { + addActionError("Please use a single API collection ID with source code APIs"); + return ERROR.toUpperCase(); + } + + if(!sourceCodeApis){ + ApiCollectionsAction apiCollectionsAction = new ApiCollectionsAction(); + apiCollectionsAction.setCollectionName("temp " + Context.now()); + apiCollectionsAction.createCollection(); + List apiCollections = apiCollectionsAction.getApiCollections();; + + if (apiCollections == null || apiCollections.size() == 0) { + addActionError("Couldn't create collection"); + return ERROR.toUpperCase(); + } + newCollectionId = apiCollections.get(0).getId(); + } else { + // Insert in original collection for source code APIs. + newCollectionId = apiCollectionIds.get(0); + } int accountId = Context.accountId.get(); @@ -121,7 +140,7 @@ public void run() { for (ReplaceDetail replaceDetail: replaceDetailsFromDb) { replaceDetailMap.put(replaceDetail.hashCode(), replaceDetail); } - List runResults = build.run(apiCollectionIds, modifyHostDetails, replaceDetailMap); + List runResults = build.run(apiCollectionIds, modifyHostDetails, replaceDetailMap, sourceCodeApis); List messages = new ArrayList<>(); for (Build.RunResult runResult: runResults) { @@ -129,6 +148,11 @@ public void run() { messages.add(currentMessage); } + if(messages.isEmpty()){ + loggerMaker.infoAndAddToDb("No messages found for invokeDependencyTable"); + return; + } + try { Utils.pushDataToKafka(newCollectionId, "", messages, new ArrayList<>(), true, true); } catch (Exception e) { @@ -313,5 +337,12 @@ public boolean getDependencyGraphExists() { return dependencyGraphExists; } - + public boolean getSourceCodeApis() { + return sourceCodeApis; + } + + public void setSourceCodeApis(boolean sourceCodeApis) { + this.sourceCodeApis = sourceCodeApis; + } + } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js index 53132ae0f5..4e0070e4bb 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/DependencyTable.js @@ -1,5 +1,5 @@ -import { Box, Button, HorizontalStack, Icon, Link, Modal, Select, Spinner, Text, TextField, VerticalStack } from "@shopify/polaris"; -import { useEffect, useRef, useState } from "react"; +import { Button, HorizontalStack, Icon, Link, Popover, Spinner, Text } from "@shopify/polaris"; +import { useState } from "react"; import PageWithMultipleCards from "../../../components/layouts/PageWithMultipleCards"; import GithubServerTable from "../../../components/tables/GithubServerTable"; import { CellType } from "../../../components/tables/rows/GithubRow"; @@ -58,6 +58,7 @@ function DependencyTable() { const [runResults, setRunResults] = useState({}) const [refresh, setRefresh] = useState(false) const [invokeLoading, setInvokeLoading] = useState(false) + const [invokeLoadingSecond, setInvokeLoadingSecond] = useState(false) const [active, setActive] = useState(false); const [editApiCollectionId, setEditApiCollectionId] = useState(null) @@ -113,6 +114,10 @@ function DependencyTable() { }) }) + res = res.sort((a, b) => { + return a.childParam.localeCompare(b.childParam) + }) + return res } @@ -194,6 +199,15 @@ function DependencyTable() { setEditData(newEditData); } + function isBoolean(value) { + return ( + typeof value === "boolean" || + (typeof value === "string" && (value.toLowerCase() === "true" || value.toLowerCase() === "false")) + ); + } + + const isInvalidNumber = (value) => value.trim() === "" || isNaN(value); + const convertDataToKVPairList = (data) => { let kvPairs = [] data.forEach((x) => { @@ -203,7 +217,7 @@ function DependencyTable() { "isHeader": x["childParamIsHeader"], "isUrlParam": x["childParamIsUrlParam"], "value": x["value"], - "type": "STRING" + "type": isInvalidNumber(x["value"]) ? (isBoolean(x["value"]) ? "BOOLEAN" : "STRING") : "INTEGER" }) }) @@ -253,10 +267,10 @@ function DependencyTable() { const components = [resultTable, modalComponent, globalVarModalComponent] - const invokeDependencyTable = () => { - if (invokeLoading) return - setInvokeLoading(true) - api.invokeDependencyTable(apiCollectionIds).then((resp) => { + const invokeDependencyTable = (sourceCodeApis, updateFunc) => { + if (invokeLoading || invokeLoadingSecond) return + updateFunc(true) + api.invokeDependencyTable(apiCollectionIds, sourceCodeApis).then((resp) => { let newCollectionId = resp["newCollectionId"] // let temp = {} // runResultList.forEach((runResult) => { @@ -264,10 +278,12 @@ function DependencyTable() { // temp[apiInfoKey["method"] + " " + apiInfoKey["url"]] = runResult // }) - setInvokeLoading(false) + updateFunc(false) // setRunResults(temp) // setRefresh(!refresh) + if(!sourceCodeApis){ + const url = "/dashboard/observe/inventory/" + newCollectionId const forwardLink = ( @@ -279,13 +295,37 @@ function DependencyTable() { ) func.setToast(true, false, forwardLink) + } }) } + const [moreActions, setMoreActions] = useState(false) + const secondaryActionsComponent = ( - + setMoreActions(!moreActions)} disclosure removeUnderline> + Invoke + + )} + autofocusTarget="first-node" + onClose={() => { setMoreActions(false) }} + preferredAlignment="right" + > + + + + + + + + + ) const globalVarsComponent = ( diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditModal.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditModal.js index 5d11e96f1d..f62f7ea51e 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditModal.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditModal.js @@ -17,6 +17,7 @@ function EditModal(props) { content: 'Save', onAction: () => {saveEditData(editData)}, }} + large={true} > diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditTextField.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditTextField.js index 87d95c5ba5..582009c056 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditTextField.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/DependencyTable/EditTextField.js @@ -23,7 +23,7 @@ function EditTextField(ele, modifyEditData) { onChange={(newVal) => { handleTextFieldChange(newVal) }} autoComplete="off" connectedLeft={ - + } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js index f89ffdd48e..4c91333421 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/api.js @@ -318,12 +318,13 @@ export default { data: {} }) }, - invokeDependencyTable(apiCollectionIds){ + invokeDependencyTable(apiCollectionIds, sourceCodeApis){ return request({ url: '/api/invokeDependencyTable', method: 'post', data: { - apiCollectionIds + apiCollectionIds, + sourceCodeApis } }) }, diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/UserConfig.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/UserConfig.jsx index 86c4e06811..1613fa4866 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/UserConfig.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/testing/user_config/UserConfig.jsx @@ -69,8 +69,10 @@ function UserConfig() { async function addOrUpdateScript() { if (preRequestScript.id) { api.updateScript(preRequestScript.id, preRequestScript.javascript) + func.setToast(true, false, "Pre-request script updated") } else { api.addScript(preRequestScript) + func.setToast(true, false, "Pre-request script added") } } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java index 275e481029..9653d84243 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -17,13 +17,13 @@ import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; import com.akto.util.JSONUtils; +import com.akto.util.grpc.ParameterTransformer; import com.akto.util.modifier.SetValueModifier; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.mongodb.client.model.Filters; import joptsimple.internal.Strings; import org.bson.conversions.Bson; - import java.net.URI; import java.util.*; @@ -152,20 +152,27 @@ public boolean getIsSuccess() { public void setIsSuccess(boolean success) { this.success = success; } - - } Set apisReplayedSet = new HashSet<>(); - public static List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap, Map parentToChildMap, Set apisReplayedSet) { + public static List runPerLevel(List sdList, Map modifyHostDetailMap, Map replaceDetailsMap, Map parentToChildMap, Set apisReplayedSet, boolean sourceCodeApis) { List runResults = new ArrayList<>(); for (SampleData sampleData: sdList) { Key id = sampleData.getId(); try { List samples = sampleData.getSamples(); - if (samples.isEmpty()) continue;; + ReplaceDetail replaceDetail = replaceDetailsMap.get(Objects.hash(id.getApiCollectionId(), id.getUrl(), id.getMethod().name())); + boolean usedReplaceDetail = false; + if (sourceCodeApis) { + usedReplaceDetail = true; + samples.addAll(ParameterTransformer.createSampleUsingReplaceDetails(id, replaceDetail.getKvPairs())); + } + + if (samples.isEmpty()) continue; + + // Use latest sample. + String sample = samples.get(samples.size()-1); - String sample = samples.get(0); OriginalHttpRequest request = new OriginalHttpRequest(); request.buildFromSampleMessage(sample); String newHost = findNewHost(request, modifyHostDetailMap); @@ -174,8 +181,11 @@ public static List runPerLevel(List sdList, Map(), new ArrayList<>(), null,newHost, null); @@ -186,8 +196,19 @@ public static List runPerLevel(List sdList, Map run(List apiCollectionsIds, List modifyHostDetails, Map replaceDetailsMap) { + public List run(List apiCollectionsIds, List modifyHostDetails, Map replaceDetailsMap, boolean sourceCodeApis) { if (replaceDetailsMap == null) replaceDetailsMap = new HashMap<>(); if (modifyHostDetails == null) modifyHostDetails = new ArrayList<>(); @@ -242,7 +263,6 @@ public List run(List apiCollectionsIds, List nodes = DependencyFlowNodesDao.instance.findNodesForCollectionIds(apiCollectionsIds,false,0, 10_000); buildParentToChildMap(nodes, parentToChildMap); Map> levelsToSampleDataMap = buildLevelsToSampleDataMap(nodes); @@ -251,12 +271,20 @@ public List run(List apiCollectionsIds, List sdList =levelsToSampleDataMap.get(level); - sdList = fillSdList(sdList); - if (sdList.isEmpty()) continue; + List sdListDb = fillSdList(sdList); + if (sdListDb != null && !sdListDb.isEmpty()) { + sdList = sdListDb; + } + /* + * For source code APIs, the sample data is filled in the next step. + */ + if (sdList.isEmpty() && !sourceCodeApis){ + continue; + } loggerMaker.infoAndAddToDb("Running level: " + level, LoggerMaker.LogDb.DASHBOARD); try { - List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); + List runResultsPerLevel = runPerLevel(sdList, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet, sourceCodeApis); runResults.addAll(runResultsPerLevel); loggerMaker.infoAndAddToDb("Finished running level " + level, LoggerMaker.LogDb.DASHBOARD); } catch (Exception e) { @@ -277,7 +305,7 @@ public List run(List apiCollectionsIds, List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet); + List runResultsAll = runPerLevel(filtered, modifyHostDetailMap, replaceDetailsMap, parentToChildMap, apisReplayedSet, sourceCodeApis); runResults.addAll(runResultsAll); skip += limit; if (all.size() < limit) break; diff --git a/apps/testing/src/main/java/com/akto/testing/Main.java b/apps/testing/src/main/java/com/akto/testing/Main.java index ba0c9b7b49..c2715b6e75 100644 --- a/apps/testing/src/main/java/com/akto/testing/Main.java +++ b/apps/testing/src/main/java/com/akto/testing/Main.java @@ -321,7 +321,7 @@ public void run() { // Pause for 1 second try { - Thread.sleep(1000); + Thread.sleep(15000); } catch (InterruptedException e) { System.err.println("Memory monitor thread interrupted: " + e.getMessage()); break; // Exit the loop if thread is interrupted diff --git a/libs/dao/src/main/java/com/akto/dao/CodeAnalysisSingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/CodeAnalysisSingleTypeInfoDao.java index 477d9d914e..43b9a3b5b6 100644 --- a/libs/dao/src/main/java/com/akto/dao/CodeAnalysisSingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/CodeAnalysisSingleTypeInfoDao.java @@ -1,8 +1,19 @@ package com.akto.dao; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.bson.conversions.Bson; + +import com.akto.dto.ApiInfo; import com.akto.dto.type.SingleTypeInfo; +import com.akto.dto.type.URLMethods; +import com.mongodb.BasicDBObject; +import com.mongodb.client.MongoCursor; -public class CodeAnalysisSingleTypeInfoDao extends AccountsContextDao { +public class CodeAnalysisSingleTypeInfoDao extends AccountsContextDaoWithRbac { public static final CodeAnalysisSingleTypeInfoDao instance = new CodeAnalysisSingleTypeInfoDao(); @@ -15,4 +26,38 @@ public String getCollName() { public Class getClassT() { return SingleTypeInfo.class; } + + public Map> fetchRequestParameters(List apiInfoKeys) { + Map> result = new HashMap<>(); + if (apiInfoKeys == null || apiInfoKeys.isEmpty()) return result; + + /* + * Since singleTypeInfo is the implemented class for this and SingleTypeInfoDao. + * reusing the filters here. + */ + List pipeline = SingleTypeInfoDao.instance.createPipelineForFetchRequestParams(apiInfoKeys); + + MongoCursor stiCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); + while (stiCursor.hasNext()) { + BasicDBObject next = stiCursor.next(); + BasicDBObject id = (BasicDBObject) next.get("_id"); + Object paramsObj = next.get("params"); + List params = new ArrayList<>(); + if (paramsObj instanceof List) { + for (Object param : (List) paramsObj) { + if (param instanceof String) { + params.add((String) param); + } + } + } + ApiInfo.ApiInfoKey apiInfoKey = new ApiInfo.ApiInfoKey(id.getInt("apiCollectionId"), id.getString("url"), URLMethods.Method.fromString(id.getString("method"))); + result.put(apiInfoKey, params); + } + return result; + } + + @Override + public String getFilterKeyString() { + return SingleTypeInfo._API_COLLECTION_ID; + } } diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index a4b89ebf2d..12cbaf1f64 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -598,10 +598,7 @@ public int countEndpoints(Bson filters) { return ret; } - public Map> fetchRequestParameters(List apiInfoKeys) { - Map> result = new HashMap<>(); - if (apiInfoKeys == null || apiInfoKeys.isEmpty()) return result; - + public List createPipelineForFetchRequestParams(List apiInfoKeys){ List pipeline = new ArrayList<>(); List filters = new ArrayList<>(); @@ -639,7 +636,15 @@ public Map> fetchRequestParameters(List> fetchRequestParameters(List apiInfoKeys) { + Map> result = new HashMap<>(); + if (apiInfoKeys == null || apiInfoKeys.isEmpty()) return result; + List pipeline = createPipelineForFetchRequestParams(apiInfoKeys); MongoCursor stiCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); while (stiCursor.hasNext()) { @@ -649,7 +654,6 @@ public Map> fetchRequestParameters(List> public static String convertGRPCEncodedToJson(byte[] rawRequest) { String base64 = Base64.getEncoder().encodeToString(rawRequest); + + // empty grpc response, only headers present + if (rawRequest.length <= 5) { + return "{}"; + } + try { Map map = ProtoBufUtils.getInstance().decodeProto(rawRequest); if (map.isEmpty()) { diff --git a/libs/dao/src/main/java/com/akto/util/grpc/ParameterTransformer.java b/libs/dao/src/main/java/com/akto/util/grpc/ParameterTransformer.java new file mode 100644 index 0000000000..3657ba682b --- /dev/null +++ b/libs/dao/src/main/java/com/akto/util/grpc/ParameterTransformer.java @@ -0,0 +1,153 @@ +package com.akto.util.grpc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.akto.dao.CodeAnalysisSingleTypeInfoDao; +import com.akto.dao.context.Context; +import com.akto.dto.ApiInfo; +import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.dependency_flow.KVPair; +import com.akto.dto.dependency_flow.KVPair.KVType; +import com.akto.dto.traffic.Key; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.*; + +public class ParameterTransformer { + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static List createSampleUsingReplaceDetails(Key id, List keyValue){ + List apiInfoKeys = new ArrayList<>(); + apiInfoKeys.add(new ApiInfo.ApiInfoKey(id.getApiCollectionId(), id.getUrl(), id.getMethod())); + Map> parametersMap = CodeAnalysisSingleTypeInfoDao.instance.fetchRequestParameters(apiInfoKeys); + List samples = new ArrayList<>(); + for(ApiInfoKey apiInfoKey: parametersMap.keySet()){ + List params = parametersMap.get(apiInfoKey); + Map paramValueMap = new HashMap<>(); + for (String param : params) { + Object value = null; + for (KVPair kv : keyValue) { + if (kv.getKey().equals(param)) { + if (KVType.INTEGER.equals(kv.getType())) { + value = Integer.parseInt(kv.getValue()); + } else if (KVType.STRING.equals(kv.getType())) { + value = kv.getValue(); + } else if (KVType.BOOLEAN.equals(kv.getType())) { + value = Boolean.valueOf(kv.getValue()); + } + continue; + } + } + if (value != null) { + paramValueMap.put(transformKey(param), value); + } + } + JsonNode requestJsonNode = transform(paramValueMap); + + Map sampleJson = new HashMap<>(); + sampleJson.put("destIp", ""); + sampleJson.put("method", id.getMethod().name()); + sampleJson.put("requestPayload", requestJsonNode.toString()); + sampleJson.put("responsePayload", "{}"); + sampleJson.put("ip", ""); + sampleJson.put("source", "HAR"); + sampleJson.put("type", "HTTP/1.1"); + sampleJson.put("path", id.getUrl()); + sampleJson.put("requestHeaders", "{}"); + sampleJson.put("responseHeaders", "{}"); + sampleJson.put("time", Context.now()+""); + sampleJson.put("statusCode", "200"); + sampleJson.put("akto_account_id", Context.accountId.get()); + try{ + samples.add(objectMapper.writeValueAsString(sampleJson)); + } catch (Exception e){ + } + } + return samples; + } + + public static JsonNode transform(Map params) { + ObjectNode root = objectMapper.createObjectNode(); + + for (Map.Entry entry : params.entrySet()) { + String[] parts = entry.getKey().split("#"); + processPath(root, parts, 0, entry.getValue()); + } + + return root; + } + + private static void processPath(JsonNode current, String[] parts, int index, Object value) { + if (index == parts.length - 1) { + if (value instanceof Integer) { + ((ObjectNode) current).put(parts[index], (Integer) value); + } else if(value instanceof Boolean){ + ((ObjectNode) current).put(parts[index], (Boolean) value); + } else { + // Default string + ((ObjectNode) current).put(parts[index], (String) value); + } + return; + } + + String part = parts[index]; + // Check if next part is array marker + boolean nextIsArray = (index + 1 < parts.length && parts[index + 1].equals("$")); + + if (nextIsArray) { + ArrayNode arrayNode; + if (current.has(parts[index]) && current.get(parts[index]).isArray()) { + arrayNode = (ArrayNode) current.get(parts[index]); + } else { + arrayNode = objectMapper.createArrayNode(); + ((ObjectNode) current).set(parts[index], arrayNode); + } + + ObjectNode newNode = objectMapper.createObjectNode(); + arrayNode.add(newNode); + // Skip the next part (the $ symbol) in recursive call + processPath(newNode, parts, index + 2, value); + } else { + + ObjectNode nextNode; + if (current.has(part) && current.get(part).isObject()) { + nextNode = (ObjectNode) current.get(part); + } else { + nextNode = objectMapper.createObjectNode(); + ((ObjectNode) current).set(part, nextNode); + } + + processPath(nextNode, parts, index + 1, value); + } + } + + public static String transformKey(String input) { + StringBuilder result = new StringBuilder(); + int i = 0; + + while (i < input.length()) { + char currentChar = input.charAt(i); + + // Check if we're at a potential type indicator + if (currentChar == '$' && i < input.length() - 1) { + // Look ahead for digits + int j = i + 1; + while (j < input.length() && Character.isDigit(input.charAt(j))) { + j++; + } + + // If we found digits after $, skip both $ and digits + if (j > i + 1) { + i = j; + continue; + } + } + + // Add the current character to result + result.append(currentChar); + i++; + } + + return result.toString(); + } +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/util/grpc/ProtoBufUtils.java b/libs/dao/src/main/java/com/akto/util/grpc/ProtoBufUtils.java index a552039f8b..cc95534260 100644 --- a/libs/dao/src/main/java/com/akto/util/grpc/ProtoBufUtils.java +++ b/libs/dao/src/main/java/com/akto/util/grpc/ProtoBufUtils.java @@ -10,6 +10,10 @@ import java.nio.charset.StandardCharsets; import java.util.*; +import static org.apache.commons.codec.binary.Base64.decodeBase64; +import static org.apache.commons.codec.binary.Base64.encodeBase64String; + + public class ProtoBufUtils { public static final String RAW_QUERY = "raw_query"; @@ -121,11 +125,52 @@ public static String base64EncodedJsonToProtobuf(String payload) throws Exceptio Map map = null; try { map = ProtoBufUtils.getInstance().mapper.readValue(payload, Map.class); + map = decodeBase64ValuesIfAny(map); } catch (Exception e) { throw new InvalidObjectException("Unable to parse payload"); } return base64EncodedJsonToProtobuf(map); } + + public static boolean isBase64Encoded(String value) { + if (value == null || value.trim().isEmpty()) { + return false; + } + try { + byte[] decodedBytes = decodeBase64(value); + String encodedAgain = encodeBase64String(decodedBytes).trim(); + return encodedAgain.equals(value.trim()) || (encodedAgain + "=").equals(value.trim()) || (encodedAgain + "==").equals(value.trim()); + } catch (Exception e) { + return false; + } + } + + /* + * This method enables us to send byte[] data which was encoded as base64. + */ + private static Map decodeBase64ValuesIfAny(Map map) { + Map processedMap = new HashMap<>(); + + for (Map.Entry entry : map.entrySet()) { + Object key = entry.getKey(); + Object value = entry.getValue(); + + // Check if the value is a string and is Base64-encoded + if (value instanceof String) { + String strValue = (String) value; + + if (isBase64Encoded(strValue)) { + processedMap.put(key, Base64.getDecoder().decode(strValue)); + } else { + processedMap.put(key, strValue); + } + } else { + processedMap.put(key, value); + } + } + return processedMap; + } + public static String base64EncodedJsonToProtobuf(Map jsonMap) throws IOException{ byte[] protobufArray = encodeJsonToProtobuf(jsonMap); byte[] FIRST_BYTES = new byte[5]; @@ -180,6 +225,8 @@ private static void encodeMapToProto(Map map, CodedOutputStream codedOutputStream.writeBytesNoTag(ByteString.copyFrom(nestedMessage)); } else if (value instanceof Float) { codedOutputStream.writeFixed32NoTag(Float.floatToIntBits((Float) value)); + } else if(value instanceof byte[]){ + codedOutputStream.writeBytesNoTag(ByteString.copyFrom((byte[]) value)); } else { throw new IOException("Unsupported type: " + value.getClass().getName()); } @@ -191,7 +238,7 @@ private static int getWireType(Object value) { return WireFormat.WIRETYPE_VARINT; } else if (value instanceof Double) { return WireFormat.WIRETYPE_FIXED64; - } else if (value instanceof String) { + } else if (value instanceof String || value instanceof byte[]) { return WireFormat.WIRETYPE_LENGTH_DELIMITED; } else if (value instanceof Map) { return WireFormat.WIRETYPE_LENGTH_DELIMITED; diff --git a/libs/dao/src/test/java/com/akto/util/grpc/TestParameterTransformer.java b/libs/dao/src/test/java/com/akto/util/grpc/TestParameterTransformer.java new file mode 100644 index 0000000000..5565cbca2e --- /dev/null +++ b/libs/dao/src/test/java/com/akto/util/grpc/TestParameterTransformer.java @@ -0,0 +1,69 @@ +package com.akto.util.grpc; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import com.fasterxml.jackson.databind.JsonNode; + +public class TestParameterTransformer { + + @Test + public void testTransform() { + Map params = new HashMap<>(); + + // Test cases + params.put("a#b", "value1"); + params.put("a#c#$#d", "value2"); + params.put("x#y#$#z", "value3"); + params.put("x#y#$#z", "value4"); + params.put("p#q#r#$#s#t", "value5"); + params.put("p#q#r#$#s#q#$#w", "value5"); + params.put("m#n", "value6"); + params.put("m#o", "value7"); + + JsonNode result = ParameterTransformer.transform(params); + + JsonNode m = result.get("m"); + assertEquals(m.get("n").toString(), "\"value6\""); + + JsonNode a = result.get("a"); + JsonNode c = a.get("c"); + JsonNode firstElement = c.get(0); + assertEquals(firstElement.get("d").toString(), "\"value2\""); + + } + + @Test + public void testTransformKey(){ + + String[] tests = { + "keys$1#$#delegatable_contract_id$8#shardNum$2", + "simple#key", + "object$3#nested$2#field$1", + "array#$#element$5", + "$1#startsWith#type", + "ends#with#type$8", + "multiple$1#$#types$2#in$3#one$4#key" + }; + + String[] actual = { + "keys#$#delegatable_contract_id#shardNum", + "simple#key", + "object#nested#field", + "array#$#element", + "#startsWith#type", + "ends#with#type", + "multiple#$#types#in#one#key" + }; + + for (int i = 0; i < tests.length; i++) { + String transformed = ParameterTransformer.transformKey(tests[i]); + assertEquals(transformed, actual[i]); + } + } + +} diff --git a/libs/dao/src/test/java/com/akto/utils/grpc/TestProtobufUtils.java b/libs/dao/src/test/java/com/akto/utils/grpc/TestProtobufUtils.java index fc9826414f..637ba3f7b9 100644 --- a/libs/dao/src/test/java/com/akto/utils/grpc/TestProtobufUtils.java +++ b/libs/dao/src/test/java/com/akto/utils/grpc/TestProtobufUtils.java @@ -4,6 +4,8 @@ import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.assertEquals; + import java.util.Map; public class TestProtobufUtils { @@ -13,4 +15,15 @@ public void testProtobufDecoder () { Map map = ProtoBufUtils.getInstance().decodeProto(str1); Assert.assertTrue("World".equals(map.get("param_1"))); } + + @Test + public void testIsBase64Encoded(){ + + String base64EncodedString = "SGVsbG8sIFdvcmxkIQ=="; + String notBase64EncodedString = "helloworld"; + + assertEquals(ProtoBufUtils.isBase64Encoded(notBase64EncodedString), false); + assertEquals(ProtoBufUtils.isBase64Encoded(base64EncodedString), true); + + } }