diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java index c3a05c9c3a..83c2dca10c 100644 --- a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java @@ -57,9 +57,15 @@ public List fetchSensitiveParams() { public List fetchRecentEndpoints(int startTimestamp, int endTimestamp) { List endpoints = new ArrayList<>(); + List nonHostApiCollectionIds = ApiCollectionsDao.instance.fetchNonTrafficApiCollectionsIds(); Bson hostFilterQ = SingleTypeInfoDao.filterForHostHeader(0, false); - Bson filterQWithTs = Filters.and(Filters.gte("timestamp", startTimestamp), Filters.lte("timestamp", endTimestamp), hostFilterQ); + Bson filterQWithTs = Filters.and( + Filters.gte(SingleTypeInfo._TIMESTAMP, startTimestamp), + Filters.lte(SingleTypeInfo._TIMESTAMP, endTimestamp), + Filters.nin(SingleTypeInfo._API_COLLECTION_ID, nonHostApiCollectionIds), + hostFilterQ + ); List latestHosts = SingleTypeInfoDao.instance.findAll(filterQWithTs, 0, 1_000, Sorts.descending("timestamp"), Projections.exclude("values")); for(SingleTypeInfo sti: latestHosts) { BasicDBObject id = @@ -71,7 +77,6 @@ public List fetchRecentEndpoints(int startTimestamp, int endTimes endpoints.add(endpoint); } - List nonHostApiCollectionIds = ApiCollectionsDao.instance.fetchNonTrafficApiCollectionsIds(); if (nonHostApiCollectionIds != null && nonHostApiCollectionIds.size() > 0){ List pipeline = new ArrayList<>(); diff --git a/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java b/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java index 3e6bc319b7..0923316ee9 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java @@ -314,7 +314,7 @@ public static void addLlmSampleData(int accountId) { } - private static String convertStreamToString(InputStream in) throws Exception { + public static String convertStreamToString(InputStream in) throws Exception { BufferedReader reader = new BufferedReader(new InputStreamReader(in)); StringBuilder stringbuilder = new StringBuilder(); diff --git a/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java b/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java index 10a23f4be7..df3e4d4c7a 100644 --- a/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java +++ b/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java @@ -3,17 +3,25 @@ import com.akto.MongoBasedTest; import com.akto.dao.ApiCollectionsDao; import com.akto.dao.SingleTypeInfoDao; +import com.akto.dao.context.Context; import com.akto.dto.ApiCollection; import com.akto.dto.type.SingleTypeInfo; +import com.akto.listener.InitializerListener; +import com.akto.listener.RuntimeListener; +import com.akto.parsers.HttpCallParser; +import com.akto.runtime.policies.AktoPolicyNew; import com.akto.types.CappedSet; +import com.akto.utils.AccountHTTPCallParserAktoPolicyInfo; import com.mongodb.BasicDBObject; +import org.json.JSONArray; import org.junit.Test; import java.util.Arrays; import java.util.HashSet; +import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static com.akto.listener.RuntimeListener.convertStreamToString; +import static org.junit.Assert.*; public class TestInventoryAction extends MongoBasedTest { @@ -63,4 +71,29 @@ public void testFetchEndpointsBasedOnHostName() { assertNotNull(basicDBObject.get("url")); assertNotNull(basicDBObject.get("method")); } + + @Test + public void testFetchRecentEndpoints() throws Exception { + SingleTypeInfoDao.instance.getMCollection().drop(); + ApiCollectionsDao.instance.getMCollection().drop(); + + AccountHTTPCallParserAktoPolicyInfo info = new AccountHTTPCallParserAktoPolicyInfo(); + HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false); + info.setHttpCallParser(callParser); + info.setPolicy(new AktoPolicyNew(false)); + int accountId = Context.accountId.get(); + RuntimeListener.accountHTTPParserMap.put(accountId, info); + + + String data = convertStreamToString(InitializerListener.class.getResourceAsStream("/SampleApiData.json")); + int endpointCount = new JSONArray(data).length(); + + assertTrue(endpointCount > 0); + + RuntimeListener.addSampleData(); + + InventoryAction inventoryAction = new InventoryAction(); + List basicDBObjects = inventoryAction.fetchRecentEndpoints(0, Context.now()); + assertEquals(endpointCount, basicDBObjects.size()); + } } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java b/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java index 9fabbdfca9..fc41179d39 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java @@ -220,20 +220,56 @@ public static Boolean isWordListVariable(Object key, Map varMap) String expression = key.toString(); - expression = expression.substring(2, expression.length()); - expression = expression.substring(0, expression.length() - 1); + Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + Matcher matcher = pattern.matcher(expression); + while (matcher.find()) { + try { + String match = matcher.group(0); + match = match.substring(2, match.length()); + match = match.substring(0, match.length() - 1); + + Boolean isWordListVar = varMap.containsKey("wordList_" + match); + if (isWordListVar) return true; + } catch (Exception e) { + e.printStackTrace(); + } + } - Boolean isWordListVar = varMap.containsKey("wordList_" + expression); - return isWordListVar; + return false; } public static List resolveWordListVar(String key, Map varMap) { String expression = key.toString(); - expression = expression.substring(2, expression.length()); - expression = expression.substring(0, expression.length() - 1); + List wordList = new ArrayList<>(); + String wordListKey = null; + + Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + Matcher matcher = pattern.matcher(expression); + while (matcher.find()) { + try { + String match = matcher.group(0); + String originalKey = match; + match = match.substring(2, match.length()); + match = match.substring(0, match.length() - 1); + + Boolean isWordListVar = varMap.containsKey("wordList_" + match); + if (isWordListVar) { + wordList = (List) varMap.get("wordList_" + match); + wordListKey = originalKey; + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + List result = new ArrayList<>(); + for (String word: wordList) { + result.add(expression.replace(wordListKey, word)); + } - return (List) varMap.get("wordList_" + expression); + return result; } // public Object resolveExpression(Map varMap, String expression) { diff --git a/apps/testing/src/test/java/com/akto/test_editor/execution/TestVariableResolver.java b/apps/testing/src/test/java/com/akto/test_editor/execution/TestVariableResolver.java index edc6aac402..1c00d69517 100644 --- a/apps/testing/src/test/java/com/akto/test_editor/execution/TestVariableResolver.java +++ b/apps/testing/src/test/java/com/akto/test_editor/execution/TestVariableResolver.java @@ -2,7 +2,9 @@ import org.junit.Test; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import static junit.framework.TestCase.assertEquals; @@ -44,4 +46,45 @@ public void testResolveExpression() { assertEquals("akto", result); } + @Test + public void testResolveWordListVar() { + Map varMap = new HashMap<>(); + varMap.put("changed_body_value", "akto"); + varMap.put("randomVar", "random"); + varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/")); + String key = "${changed_body_value}${specialCharacters}${randomVar}"; + + List result = VariableResolver.resolveWordListVar(key, varMap); + assertEquals(Arrays.asList("${changed_body_value}.${randomVar}", "${changed_body_value}$${randomVar}", "${changed_body_value}/${randomVar}"), result); + + + varMap = new HashMap<>(); + varMap.put("changed_body_value", "akto"); + varMap.put("randomVar", "random"); + varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/")); + key = "asdf${specialCharacters}xyz"; + + result = VariableResolver.resolveWordListVar(key, varMap); + assertEquals(Arrays.asList("asdf.xyz", "asdf$xyz", "asdf/xyz"), result); + + varMap = new HashMap<>(); + varMap.put("changed_body_value", "akto"); + varMap.put("randomVar", "random"); + varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/")); + key = "${specialCharacters}"; + + result = VariableResolver.resolveWordListVar(key, varMap); + assertEquals(Arrays.asList(".", "$", "/"), result); + + varMap = new HashMap<>(); + varMap.put("changed_body_value", "akto"); + varMap.put("randomVar", "random"); + varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/")); + varMap.put("wordList_names", Arrays.asList(".", "$", "/")); + key = "${changed_body_value}${specialCharacters}${names}"; + + result = VariableResolver.resolveWordListVar(key, varMap); + assertEquals(Arrays.asList("${changed_body_value}.${names}", "${changed_body_value}$${names}", "${changed_body_value}/${names}"), result); + } + } diff --git a/libs/dao/src/main/java/com/akto/DaoInit.java b/libs/dao/src/main/java/com/akto/DaoInit.java index 85ccda89a5..2555d94208 100644 --- a/libs/dao/src/main/java/com/akto/DaoInit.java +++ b/libs/dao/src/main/java/com/akto/DaoInit.java @@ -229,6 +229,7 @@ public static CodecRegistry createCodecRegistry(){ new EnumCodec<>(GlobalEnums.YamlTemplateSource.class), new EnumCodec<>(AktoGptConfigState.class), new EnumCodec<>(CustomWebhook.WebhookOptions.class), + new EnumCodec<>(User.AktoUIMode.class), new EnumCodec<>(TrafficMetricsAlert.FilterType.class)); return fromRegistries(MongoClientSettings.getDefaultCodecRegistry(), pojoCodecRegistry,