Merge branch 'master' into develop

akto-api-security · Nov 30, 2023 · d1e53f4 · d1e53f4
2 parents 4098050 + 72db65f
commit d1e53f4
Show file tree

Hide file tree

Showing 6 changed files with 130 additions and 12 deletions.
diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java
@@ -57,9 +57,15 @@ public List<SingleTypeInfo> fetchSensitiveParams() {
 
     public List<BasicDBObject> fetchRecentEndpoints(int startTimestamp, int endTimestamp) {
         List<BasicDBObject> endpoints = new ArrayList<>();
+        List <Integer> nonHostApiCollectionIds = ApiCollectionsDao.instance.fetchNonTrafficApiCollectionsIds();
 
         Bson hostFilterQ = SingleTypeInfoDao.filterForHostHeader(0, false);
-        Bson filterQWithTs = Filters.and(Filters.gte("timestamp", startTimestamp), Filters.lte("timestamp", endTimestamp), hostFilterQ);
+        Bson filterQWithTs = Filters.and(
+                Filters.gte(SingleTypeInfo._TIMESTAMP, startTimestamp),
+                Filters.lte(SingleTypeInfo._TIMESTAMP, endTimestamp),
+                Filters.nin(SingleTypeInfo._API_COLLECTION_ID, nonHostApiCollectionIds),
+                hostFilterQ
+        );
         List<SingleTypeInfo> latestHosts = SingleTypeInfoDao.instance.findAll(filterQWithTs, 0, 1_000, Sorts.descending("timestamp"), Projections.exclude("values"));
         for(SingleTypeInfo sti: latestHosts) {
             BasicDBObject id = 
@@ -71,7 +77,6 @@ public List<BasicDBObject> fetchRecentEndpoints(int startTimestamp, int endTimes
             endpoints.add(endpoint);
         }
 
-        List <Integer> nonHostApiCollectionIds = ApiCollectionsDao.instance.fetchNonTrafficApiCollectionsIds();
 
         if (nonHostApiCollectionIds != null && nonHostApiCollectionIds.size() > 0){
             List<Bson> pipeline = new ArrayList<>();

diff --git a/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java b/apps/dashboard/src/main/java/com/akto/listener/RuntimeListener.java
@@ -314,7 +314,7 @@ public static void addLlmSampleData(int accountId) {
 
     }
 
-    private static String convertStreamToString(InputStream in) throws Exception {
+    public static String convertStreamToString(InputStream in) throws Exception {
 
         BufferedReader reader = new BufferedReader(new InputStreamReader(in));
         StringBuilder stringbuilder = new StringBuilder();

diff --git a/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java b/apps/dashboard/src/test/java/com/akto/action/observe/TestInventoryAction.java
@@ -3,17 +3,25 @@
 import com.akto.MongoBasedTest;
 import com.akto.dao.ApiCollectionsDao;
 import com.akto.dao.SingleTypeInfoDao;
+import com.akto.dao.context.Context;
 import com.akto.dto.ApiCollection;
 import com.akto.dto.type.SingleTypeInfo;
+import com.akto.listener.InitializerListener;
+import com.akto.listener.RuntimeListener;
+import com.akto.parsers.HttpCallParser;
+import com.akto.runtime.policies.AktoPolicyNew;
 import com.akto.types.CappedSet;
+import com.akto.utils.AccountHTTPCallParserAktoPolicyInfo;
 import com.mongodb.BasicDBObject;
+import org.json.JSONArray;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static com.akto.listener.RuntimeListener.convertStreamToString;
+import static org.junit.Assert.*;
 
 public class TestInventoryAction extends MongoBasedTest {
 
@@ -63,4 +71,29 @@ public void testFetchEndpointsBasedOnHostName() {
         assertNotNull(basicDBObject.get("url"));
         assertNotNull(basicDBObject.get("method"));
     }
+
+    @Test
+    public void testFetchRecentEndpoints() throws Exception {
+        SingleTypeInfoDao.instance.getMCollection().drop();
+        ApiCollectionsDao.instance.getMCollection().drop();
+
+        AccountHTTPCallParserAktoPolicyInfo info = new AccountHTTPCallParserAktoPolicyInfo();
+        HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false);
+        info.setHttpCallParser(callParser);
+        info.setPolicy(new AktoPolicyNew(false));
+        int accountId = Context.accountId.get();
+        RuntimeListener.accountHTTPParserMap.put(accountId, info);
+
+
+        String data = convertStreamToString(InitializerListener.class.getResourceAsStream("/SampleApiData.json"));
+        int endpointCount = new JSONArray(data).length();
+
+        assertTrue(endpointCount > 0);
+
+        RuntimeListener.addSampleData();
+
+        InventoryAction inventoryAction = new InventoryAction();
+        List<BasicDBObject> basicDBObjects = inventoryAction.fetchRecentEndpoints(0, Context.now());
+        assertEquals(endpointCount, basicDBObjects.size());
+    }
 }
diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java b/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java
@@ -220,20 +220,56 @@ public static Boolean isWordListVariable(Object key, Map<String, Object> varMap)
 
         String expression = key.toString();
 
-        expression = expression.substring(2, expression.length());
-        expression = expression.substring(0, expression.length() - 1);
+        Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}");
+        Matcher matcher = pattern.matcher(expression);
+        while (matcher.find()) {
+            try {
+                String match = matcher.group(0);
+                match = match.substring(2, match.length());
+                match = match.substring(0, match.length() - 1);
+
+                Boolean isWordListVar = varMap.containsKey("wordList_" + match);
+                if (isWordListVar) return true;
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
 
-        Boolean isWordListVar = varMap.containsKey("wordList_" + expression);
-        return isWordListVar;
+        return false;
     }
 
     public static List<String> resolveWordListVar(String key, Map<String, Object> varMap) {
         String expression = key.toString();
 
-        expression = expression.substring(2, expression.length());
-        expression = expression.substring(0, expression.length() - 1);
+        List<String> wordList = new ArrayList<>();
+        String wordListKey = null;
+
+        Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}");
+        Matcher matcher = pattern.matcher(expression);
+        while (matcher.find()) {
+            try {
+                String match = matcher.group(0);
+                String originalKey = match;
+                match = match.substring(2, match.length());
+                match = match.substring(0, match.length() - 1);
+
+                Boolean isWordListVar = varMap.containsKey("wordList_" + match);
+                if (isWordListVar) {
+                    wordList = (List<String>) varMap.get("wordList_" + match);
+                    wordListKey = originalKey;
+                    break;
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+
+        List<String> result = new ArrayList<>();
+        for (String word: wordList) {
+            result.add(expression.replace(wordListKey, word));
+        }
 
-        return (List<String>) varMap.get("wordList_" + expression);
+        return result;
     }
 
     // public Object resolveExpression(Map<String, Object> varMap, String expression) {

diff --git a/apps/testing/src/test/java/com/akto/test_editor/execution/TestVariableResolver.java b/apps/testing/src/test/java/com/akto/test_editor/execution/TestVariableResolver.java
@@ -2,7 +2,9 @@
 
 import org.junit.Test;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static junit.framework.TestCase.assertEquals;
@@ -44,4 +46,45 @@ public void testResolveExpression() {
         assertEquals("akto", result);
     }
 
+    @Test
+    public void testResolveWordListVar() {
+        Map<String, Object> varMap = new HashMap<>();
+        varMap.put("changed_body_value", "akto");
+        varMap.put("randomVar", "random");
+        varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/"));
+        String key = "${changed_body_value}${specialCharacters}${randomVar}";
+
+        List<String> result = VariableResolver.resolveWordListVar(key, varMap);
+        assertEquals(Arrays.asList("${changed_body_value}.${randomVar}", "${changed_body_value}$${randomVar}", "${changed_body_value}/${randomVar}"), result);
+
+
+        varMap = new HashMap<>();
+        varMap.put("changed_body_value", "akto");
+        varMap.put("randomVar", "random");
+        varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/"));
+        key = "asdf${specialCharacters}xyz";
+
+        result = VariableResolver.resolveWordListVar(key, varMap);
+        assertEquals(Arrays.asList("asdf.xyz", "asdf$xyz", "asdf/xyz"), result);
+
+        varMap = new HashMap<>();
+        varMap.put("changed_body_value", "akto");
+        varMap.put("randomVar", "random");
+        varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/"));
+        key = "${specialCharacters}";
+
+        result = VariableResolver.resolveWordListVar(key, varMap);
+        assertEquals(Arrays.asList(".", "$", "/"), result);
+
+        varMap = new HashMap<>();
+        varMap.put("changed_body_value", "akto");
+        varMap.put("randomVar", "random");
+        varMap.put("wordList_specialCharacters", Arrays.asList(".", "$", "/"));
+        varMap.put("wordList_names", Arrays.asList(".", "$", "/"));
+        key = "${changed_body_value}${specialCharacters}${names}";
+
+        result = VariableResolver.resolveWordListVar(key, varMap);
+        assertEquals(Arrays.asList("${changed_body_value}.${names}", "${changed_body_value}$${names}", "${changed_body_value}/${names}"), result);
+    }
+
 }
diff --git a/libs/dao/src/main/java/com/akto/DaoInit.java b/libs/dao/src/main/java/com/akto/DaoInit.java
@@ -229,6 +229,7 @@ public static CodecRegistry createCodecRegistry(){
                 new EnumCodec<>(GlobalEnums.YamlTemplateSource.class),
                 new EnumCodec<>(AktoGptConfigState.class),
                 new EnumCodec<>(CustomWebhook.WebhookOptions.class),
+                new EnumCodec<>(User.AktoUIMode.class),
                 new EnumCodec<>(TrafficMetricsAlert.FilterType.class));
 
         return fromRegistries(MongoClientSettings.getDefaultCodecRegistry(), pojoCodecRegistry,