diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index 0cedba3078..ff30db7280 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -9,10 +9,12 @@ import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; +import com.akto.listener.RuntimeListener; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.parsers.HttpCallParser; import com.akto.util.JSONUtils; +import com.akto.utils.AccountHTTPCallParserAktoPolicyInfo; import com.akto.utils.AktoCustomException; import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; @@ -40,7 +42,7 @@ import static com.akto.utils.Utils.extractJsonResponse; public class CustomDataTypeAction extends UserAction{ - private static LoggerMaker loggerMaker = new LoggerMaker(CustomDataTypeAction.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(CustomDataTypeAction.class, LogDb.DASHBOARD); private static final ExecutorService service = Executors.newFixedThreadPool(1); private boolean createNew; @@ -507,6 +509,112 @@ public void setValue(String value) { private long totalSampleDataCount; private long currentProcessed; private static final int pageSize = 1000; + + public String resetDataTypeRetro() { + customDataType = CustomDataTypeDao.instance.findOne(CustomDataType.NAME, name); + aktoDataType = AktoDataTypeDao.instance.findOne(AktoDataType.NAME, name); + + if (customDataType == null && aktoDataType == null) { + addActionError("Data type does not exist"); + return ERROR.toUpperCase(); + } + + /* + * we find all samples which contain the data type + * and re-run runtime on the samples. + */ + + int accountId = Context.accountId.get(); + service.submit(() -> { + Context.accountId.set(accountId); + loggerMaker.infoAndAddToDb("Triggered a job to recalculate data types"); + int skip = 0; + final int LIMIT = 100; + final int BATCH_SIZE = 200; + List sampleDataList = new ArrayList<>(); + Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); + List responses = new ArrayList<>(); + this.customSubTypeMatches = new ArrayList<>(); + do { + sampleDataList = SampleDataDao.instance.findAll(Filters.empty(), skip, LIMIT, sort); + skip += LIMIT; + for(SampleData sampleData : sampleDataList){ + List samples = sampleData.getSamples(); + Set foundSet = new HashSet<>(); + for (String sample: samples) { + Key apiKey = sampleData.getId(); + try { + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + boolean skip1=false, skip2=false, skip3=false, skip4=false; + try { + skip1 = forHeaders(httpResponseParams.getHeaders(), customDataType, apiKey); + skip2 = forHeaders(httpResponseParams.requestParams.getHeaders(), customDataType, apiKey); + } catch (Exception e) { + } + try { + skip3 = forPayload(httpResponseParams.getPayload(), customDataType, apiKey); + skip4 = forPayload(httpResponseParams.requestParams.getPayload(), customDataType, apiKey); + } catch (Exception e) { + } + String key = skip1 + " " + skip2 + " " + skip3 + " " + skip4 + " " + sampleData.getId().toString(); + if ((skip1 || skip2 || skip3 || skip4) && !foundSet.contains(key)) { + foundSet.add(key); + responses.add(httpResponseParams); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + if (sampleDataList != null) { + loggerMaker.infoAndAddToDb("Read sampleData to recalculate data types " + sampleDataList.size()); + } + + if (!responses.isEmpty() && responses.size() >= BATCH_SIZE) { + loggerMaker.infoAndAddToDb( + "Starting processing responses to recalculate data types " + responses.size()); + SingleTypeInfo.fetchCustomDataTypes(accountId); + AccountHTTPCallParserAktoPolicyInfo info = RuntimeListener.accountHTTPParserMap.get(accountId); + if (info == null) { + info = new AccountHTTPCallParserAktoPolicyInfo(); + HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false); + info.setHttpCallParser(callParser); + RuntimeListener.accountHTTPParserMap.put(accountId, info); + } + AccountSettings accountSettings = AccountSettingsDao.instance + .findOne(AccountSettingsDao.generateFilter()); + responses = com.akto.runtime.Main.filterBasedOnHeaders(responses, accountSettings); + info.getHttpCallParser().syncFunction(responses, true, false, accountSettings); + info.getHttpCallParser().apiCatalogSync.buildFromDB(false, false); + loggerMaker.infoAndAddToDb("Processed responses to recalculate data types " + responses.size()); + responses.clear(); + } + + }while(sampleDataList!=null && !sampleDataList.isEmpty()); + + if (!responses.isEmpty()) { + loggerMaker.infoAndAddToDb("Starting processing responses to recalculate data types " + responses.size()); + SingleTypeInfo.fetchCustomDataTypes(accountId); + AccountHTTPCallParserAktoPolicyInfo info = RuntimeListener.accountHTTPParserMap.get(accountId); + if (info == null) { + info = new AccountHTTPCallParserAktoPolicyInfo(); + HttpCallParser callParser = new HttpCallParser("userIdentifier", 1, 1, 1, false); + info.setHttpCallParser(callParser); + RuntimeListener.accountHTTPParserMap.put(accountId, info); + } + AccountSettings accountSettings = AccountSettingsDao.instance + .findOne(AccountSettingsDao.generateFilter()); + responses = com.akto.runtime.Main.filterBasedOnHeaders(responses, accountSettings); + info.getHttpCallParser().syncFunction(responses, true, false, accountSettings); + info.getHttpCallParser().apiCatalogSync.buildFromDB(false, false); + loggerMaker.infoAndAddToDb("Processed responses to recalculate data types " + responses.size()); + } + loggerMaker.infoAndAddToDb("Finished a job to recalculate data types"); + }); + + return SUCCESS.toUpperCase(); + } + public String reviewCustomDataType() { customSubTypeMatches = new ArrayList<>(); CustomDataType customDataType; diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 929ba20edd..ee1eb16dc1 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -2566,6 +2566,26 @@ + + + + + SENSITIVE_DATA + READ_WRITE + + + 403 + false + ^actionErrors.* + + + + 422 + false + ^actionErrors.* + + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx index 2da740efaa..16d2e5295f 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/SensitiveDataExposure/SensitiveDataExposure.jsx @@ -196,8 +196,14 @@ const handleRedirect = () => { navigate("/dashboard/observe/data-types") } +const handleReset = async () => { + await api.resetDataTypeRetro(subType) + func.setToast(true, false, "Resetting data types") +} + const primaryActions = ( + dispatchCurrDateRange({type: "update", period: dateObj.period, title: dateObj.title, alias: dateObj.alias})}/> diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js index c6b9c5ba96..c5b341d5f2 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js @@ -733,5 +733,15 @@ export default { apiCollectionId: apiInfoKey.apiCollectionId } }) + }, + resetDataTypeRetro(name){ + return request({ + url: '/api/resetDataTypeRetro', + method: 'post', + data: { + name: name, + } + }) } + } \ No newline at end of file