From 03d16f16c8f0d5ce854f00fa3395e155e73e3832 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 29 Nov 2024 15:23:48 +0530 Subject: [PATCH 1/2] change API to delete based on host --- .../java/com/akto/action/InternalAction.java | 109 +++++++++++------- apps/dashboard/src/main/resources/struts.xml | 2 +- .../java/com/akto/dao/SingleTypeInfoDao.java | 8 +- 3 files changed, 68 insertions(+), 51 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/InternalAction.java b/apps/dashboard/src/main/java/com/akto/action/InternalAction.java index fc59ae3cc9..d5c341bb63 100644 --- a/apps/dashboard/src/main/java/com/akto/action/InternalAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/InternalAction.java @@ -1,34 +1,32 @@ package com.akto.action; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import org.bson.Document; import org.bson.conversions.Bson; +import com.akto.dao.ApiCollectionsDao; import com.akto.dao.SingleTypeInfoDao; import com.akto.dao.context.Context; +import com.akto.dto.ApiCollection; import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.User; import com.akto.dto.traffic.Key; -import com.akto.dto.type.SingleTypeInfo; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.usage.UsageMetricCalculator; -import com.mongodb.client.model.Filters; public class InternalAction extends UserAction { private static final LoggerMaker loggerMaker = new LoggerMaker(InternalAction.class, LogDb.DASHBOARD); - String headerKey; boolean actuallyDelete; int count; - public String deleteApisBasedOnHeader() { - - if (headerKey == null || headerKey.isEmpty()) { - addActionError("Invalid header key"); - return ERROR.toUpperCase(); - } + public String deleteApisWithoutHost() { User user = getSUser(); if (user.getLogin() != null && !user.getLogin().contains("@akto")) { @@ -36,48 +34,73 @@ public String deleteApisBasedOnHeader() { return ERROR.toUpperCase(); } - int time = Context.now(); - Bson filter = Filters.and(Filters.eq(SingleTypeInfo._PARAM, headerKey), - UsageMetricCalculator.excludeDemosAndDeactivated(SingleTypeInfo._API_COLLECTION_ID)); - loggerMaker.infoAndAddToDb("Executing deleteApisBasedOnHeader find query"); - List apiList = SingleTypeInfoDao.instance.fetchEndpointsInCollection(filter); + List apiCollections = ApiCollectionsDao.fetchAllHosts(); - int delta = Context.now() - time; - loggerMaker.infoAndAddToDb("Finished deleteApisBasedOnHeader find query " + delta); + Set keys = new HashSet<>(); + int time = Context.nowInMillis(); + int delta = Context.nowInMillis(); + Set demosAndDeactivated = UsageMetricCalculator.getDemosAndDeactivated(); + for (ApiCollection apiCollection : apiCollections) { - if (apiList != null && !apiList.isEmpty()) { - - List keys = new ArrayList<>(); - for (ApiInfoKey apiInfoKey : apiList) { - loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader " + apiInfoKey.toString()); - keys.add(new Key(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod(), -1, 0, - 0)); + + int apiCollectionId = apiCollection.getId(); + if (demosAndDeactivated.contains(apiCollectionId)) { + loggerMaker.infoAndAddToDb("Skipping deleteApisBasedOnHeader for apiCollectionId " + apiCollectionId); } - - count = apiList.size(); - if (actuallyDelete) { - try { - time = Context.now(); - loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleting APIs"); - com.akto.utils.Utils.deleteApis(keys); - delta = Context.now() - time; - loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleted APIs " + delta); - } catch (Exception e) { - e.printStackTrace(); - addActionError("Error deleting APIs"); - return ERROR.toUpperCase(); + + List pipeline = Arrays.asList(new Document("$match", + new Document("apiCollectionId", apiCollectionId)), + new Document("$group", + new Document("_id", + new Document("apiCollectionId", "$apiCollectionId") + .append("url", "$url") + .append("method", "$method")) + .append("hasHostParam", + new Document("$max", + new Document("$cond", + Arrays.asList(new Document("$eq", + Arrays.asList("$param", "host")), 1L, 0L)))) + .append("documents", + new Document("$push", "$$ROOT"))), + new Document("$match", + new Document("hasHostParam", + new Document("$eq", 0L))), + new Document("$project", + new Document("_id", 1L))); + + time = Context.nowInMillis(); + loggerMaker.infoAndAddToDb("Executing deleteApisBasedOnHeader find query " + apiCollectionId); + List apiList = SingleTypeInfoDao.instance.processPipelineForEndpoint(pipeline); + delta = Context.nowInMillis() - time; + loggerMaker.infoAndAddToDb("Finished deleteApisBasedOnHeader find query " + delta + " " + apiCollectionId); + if (apiList != null && !apiList.isEmpty()) { + for (ApiInfoKey apiInfoKey : apiList) { + loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader " + apiInfoKey.toString()); + keys.add( + new Key(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod(), -1, 0, + 0)); } } } - return SUCCESS.toUpperCase(); - } - public String getHeaderKey() { - return headerKey; - } + count = keys.size(); + if (keys != null && !keys.isEmpty() && actuallyDelete) { + try { + time = Context.nowInMillis(); + loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleting APIs"); + List keyList = new ArrayList<>(); + keyList.addAll(keys); + com.akto.utils.Utils.deleteApis(keyList); + delta = Context.nowInMillis() - time; + loggerMaker.infoAndAddToDb("deleteApisBasedOnHeader deleted APIs " + delta); + } catch (Exception e) { + e.printStackTrace(); + addActionError("Error deleting APIs"); + return ERROR.toUpperCase(); + } + } - public void setHeaderKey(String headerKey) { - this.headerKey = headerKey; + return SUCCESS.toUpperCase(); } public boolean getActuallyDelete() { diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 611bd05a8f..4cf38704bb 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -2060,7 +2060,7 @@ - + diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 8a32ebc9e0..54dd13c3a8 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -365,12 +365,6 @@ public List fetchEndpointsInCollection(int apiCollectionId) return processPipelineForEndpoint(pipeline); } - public List fetchEndpointsInCollection(Bson filter) { - List pipeline = getPipelineForEndpoints(filter); - pipeline.add(Aggregates.limit(SingleTypeInfoDao.LARGE_LIMIT)); - return processPipelineForEndpoint(pipeline); - } - public List fetchEndpointsInCollection(Method method) { Bson filter = null; if (method == null) { @@ -412,7 +406,7 @@ private List getPipelineForEndpoints(Bson matchCriteria) { return pipeline; } - private List processPipelineForEndpoint(List pipeline){ + public List processPipelineForEndpoint(List pipeline){ MongoCursor endpointsCursor = instance.getMCollection().aggregate(pipeline, BasicDBObject.class).cursor(); List endpoints = new ArrayList<>(); From 55d33330ac08df5894ae5ebb58b5f5b9520ac853 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Sat, 30 Nov 2024 00:03:58 +0530 Subject: [PATCH 2/2] add timestamp filter --- .../main/java/com/akto/action/InternalAction.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/InternalAction.java b/apps/dashboard/src/main/java/com/akto/action/InternalAction.java index d5c341bb63..b1f3ef6c76 100644 --- a/apps/dashboard/src/main/java/com/akto/action/InternalAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/InternalAction.java @@ -25,6 +25,7 @@ public class InternalAction extends UserAction { boolean actuallyDelete; int count; + int timestamp; public String deleteApisWithoutHost() { @@ -46,10 +47,14 @@ public String deleteApisWithoutHost() { int apiCollectionId = apiCollection.getId(); if (demosAndDeactivated.contains(apiCollectionId)) { loggerMaker.infoAndAddToDb("Skipping deleteApisBasedOnHeader for apiCollectionId " + apiCollectionId); + continue; } List pipeline = Arrays.asList(new Document("$match", - new Document("apiCollectionId", apiCollectionId)), + new Document("$and", Arrays.asList(new Document("$or", Arrays.asList( + new Document("lastSeen", new Document("$gt", timestamp)), + new Document("timestamp", new Document("$gt", timestamp)))), + new Document("apiCollectionId", apiCollectionId)))), new Document("$group", new Document("_id", new Document("apiCollectionId", "$apiCollectionId") @@ -118,4 +123,12 @@ public int getCount() { public void setCount(int count) { this.count = count; } + + public int getTimestamp() { + return timestamp; + } + + public void setTimestamp(int timestamp) { + this.timestamp = timestamp; + } }