From 551e0c88fb3f4ccd341b2a31486045151784a053 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 21 Feb 2025 01:54:39 +0530 Subject: [PATCH 1/3] add data structure and APIs for agents --- .../java/com/akto/action/AgentAction.java | 317 ++++++++++++++++++ .../src/main/resources/struts.xml | 55 +++ .../akto/dao/agents/AgentHealthCheckDao.java | 20 ++ .../java/com/akto/dao/agents/AgentRunDao.java | 22 ++ .../AgentSubProcessSingleAttemptDao.java | 22 ++ .../main/java/com/akto/dto/agents/Agent.java | 18 + .../java/com/akto/dto/agents/AgentLog.java | 31 ++ .../java/com/akto/dto/agents/AgentRun.java | 105 ++++++ .../agents/AgentSubProcessSingleAttempt.java | 171 ++++++++++ .../java/com/akto/dto/agents/HealthCheck.java | 68 ++++ .../main/java/com/akto/dto/agents/State.java | 13 + 11 files changed, 842 insertions(+) create mode 100644 apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java create mode 100644 libs/dao/src/main/java/com/akto/dao/agents/AgentHealthCheckDao.java create mode 100644 libs/dao/src/main/java/com/akto/dao/agents/AgentRunDao.java create mode 100644 libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java create mode 100644 libs/dao/src/main/java/com/akto/dto/agents/Agent.java create mode 100644 libs/dao/src/main/java/com/akto/dto/agents/AgentLog.java create mode 100644 libs/dao/src/main/java/com/akto/dto/agents/AgentRun.java create mode 100644 libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java create mode 100644 libs/dao/src/main/java/com/akto/dto/agents/HealthCheck.java create mode 100644 libs/dao/src/main/java/com/akto/dto/agents/State.java diff --git a/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java b/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java new file mode 100644 index 0000000000..9e05ad11d7 --- /dev/null +++ b/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java @@ -0,0 +1,317 @@ +package com.akto.action; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.bson.conversions.Bson; + +import com.akto.dao.agents.AgentHealthCheckDao; +import com.akto.dao.agents.AgentRunDao; +import com.akto.dao.agents.AgentSubProcessSingleAttemptDao; +import com.akto.dao.context.Context; +import com.akto.dto.agents.AgentLog; +import com.akto.dto.agents.AgentRun; +import com.akto.dto.agents.AgentSubProcessSingleAttempt; +import com.akto.dto.agents.HealthCheck; +import com.akto.dto.agents.State; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.Updates; +import com.opensymphony.xwork2.Action; +import com.opensymphony.xwork2.ActionSupport; + +public class AgentAction extends ActionSupport { + + String instanceId; + String version; + String processId; + + public String agentHealth() { + + if (instanceId == null || instanceId.isEmpty()) { + addActionError("InstanceId is empty"); + return Action.ERROR.toUpperCase(); + } + + Bson instanceFilter = Filters.eq(HealthCheck.INSTANCE_ID, instanceId); + + List updates = new ArrayList<>(); + + if (version == null || version.isEmpty()) { + addActionError("Agent Version is missing"); + return Action.ERROR.toUpperCase(); + } + + updates.add(Updates.set(HealthCheck.LAST_HEALTH_CHECK_TIMESTAMP, Context.now())); + updates.add(Updates.set(HealthCheck._VERSION, version)); + updates.add(Updates.set(HealthCheck.PROCESS_ID, processId)); + + /* + * This operation has upsert: true. + */ + AgentHealthCheckDao.instance.updateOne(instanceFilter, Updates.combine(updates)); + + Bson timeoutFilter = Filters.lt(HealthCheck.LAST_HEALTH_CHECK_TIMESTAMP, + Context.now() - HealthCheck.HEALTH_CHECK_TIMEOUT); + + AgentHealthCheckDao.instance.deleteAll(timeoutFilter); + + return Action.SUCCESS.toUpperCase(); + } + + AgentRun agentRun; + + public String findEarliestPendingAgentRun() { + + List oldestRunList = AgentRunDao.instance.findAll(Filters.eq(AgentRun._STATE, State.SCHEDULED), 0, 1, + Sorts.ascending(AgentRun.CREATED_TIMESTAMP)); + + if (oldestRunList.isEmpty()) { + return Action.SUCCESS.toUpperCase(); + } + + agentRun = oldestRunList.get(0); + return Action.SUCCESS.toUpperCase(); + } + + String state; + + public String updateAgentProcessState() { + + if (processId == null || processId.isEmpty()) { + addActionError("Process id is invalid"); + return Action.ERROR.toUpperCase(); + } + + Bson processIdFilter = Filters.eq(AgentRun.PROCESS_ID, processId); + + agentRun = AgentRunDao.instance.findOne(processIdFilter); + + if (agentRun == null) { + addActionError("No process found"); + return Action.ERROR.toUpperCase(); + } + + State updatedState = null; + try { + updatedState = State.valueOf(state); + } catch (Exception e) { + addActionError("Invalid process state"); + return Action.ERROR.toUpperCase(); + } + + List updates = new ArrayList<>(); + + /* + * TODO: Add all validations for state filters based + * on current filters and updated filters + */ + if (State.RUNNING.equals(updatedState)) { + if (State.SCHEDULED.equals(agentRun.getState())) { + updates.add(Updates.set(AgentRun._STATE, State.RUNNING)); + updates.add(Updates.set(AgentRun.START_TIMESTAMP, Context.now())); + } else { + addActionError("Only scheduled process can be started"); + return Action.ERROR.toUpperCase(); + } + } + + if (State.COMPLETED.equals(updatedState)) { + if (State.RUNNING.equals(agentRun.getState())) { + updates.add(Updates.set(AgentRun._STATE, State.COMPLETED)); + updates.add(Updates.set(AgentRun.END_TIMESTAMP, Context.now())); + } else { + addActionError("Only running process can be completed"); + return Action.ERROR.toUpperCase(); + } + } + + AgentRunDao.instance.updateOne(processIdFilter, Updates.combine(updates)); + + return Action.SUCCESS.toUpperCase(); + + } + + int attemptId; + int subProcessId; + String subProcessHeading; + String log; + Map processOutput; + AgentSubProcessSingleAttempt subprocess; + + public String updateAgentSubprocess() { + + if (processId == null || processId.isEmpty()) { + addActionError("Process id is invalid"); + return Action.ERROR.toUpperCase(); + } + + Bson processIdFilter = Filters.eq(AgentRun.PROCESS_ID, processId); + + agentRun = AgentRunDao.instance.findOne(processIdFilter); + + if (agentRun == null) { + addActionError("No process found"); + return Action.ERROR.toUpperCase(); + } + + List filters = new ArrayList<>(); + + filters.add(Filters.eq(AgentSubProcessSingleAttempt.PROCESS_ID, processId)); + filters.add(Filters.eq(AgentSubProcessSingleAttempt.SUB_PROCESS_ID, subProcessId)); + filters.add(Filters.eq(AgentSubProcessSingleAttempt.ATTEMPT_ID, attemptId)); + + List updates = new ArrayList<>(); + + if (subProcessHeading != null) { + updates.add(Updates.set(AgentSubProcessSingleAttempt.SUB_PROCESS_HEADING, subProcessHeading)); + } + updates.add(Updates.setOnInsert(AgentSubProcessSingleAttempt.START_TIMESTAMP, Context.now())); + if (log != null) { + updates.add(Updates.addToSet(AgentSubProcessSingleAttempt._LOGS, new AgentLog(log, Context.now()))); + } + if (processOutput == null) { + updates.add(Updates.set(AgentSubProcessSingleAttempt._STATE, State.RUNNING)); + } else { + updates.add(Updates.set(AgentSubProcessSingleAttempt.PROCESS_OUTPUT, processOutput)); + updates.add(Updates.set(AgentSubProcessSingleAttempt._STATE, State.COMPLETED)); + updates.add(Updates.set(AgentSubProcessSingleAttempt.END_TIMESTAMP, Context.now())); + } + + /* + * Upsert: true. + */ + subprocess = AgentSubProcessSingleAttemptDao.instance.updateOne(Filters.and(filters), Updates.combine(updates)); + + return Action.SUCCESS.toUpperCase(); + } + + /* + * + * flow for subprocess: + * + * subprocess complete + * |-> user accepts -> accept state marked from dashboard + * and agent moves to next subprocess + * |-> user declines -> declined state marked from dashboard + * and new subprocess single attempt created from dashboard with user input. + * + */ + + public String getSubProcess() { + + if (processId == null || processId.isEmpty()) { + addActionError("Process id is invalid"); + return Action.ERROR.toUpperCase(); + } + + Bson processIdFilter = Filters.eq(AgentRun.PROCESS_ID, processId); + + agentRun = AgentRunDao.instance.findOne(processIdFilter); + + if (agentRun == null) { + addActionError("No process found"); + return Action.ERROR.toUpperCase(); + } + + List filters = new ArrayList<>(); + + filters.add(Filters.eq(AgentSubProcessSingleAttempt.PROCESS_ID, processId)); + filters.add(Filters.eq(AgentSubProcessSingleAttempt.SUB_PROCESS_ID, subProcessId)); + filters.add(Filters.eq(AgentSubProcessSingleAttempt.ATTEMPT_ID, attemptId)); + + subprocess = AgentSubProcessSingleAttemptDao.instance.findOne(Filters.and(filters)); + + return Action.SUCCESS.toUpperCase(); + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + + public AgentRun getAgentRun() { + return agentRun; + } + + public void setAgentRun(AgentRun agentRun) { + this.agentRun = agentRun; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public int getAttemptId() { + return attemptId; + } + + public void setAttemptId(int attemptId) { + this.attemptId = attemptId; + } + + public int getSubProcessId() { + return subProcessId; + } + + public void setSubProcessId(int subProcessId) { + this.subProcessId = subProcessId; + } + + public String getSubProcessHeading() { + return subProcessHeading; + } + + public void setSubProcessHeading(String subProcessHeading) { + this.subProcessHeading = subProcessHeading; + } + + public String getLog() { + return log; + } + + public void setLog(String log) { + this.log = log; + } + + public Map getProcessOutput() { + return processOutput; + } + + public void setProcessOutput(Map processOutput) { + this.processOutput = processOutput; + } + + public AgentSubProcessSingleAttempt getSubprocess() { + return subprocess; + } + + public void setSubprocess(AgentSubProcessSingleAttempt subprocess) { + this.subprocess = subprocess; + } + +} \ No newline at end of file diff --git a/apps/database-abstractor/src/main/resources/struts.xml b/apps/database-abstractor/src/main/resources/struts.xml index 7ca1ac94c3..b82d9a9cb7 100644 --- a/apps/database-abstractor/src/main/resources/struts.xml +++ b/apps/database-abstractor/src/main/resources/struts.xml @@ -1420,6 +1420,61 @@ + + + + + + 422 + false + ^actionErrors.* + + + + + + + + + 422 + false + ^actionErrors.* + + + + + + + + + 422 + false + ^actionErrors.* + + + + + + + + + 422 + false + ^actionErrors.* + + + + + + + + + 422 + false + ^actionErrors.* + + + diff --git a/libs/dao/src/main/java/com/akto/dao/agents/AgentHealthCheckDao.java b/libs/dao/src/main/java/com/akto/dao/agents/AgentHealthCheckDao.java new file mode 100644 index 0000000000..ac36523840 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/agents/AgentHealthCheckDao.java @@ -0,0 +1,20 @@ +package com.akto.dao.agents; + +import com.akto.dao.CommonContextDao; +import com.akto.dto.agents.HealthCheck; + +public class AgentHealthCheckDao extends CommonContextDao { + + public static final AgentHealthCheckDao instance = new AgentHealthCheckDao(); + + @Override + public String getCollName() { + return "agent_health_check"; + } + + @Override + public Class getClassT() { + return HealthCheck.class; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dao/agents/AgentRunDao.java b/libs/dao/src/main/java/com/akto/dao/agents/AgentRunDao.java new file mode 100644 index 0000000000..1984d07323 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/agents/AgentRunDao.java @@ -0,0 +1,22 @@ +package com.akto.dao.agents; + +import com.akto.dao.AccountsContextDao; +import com.akto.dto.agents.AgentRun; + +public class AgentRunDao extends AccountsContextDao { + + public static final AgentRunDao instance = new AgentRunDao(); + + // TODO: create indices + + @Override + public String getCollName() { + return "agent_runs"; + } + + @Override + public Class getClassT() { + return AgentRun.class; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java b/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java new file mode 100644 index 0000000000..ad286622f1 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java @@ -0,0 +1,22 @@ +package com.akto.dao.agents; + +import com.akto.dao.AccountsContextDao; +import com.akto.dto.agents.AgentSubProcessSingleAttempt; + +public class AgentSubProcessSingleAttemptDao extends AccountsContextDao{ + + public static final AgentSubProcessSingleAttemptDao instance = new AgentSubProcessSingleAttemptDao(); + + // TODO: create indices + + @Override + public String getCollName() { + return "agent_sub_process_attempts"; + } + + @Override + public Class getClassT() { + return AgentSubProcessSingleAttempt.class; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dto/agents/Agent.java b/libs/dao/src/main/java/com/akto/dto/agents/Agent.java new file mode 100644 index 0000000000..c02aa89573 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/agents/Agent.java @@ -0,0 +1,18 @@ +package com.akto.dto.agents; + +public enum Agent { + + FIND_APIS_FROM_SOURCE_CODE("Matt"), + FIND_VULNERABILITIES_FROM_SOURCE_CODE("Lisa"); + + private final String agentName; + + Agent(String agentName) { + this.agentName = agentName; + } + + public String getAgentName() { + return agentName; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dto/agents/AgentLog.java b/libs/dao/src/main/java/com/akto/dto/agents/AgentLog.java new file mode 100644 index 0000000000..436c357182 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/agents/AgentLog.java @@ -0,0 +1,31 @@ +package com.akto.dto.agents; + +public class AgentLog { + String log; + int eventTimestamp; + + public AgentLog(String log, int eventTimestamp) { + this.log = log; + this.eventTimestamp = eventTimestamp; + } + + public AgentLog() { + } + + public String getLog() { + return log; + } + + public void setLog(String log) { + this.log = log; + } + + public int getEventTimestamp() { + return eventTimestamp; + } + + public void setEventTimestamp(int eventTimestamp) { + this.eventTimestamp = eventTimestamp; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dto/agents/AgentRun.java b/libs/dao/src/main/java/com/akto/dto/agents/AgentRun.java new file mode 100644 index 0000000000..fe03dd638f --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/agents/AgentRun.java @@ -0,0 +1,105 @@ +package com.akto.dto.agents; + +import java.util.Map; + +public class AgentRun { + + /* + * This is a random generated UUID + * The process id for the entire agent. Created once per agent. + * process e.g. finding vulnerabilities from source code + * Same as AgentSubProcessSingleAttempt.processId + */ + String processId; + final public static String PROCESS_ID = "processId"; + /* + * Contains agent init information + * e.g. for vulnerability finding agent may contain the repository to run the + * agent on. + * for sensitive data finding agent may contain the collection to run the agent + * on. + * + * Generic Map data type because the data may vary according to agent. + */ + Map agentInitDocument; + Agent agent; + int createdTimestamp; + final public static String CREATED_TIMESTAMP = "createdTimestamp"; + final public static String START_TIMESTAMP = "startTimestamp"; + int startTimestamp; + final public static String END_TIMESTAMP = "endTimestamp"; + int endTimestamp; + State state; + final public static String _STATE = "state"; + + public AgentRun(String processId, Map agentInitDocument, Agent agent, int createdTimestamp, + int startTimestamp, int endTimestamp, State state) { + this.processId = processId; + this.agentInitDocument = agentInitDocument; + this.agent = agent; + this.createdTimestamp = createdTimestamp; + this.startTimestamp = startTimestamp; + this.endTimestamp = endTimestamp; + this.state = state; + } + + public AgentRun() { + } + + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + + public Map getAgentInitDocument() { + return agentInitDocument; + } + + public void setAgentInitDocument(Map agentInitDocument) { + this.agentInitDocument = agentInitDocument; + } + + public Agent getAgent() { + return agent; + } + + public void setAgent(Agent agent) { + this.agent = agent; + } + + public int getCreatedTimestamp() { + return createdTimestamp; + } + + public void setCreatedTimestamp(int createdTimestamp) { + this.createdTimestamp = createdTimestamp; + } + + public int getStartTimestamp() { + return startTimestamp; + } + + public void setStartTimestamp(int startTimestamp) { + this.startTimestamp = startTimestamp; + } + + public int getEndTimestamp() { + return endTimestamp; + } + + public void setEndTimestamp(int endTimestamp) { + this.endTimestamp = endTimestamp; + } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java b/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java new file mode 100644 index 0000000000..e4eb33c992 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java @@ -0,0 +1,171 @@ +package com.akto.dto.agents; + +import java.util.List; +import java.util.Map; + +public class AgentSubProcessSingleAttempt { + + /* + * This is a random generated UUID + * The process id for the entire agent. Created once per agent. + * process e.g. finding vulnerabilities from source code + * Same as AgentRun.processId + */ + String processId; + final public static String PROCESS_ID = "processId"; + /* + * The sub process id for the subprocess in the agent. + * subprocess e.g. find authentication mechanisms subprocess + * inside finding vulnerabilities from source code process. + * + * Cardinal number for subProcess + * + * This ID is per subProcess. + * So for multiple attempts remains same. + */ + String subProcessId; + final public static String SUB_PROCESS_ID = "subProcessId"; + String subProcessHeading; + final public static String SUB_PROCESS_HEADING = "subProcessHeading"; + + /* + * By default user input is empty. + * User can provide input if they choose to, + * to help the agent. + * + * Using a generic type here as the user input + * could be in a variety of formats + * + * TODO: make abstract classes for userInput, agentOutput and apiInitDocument + * and implement them based on feedback on what to expect to/from agent. + * + */ + Map userInput; + int startTimestamp; + final public static String START_TIMESTAMP = "startTimestamp"; + int endTimestamp; + final public static String END_TIMESTAMP = "endTimestamp"; + State state; + final public static String _STATE = "state"; + /* + * Cardinal number for attempt + */ + int attemptId; + final public static String ATTEMPT_ID = "attemptId"; + List logs; + final public static String _LOGS = "logs"; + /* + * Using a generic type here as the agent output + * could be in a variety of formats + * + * e.g. + * + * backendDirectoriesFound: [ '/app/dir1', '/app/dir2' ] + * backendFrameworksFound: ['struts2', 'mux'] + * vulnerabilities: "The source code contains no vulnerabilities" + * vulnerabilitiesFound: [ {vulnerability: 'BOLA', severity: 'HIGH', + * listOfAPIs:['/api/testing', '/api/finalTesting', '/api/finalFinalTesting']} ] + * + */ + Map processOutput; + final public static String PROCESS_OUTPUT = "processOutput"; + + public AgentSubProcessSingleAttempt(String processId, String subProcessId, String subProcessHeading, + Map userInput, int startTimestamp, int endTimestamp, State state, int attemptId, + List logs, Map processOutput) { + this.processId = processId; + this.subProcessId = subProcessId; + this.subProcessHeading = subProcessHeading; + this.userInput = userInput; + this.startTimestamp = startTimestamp; + this.endTimestamp = endTimestamp; + this.state = state; + this.attemptId = attemptId; + this.logs = logs; + this.processOutput = processOutput; + } + + public AgentSubProcessSingleAttempt() { + } + + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + + public String getSubProcessId() { + return subProcessId; + } + + public void setSubProcessId(String subProcessId) { + this.subProcessId = subProcessId; + } + + public String getSubProcessHeading() { + return subProcessHeading; + } + + public void setSubProcessHeading(String subProcessHeading) { + this.subProcessHeading = subProcessHeading; + } + + public int getStartTimestamp() { + return startTimestamp; + } + + public void setStartTimestamp(int startTimestamp) { + this.startTimestamp = startTimestamp; + } + + public int getEndTimestamp() { + return endTimestamp; + } + + public void setEndTimestamp(int endTimestamp) { + this.endTimestamp = endTimestamp; + } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + + public int getAttemptId() { + return attemptId; + } + + public void setAttemptId(int attemptId) { + this.attemptId = attemptId; + } + + public List getLogs() { + return logs; + } + + public void setLogs(List logs) { + this.logs = logs; + } + + public Map getProcessOutput() { + return processOutput; + } + + public void setProcessOutput(Map processOutput) { + this.processOutput = processOutput; + } + + public Map getUserInput() { + return userInput; + } + + public void setUserInput(Map userInput) { + this.userInput = userInput; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dto/agents/HealthCheck.java b/libs/dao/src/main/java/com/akto/dto/agents/HealthCheck.java new file mode 100644 index 0000000000..d747289c5b --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/agents/HealthCheck.java @@ -0,0 +1,68 @@ +package com.akto.dto.agents; + +public class HealthCheck { + + /* + * Random UUID generated on client side. + */ + String instanceId; + public static final String INSTANCE_ID = "instanceId"; + int lastHealthCheckTimestamp; + public static final String LAST_HEALTH_CHECK_TIMESTAMP = "lastHealthCheckTimestamp"; + /* + * Eventually version can help with what all agents are supported on a module. + */ + String version; + public static final String _VERSION = "version"; + /* + * If the instance is running a process, + * processId is filled. + * Else, empty. + */ + String processId; + public static final String PROCESS_ID = "processId"; + + final public static int HEALTH_CHECK_TIMEOUT = 5 * 60 * 60; + + public HealthCheck(String instanceId, int lastHealthCheckTimestamp, String version) { + this.instanceId = instanceId; + this.lastHealthCheckTimestamp = lastHealthCheckTimestamp; + this.version = version; + } + + public HealthCheck() { + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public int getLastHealthCheckTimestamp() { + return lastHealthCheckTimestamp; + } + + public void setLastHealthCheckTimestamp(int lastHealthCheckTimestamp) { + this.lastHealthCheckTimestamp = lastHealthCheckTimestamp; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dto/agents/State.java b/libs/dao/src/main/java/com/akto/dto/agents/State.java new file mode 100644 index 0000000000..33e70fa7cd --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/agents/State.java @@ -0,0 +1,13 @@ +package com.akto.dto.agents; + +public enum State { + + STOPPED, + RUNNING, + COMPLETED, + SCHEDULED, + FAILED, + DISCARDED, + ACCEPTED + +} \ No newline at end of file From f021faba81a7a22a856ecfdc70fba3de9c0b69f7 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 21 Feb 2025 20:11:26 +0530 Subject: [PATCH 2/3] add backend APIs for dashboard --- .../com/akto/action/agents/AgentAction.java | 262 ++++++++++++++++++ apps/dashboard/src/main/resources/struts.xml | 143 ++++++++++ .../java/com/akto/action/AgentAction.java | 45 +-- .../AgentSubProcessSingleAttemptDao.java | 17 ++ .../agents/AgentSubProcessSingleAttempt.java | 16 +- .../java/com/akto/dto/rbac/RbacEnums.java | 8 +- 6 files changed, 469 insertions(+), 22 deletions(-) create mode 100644 apps/dashboard/src/main/java/com/akto/action/agents/AgentAction.java diff --git a/apps/dashboard/src/main/java/com/akto/action/agents/AgentAction.java b/apps/dashboard/src/main/java/com/akto/action/agents/AgentAction.java new file mode 100644 index 0000000000..b65c068e1c --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/agents/AgentAction.java @@ -0,0 +1,262 @@ +package com.akto.action.agents; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.bson.conversions.Bson; + +import com.akto.action.UserAction; +import com.akto.dao.agents.AgentRunDao; +import com.akto.dao.agents.AgentSubProcessSingleAttemptDao; +import com.akto.dao.context.Context; +import com.akto.dto.agents.*; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import com.opensymphony.xwork2.Action; + +public class AgentAction extends UserAction { + + Map data; + String agent; + + public String createAgentRun() { + + if (data == null || data.isEmpty()) { + addActionError("Agent Init document is empty"); + return Action.ERROR.toUpperCase(); + } + + if (agent == null || agent.isEmpty()) { + addActionError("Agent is empty"); + return Action.ERROR.toUpperCase(); + } + + Agent agentModule; + try { + agentModule = Agent.valueOf(agent); + } catch (Exception e) { + addActionError("Invalid agent"); + return Action.ERROR.toUpperCase(); + } + + AgentRun existingScheduledOrRunningRuns = AgentRunDao.instance.findOne(Filters.nin( + AgentRun._STATE, Arrays.asList(State.SCHEDULED, State.RUNNING))); + + if (existingScheduledOrRunningRuns != null) { + addActionError("An existing agent run is running or scheduled. " + + "Let it complete or stop it before starting another"); + return Action.ERROR.toUpperCase(); + } + + String processId = UUID.randomUUID().toString(); + + agentRun = new AgentRun(processId, data, agentModule, + Context.now(), 0, 0, State.SCHEDULED); + + AgentRunDao.instance.insertOne(agentRun); + + return Action.SUCCESS.toUpperCase(); + } + + Agent[] agents; + + AgentRun agentRun; + + String processId; + int subProcessId; + int attemptId; + String state; + + AgentSubProcessSingleAttempt subprocess; + + public String getAgents() { + agents = Agent.values(); + return Action.SUCCESS.toUpperCase(); + } + + List subProcesses; + + public String getAllSubProcesses() { + if (processId == null || processId.isEmpty()) { + addActionError("Process id is invalid"); + return Action.ERROR.toUpperCase(); + } + Bson processIdFilter = Filters.eq(AgentRun.PROCESS_ID, processId); + + agentRun = AgentRunDao.instance.findOne(processIdFilter); + + if (agentRun == null) { + addActionError("No process found"); + return Action.ERROR.toUpperCase(); + } + + subProcesses = AgentSubProcessSingleAttemptDao.instance.findAll(processIdFilter); + return Action.SUCCESS.toUpperCase(); + } + + public String getSubProcess(){ + + if (processId == null || processId.isEmpty()) { + addActionError("Process id is invalid"); + return Action.ERROR.toUpperCase(); + } + + Bson processIdFilter = Filters.eq(AgentRun.PROCESS_ID, processId); + + agentRun = AgentRunDao.instance.findOne(processIdFilter); + + if (agentRun == null) { + addActionError("No process found"); + return Action.ERROR.toUpperCase(); + } + + Bson filter = AgentSubProcessSingleAttemptDao.instance.getFiltersForAgentSubProcess(processId, subProcessId, attemptId); + subprocess = AgentSubProcessSingleAttemptDao.instance.findOne(filter); + + return Action.SUCCESS.toUpperCase(); + + } + + public String updateAgentSubprocess() { + + if (processId == null || processId.isEmpty()) { + addActionError("Process id is invalid"); + return Action.ERROR.toUpperCase(); + } + + Bson processIdFilter = Filters.eq(AgentRun.PROCESS_ID, processId); + + agentRun = AgentRunDao.instance.findOne(processIdFilter); + + if (agentRun == null) { + addActionError("No process found"); + return Action.ERROR.toUpperCase(); + } + + Bson filter = AgentSubProcessSingleAttemptDao.instance.getFiltersForAgentSubProcess(processId, subProcessId, + attemptId); + + List updates = new ArrayList<>(); + + updates.add(Updates.setOnInsert(AgentSubProcessSingleAttempt.CREATED_TIMESTAMP, Context.now())); + updates.add(Updates.setOnInsert(AgentSubProcessSingleAttempt._STATE, State.SCHEDULED)); + + State updatedState = null; + try { + updatedState = State.valueOf(state); + } catch (Exception e) { + } + + if (updatedState != null) { + AgentSubProcessSingleAttempt subProcess = AgentSubProcessSingleAttemptDao.instance.findOne(filter); + if (subProcess == null) { + addActionError("No subprocess found"); + return Action.ERROR.toUpperCase(); + } + + if (State.COMPLETED.equals(subProcess.getState()) + && (State.ACCEPTED.equals(updatedState) || State.DISCARDED.equals(updatedState))) { + updates.add(Updates.set(AgentSubProcessSingleAttempt._STATE, updatedState)); + } else { + addActionError("Invalid state"); + return Action.ERROR.toUpperCase(); + } + } + + /* + * For a new subprocess. + */ + if (data != null) { + updates.add(Updates.set(AgentSubProcessSingleAttempt.USER_INPUT, data)); + } + + /* + * Upsert: true. + * Since state management is through dashboard, + * all subprocess' are created here and only modified using the agent-module + */ + subprocess = AgentSubProcessSingleAttemptDao.instance.updateOne(filter, Updates.combine(updates)); + + return Action.SUCCESS.toUpperCase(); + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + + public String getAgent() { + return agent; + } + + public void setAgent(String agent) { + this.agent = agent; + } + + public void setAgents(Agent[] agents) { + this.agents = agents; + } + + public AgentRun getAgentRun() { + return agentRun; + } + + public void setAgentRun(AgentRun agentRun) { + this.agentRun = agentRun; + } + + public String getProcessId() { + return processId; + } + + public void setProcessId(String processId) { + this.processId = processId; + } + + public int getSubProcessId() { + return subProcessId; + } + + public void setSubProcessId(int subProcessId) { + this.subProcessId = subProcessId; + } + + public int getAttemptId() { + return attemptId; + } + + public void setAttemptId(int attemptId) { + this.attemptId = attemptId; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public AgentSubProcessSingleAttempt getSubprocess() { + return subprocess; + } + + public void setSubprocess(AgentSubProcessSingleAttempt subprocess) { + this.subprocess = subprocess; + } + + public List getSubProcesses() { + return subProcesses; + } + + public void setSubProcesses(List subProcesses) { + this.subProcesses = subProcesses; + } + +} diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 9da744cd69..dc0bb75c32 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -6027,6 +6027,149 @@ + + + + + AI_AGENTS + READ_WRITE + Create an agent run + + + 403 + false + ^actionErrors.* + + + AI_AGENTS + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + AI_AGENTS + READ + + + 403 + false + ^actionErrors.* + + + AI_AGENTS + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + AI_AGENTS + READ + + + 403 + false + ^actionErrors.* + + + AI_AGENTS + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + AI_AGENTS + READ + + + 403 + false + ^actionErrors.* + + + AI_AGENTS + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + AI_AGENTS + READ + + + 403 + false + ^actionErrors.* + + + AI_AGENTS + + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + diff --git a/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java b/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java index 9e05ad11d7..b37c647e59 100644 --- a/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java +++ b/apps/database-abstractor/src/main/java/com/akto/action/AgentAction.java @@ -156,33 +156,47 @@ public String updateAgentSubprocess() { return Action.ERROR.toUpperCase(); } - List filters = new ArrayList<>(); + Bson filter = AgentSubProcessSingleAttemptDao.instance.getFiltersForAgentSubProcess(processId, subProcessId, attemptId); - filters.add(Filters.eq(AgentSubProcessSingleAttempt.PROCESS_ID, processId)); - filters.add(Filters.eq(AgentSubProcessSingleAttempt.SUB_PROCESS_ID, subProcessId)); - filters.add(Filters.eq(AgentSubProcessSingleAttempt.ATTEMPT_ID, attemptId)); + AgentSubProcessSingleAttempt subProcess = AgentSubProcessSingleAttemptDao.instance.findOne(filter); + + if (subProcess == null) { + addActionError("No subprocess found"); + return Action.ERROR.toUpperCase(); + } + + State updatedState = null; + try { + updatedState = State.valueOf(state); + } catch (Exception e) { + } List updates = new ArrayList<>(); + if (updatedState!=null && State.RUNNING.equals(updatedState) && State.SCHEDULED.equals(subProcess.getState())) { + updates.add(Updates.set(AgentSubProcessSingleAttempt.START_TIMESTAMP, Context.now())); + updates.add(Updates.set(AgentSubProcessSingleAttempt._STATE, State.RUNNING)); + } + if (subProcessHeading != null) { updates.add(Updates.set(AgentSubProcessSingleAttempt.SUB_PROCESS_HEADING, subProcessHeading)); } - updates.add(Updates.setOnInsert(AgentSubProcessSingleAttempt.START_TIMESTAMP, Context.now())); + if (log != null) { updates.add(Updates.addToSet(AgentSubProcessSingleAttempt._LOGS, new AgentLog(log, Context.now()))); } - if (processOutput == null) { - updates.add(Updates.set(AgentSubProcessSingleAttempt._STATE, State.RUNNING)); - } else { + + if (updatedState!=null && processOutput != null && State.COMPLETED.equals(updatedState)) { updates.add(Updates.set(AgentSubProcessSingleAttempt.PROCESS_OUTPUT, processOutput)); updates.add(Updates.set(AgentSubProcessSingleAttempt._STATE, State.COMPLETED)); updates.add(Updates.set(AgentSubProcessSingleAttempt.END_TIMESTAMP, Context.now())); } /* - * Upsert: true. + * Upsert: false + * because the state is controlled by user inputs. */ - subprocess = AgentSubProcessSingleAttemptDao.instance.updateOne(Filters.and(filters), Updates.combine(updates)); + subprocess = AgentSubProcessSingleAttemptDao.instance.updateOneNoUpsert(filter, Updates.combine(updates)); return Action.SUCCESS.toUpperCase(); } @@ -193,7 +207,7 @@ public String updateAgentSubprocess() { * * subprocess complete * |-> user accepts -> accept state marked from dashboard - * and agent moves to next subprocess + * and agent moves to next subprocess, which is also created from dashboard. * |-> user declines -> declined state marked from dashboard * and new subprocess single attempt created from dashboard with user input. * @@ -215,13 +229,8 @@ public String getSubProcess() { return Action.ERROR.toUpperCase(); } - List filters = new ArrayList<>(); - - filters.add(Filters.eq(AgentSubProcessSingleAttempt.PROCESS_ID, processId)); - filters.add(Filters.eq(AgentSubProcessSingleAttempt.SUB_PROCESS_ID, subProcessId)); - filters.add(Filters.eq(AgentSubProcessSingleAttempt.ATTEMPT_ID, attemptId)); - - subprocess = AgentSubProcessSingleAttemptDao.instance.findOne(Filters.and(filters)); + Bson filter = AgentSubProcessSingleAttemptDao.instance.getFiltersForAgentSubProcess(processId, subProcessId, attemptId); + subprocess = AgentSubProcessSingleAttemptDao.instance.findOne(filter); return Action.SUCCESS.toUpperCase(); } diff --git a/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java b/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java index ad286622f1..4589bc0e89 100644 --- a/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java +++ b/libs/dao/src/main/java/com/akto/dao/agents/AgentSubProcessSingleAttemptDao.java @@ -1,7 +1,13 @@ package com.akto.dao.agents; +import java.util.ArrayList; +import java.util.List; + +import org.bson.conversions.Bson; + import com.akto.dao.AccountsContextDao; import com.akto.dto.agents.AgentSubProcessSingleAttempt; +import com.mongodb.client.model.Filters; public class AgentSubProcessSingleAttemptDao extends AccountsContextDao{ @@ -9,6 +15,17 @@ public class AgentSubProcessSingleAttemptDao extends AccountsContextDao filters = new ArrayList<>(); + + filters.add(Filters.eq(AgentSubProcessSingleAttempt.PROCESS_ID, processId)); + filters.add(Filters.eq(AgentSubProcessSingleAttempt.SUB_PROCESS_ID, subProcessId)); + filters.add(Filters.eq(AgentSubProcessSingleAttempt.ATTEMPT_ID, attemptId)); + + return Filters.and(filters); + } + + @Override public String getCollName() { return "agent_sub_process_attempts"; diff --git a/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java b/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java index e4eb33c992..0797fe4401 100644 --- a/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java +++ b/libs/dao/src/main/java/com/akto/dto/agents/AgentSubProcessSingleAttempt.java @@ -41,6 +41,9 @@ public class AgentSubProcessSingleAttempt { * */ Map userInput; + final public static String USER_INPUT = "userInput"; + int createdTimestamp; + final public static String CREATED_TIMESTAMP = "createdTimestamp"; int startTimestamp; final public static String START_TIMESTAMP = "startTimestamp"; int endTimestamp; @@ -71,12 +74,13 @@ public class AgentSubProcessSingleAttempt { final public static String PROCESS_OUTPUT = "processOutput"; public AgentSubProcessSingleAttempt(String processId, String subProcessId, String subProcessHeading, - Map userInput, int startTimestamp, int endTimestamp, State state, int attemptId, - List logs, Map processOutput) { + Map userInput, int createdTimestamp, int startTimestamp, int endTimestamp, State state, + int attemptId, List logs, Map processOutput) { this.processId = processId; this.subProcessId = subProcessId; this.subProcessHeading = subProcessHeading; this.userInput = userInput; + this.createdTimestamp = createdTimestamp; this.startTimestamp = startTimestamp; this.endTimestamp = endTimestamp; this.state = state; @@ -168,4 +172,12 @@ public void setUserInput(Map userInput) { this.userInput = userInput; } + public int getCreatedTimestamp() { + return createdTimestamp; + } + + public void setCreatedTimestamp(int createdTimestamp) { + this.createdTimestamp = createdTimestamp; + } + } diff --git a/libs/dao/src/main/java/com/akto/dto/rbac/RbacEnums.java b/libs/dao/src/main/java/com/akto/dto/rbac/RbacEnums.java index cbe9f0b19b..d1013707ca 100644 --- a/libs/dao/src/main/java/com/akto/dto/rbac/RbacEnums.java +++ b/libs/dao/src/main/java/com/akto/dto/rbac/RbacEnums.java @@ -13,7 +13,9 @@ public enum AccessGroups { TEST_LIBRARY, SETTINGS, ADMIN, - USER; + DEBUG_INFO, + USER, + AI; public static AccessGroups[] getAccessGroups() { return values(); @@ -42,7 +44,9 @@ public enum Feature { BILLING(AccessGroups.SETTINGS), INVITE_MEMBERS(AccessGroups.SETTINGS), ADMIN_ACTIONS(AccessGroups.ADMIN), - USER_ACTIONS(AccessGroups.USER); + USER_ACTIONS(AccessGroups.USER), + AI_AGENTS(AccessGroups.AI); + private final AccessGroups accessGroup; Feature(AccessGroups accessGroup) { From f8ca61cb1f738b4298927f02e3025863bfdaea7c Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 11 Apr 2025 14:58:31 +0530 Subject: [PATCH 3/3] add slack notification for release --- .github/workflows/prod.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index cc3a7a1de5..3c3d8b076e 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -8,6 +8,7 @@ on: workflow_dispatch: inputs: release_version: + description: "The version of the release to be deployed." required: true # A workflow run is made up of one or more jobs that can run sequentially or in parallel @@ -60,7 +61,7 @@ jobs: REGISTRY_ALIAS: p7q3h0z2 IMAGE_TAG: kafkalatest IMAGE_TAG2: latest - IMAGE_TAG3: 1.42.1_local + IMAGE_TAG3: ${{ github.event.inputs.release_version }}_local run: | docker buildx create --use # Build a docker container and push it to DockerHub @@ -82,8 +83,21 @@ jobs: ECR_REPOSITORY: akto-api-security IMAGE_TAG: kafkalatest IMAGE_TAG2: latest + IMAGE_TAG3: ${{ github.event.inputs.release_version }}_local run: | docker buildx create --use # Build a docker container and push it to DockerHub cd apps/database-abstractor - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-security-database-abstractor:$IMAGE_TAG -t $ECR_REGISTRY/akto-api-security-database-abstractor:$IMAGE_TAG2 . --push \ No newline at end of file + docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-security-database-abstractor:$IMAGE_TAG -t $ECR_REGISTRY/akto-api-security-database-abstractor:$IMAGE_TAG2 -t $ECR_REGISTRY/akto-api-security-database-abstractor:$IMAGE_TAG3 . --push + + - name: Send Github release notification to Slack + id: slack + uses: slackapi/slack-github-action@v1.23.0 + with: + payload: | + { + "text": "Database abstractor (codename: cyborg) version v${{ github.event.inputs.release_version }} released!" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK