Skip to content

add support for grpc and test pre script #2170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 10, 2025
Merged

add support for grpc and test pre script #2170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d177882
add support for grpc and test pre script
notshivansh Mar 5, 2025
b6ec016
removing maven cache in staging
notshivansh Mar 5, 2025
0973dac
use v3
notshivansh Mar 5, 2025
f65d360
use v3
notshivansh Mar 5, 2025
33a8153
mark complete irrespective
notshivansh Mar 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/staging.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ jobs:
with:
java-version: 11
- name: Cache SonarCloud packages
uses: actions/cache@v1
uses: actions/cache@v3
with:
path: ~/.sonar/cache
key: ${{ runner.os }}-sonar
restore-keys: ${{ runner.os }}-sonar
- name: Cache Maven packages
uses: actions/cache@v1
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
Expand Down
2 changes: 1 addition & 1 deletion apps/mini-testing/src/main/java/com/akto/testing/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ public void run() {
scheduleTs = testingRun.getScheduleTimestamp() + 5 * 60;
}

if(GetRunningTestsStatus.getRunningTests().isTestRunning(testingRun.getId())){
if(testingRun!=null && testingRun.getId()!=null){
dataActor.updateTestingRunAndMarkCompleted(testingRun.getId().toHexString(), scheduleTs);
}

Expand Down
45 changes: 28 additions & 17 deletions apps/mini-testing/src/main/java/com/akto/testing/StatusCodeAnalyser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package com.akto.testing;

import com.akto.DaoInit;
import com.akto.dao.ApiCollectionsDao;
import com.akto.dao.AuthMechanismsDao;
import com.akto.dao.context.Context;
import com.akto.data_actor.DataActor;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.*;
Expand All @@ -19,13 +16,10 @@
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.BasicDBObject;
import com.mongodb.ConnectionString;

import java.util.*;

import static com.akto.runtime.RelationshipSync.extractAllValuesFromPayload;
import static com.akto.testing.TestExecutor.findHost;

public class StatusCodeAnalyser {

Expand All @@ -52,7 +46,7 @@ public String toString() {
}
}

public static void run(Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap, SampleMessageStore sampleMessageStore, AuthMechanismStore authMechanismStore, TestingRunConfig testingRunConfig) {
public static void run(Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap, SampleMessageStore sampleMessageStore, AuthMechanismStore authMechanismStore, TestingRunConfig testingRunConfig, Map<String, String> hostAndContentType) {
defaultPayloadsMap = new HashMap<>();
result = new ArrayList<>();
if (sampleDataMap == null) {
Expand All @@ -61,47 +55,64 @@ public static void run(Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap, Samp
}
loggerMaker.infoAndAddToDb("started calc default payloads", LogDb.TESTING);

calculateDefaultPayloads(sampleMessageStore, sampleDataMap, testingRunConfig);
calculateDefaultPayloads(sampleMessageStore, sampleDataMap, testingRunConfig, hostAndContentType);

loggerMaker.infoAndAddToDb("started fill result", LogDb.TESTING);
fillResult(sampleMessageStore, sampleDataMap, authMechanismStore, testingRunConfig);
}

public static void calculateDefaultPayloads(SampleMessageStore sampleMessageStore, Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap, TestingRunConfig testingRunConfig) {
Set<String> hosts = new HashSet<>();
public static Map<String, String> findAllHosts(SampleMessageStore sampleMessageStore, Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap){
Map<String, String> hostAndContentType = new HashMap<>();
for (ApiInfo.ApiInfoKey apiInfoKey: sampleDataMap.keySet()) {
String host;
String contentType;
try {
host = findHost(apiInfoKey, sampleDataMap, sampleMessageStore);
loggerMaker.infoAndAddToDb("Finding host for apiInfoKey: " + apiInfoKey.toString());
OriginalHttpRequest request = TestExecutor.findOriginalHttpRequest(apiInfoKey, sampleDataMap, sampleMessageStore);
host = TestExecutor.findHostFromOriginalHttpRequest(request);
contentType = TestExecutor.findContentTypeFromOriginalHttpRequest(request);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Error while finding host in status code analyser: " + e, LogDb.TESTING);
continue;
}

hosts.add(host);
if(host != null ){
hostAndContentType.put(host, contentType);
}
}
return hostAndContentType;
}

for (String host: hosts) {
public static void calculateDefaultPayloads(SampleMessageStore sampleMessageStore, Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap, TestingRunConfig testingRunConfig, Map<String, String> hostAndContentType) {
for (String host: hostAndContentType.keySet()) {
loggerMaker.infoAndAddToDb("calc default payload for host: " + host, LogDb.TESTING);
for (int idx=0; idx<11;idx++) {
try {
String url = host;
if (!url.endsWith("/")) url += "/";
if (idx > 0) url += "akto-"+idx; // we want to hit host url once too

OriginalHttpRequest request = new OriginalHttpRequest(url, null, URLMethods.Method.GET.name(), null, new HashMap<>(), "");
String contentType = hostAndContentType.get(host);
Map<String, List<String>> headers = new HashMap<>();

if (contentType != null) {
headers.put("content-type", Arrays.asList(contentType));
}
if (host != null && !host.isEmpty()) {
headers.put("host", Arrays.asList(host));
}

OriginalHttpRequest request = new OriginalHttpRequest(url, null, URLMethods.Method.GET.name(), null, headers, "");
OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, testingRunConfig, false, new ArrayList<>(), Main.SKIP_SSRF_CHECK);
boolean isStatusGood = TestPlugin.isStatusGood(response.getStatusCode());
if (!isStatusGood) continue;

String body = response.getBody();
fillDefaultPayloadsMap(body);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("", LogDb.TESTING);
loggerMaker.errorAndAddToDb(e, "Error in calculateDefaultPayloads " + e.getMessage(), LogDb.TESTING);
}
}
}

}

public static void fillDefaultPayloadsMap(String body) {
Expand Down
46 changes: 40 additions & 6 deletions apps/mini-testing/src/main/java/com/akto/testing/TestExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,18 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId, boolean debug
}
}

int currentTime = Context.now();
Map<String, String> hostAndContentType = new HashMap<>();
try {
StatusCodeAnalyser.run(sampleDataMapForStatusCodeAnalyser, sampleMessageStore , authMechanismStore, testingRun.getTestingRunConfig());
loggerMaker.infoAndAddToDb("Starting findAllHosts at: " + currentTime, LogDb.TESTING);
hostAndContentType = StatusCodeAnalyser.findAllHosts(sampleMessageStore, sampleDataMapForStatusCodeAnalyser);
loggerMaker.infoAndAddToDb("Completing findAllHosts in: " + (Context.now() - currentTime) + " at: " + Context.now(), LogDb.TESTING);
} catch (Exception e){
loggerMaker.errorAndAddToDb("Error while running findAllHosts " + e.getMessage(), LogDb.TESTING);
}

try {
StatusCodeAnalyser.run(sampleDataMapForStatusCodeAnalyser, sampleMessageStore , authMechanismStore, testingRun.getTestingRunConfig(), hostAndContentType);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Error while running status code analyser " + e.getMessage(), LogDb.TESTING);
}
Expand Down Expand Up @@ -210,7 +220,8 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId, boolean debug
}
for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeyList) {
try {
hostName = findHost(apiInfoKey, testingUtil.getSampleMessages(), testingUtil.getSampleMessageStore());
OriginalHttpRequest request = TestExecutor.findOriginalHttpRequest(apiInfoKey, testingUtil.getSampleMessages(), testingUtil.getSampleMessageStore());
hostName = TestExecutor.findHostFromOriginalHttpRequest(request);
if (hostName == null) {
continue;
}
Expand All @@ -226,7 +237,8 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId, boolean debug

for (ApiInfo.ApiInfoKey apiInfoKey: apiInfoKeyList) {
try {
hostName = findHost(apiInfoKey, testingUtil.getSampleMessages(), testingUtil.getSampleMessageStore());
OriginalHttpRequest request = TestExecutor.findOriginalHttpRequest(apiInfoKey, testingUtil.getSampleMessages(), testingUtil.getSampleMessageStore());
hostName = TestExecutor.findHostFromOriginalHttpRequest(request);
if (hostName != null && hostsToApiCollectionMap.get(hostName) == null) {
hostsToApiCollectionMap.put(hostName, apiInfoKey.getApiCollectionId());
}
Expand Down Expand Up @@ -307,25 +319,47 @@ public static Severity getSeverityFromTestingRunResult(TestingRunResult testingR
return severity;
}

public static String findHost(ApiInfo.ApiInfoKey apiInfoKey, Map<ApiInfo.ApiInfoKey, List<String>> sampleMessagesMap, SampleMessageStore sampleMessageStore) throws URISyntaxException {
public static OriginalHttpRequest findOriginalHttpRequest(ApiInfo.ApiInfoKey apiInfoKey, Map<ApiInfo.ApiInfoKey, List<String>> sampleMessagesMap, SampleMessageStore sampleMessageStore){
List<String> sampleMessages = sampleMessagesMap.get(apiInfoKey);
if (sampleMessages == null || sampleMessagesMap.isEmpty()) return null;

loggerMaker.infoAndAddToDb("Starting to find host for apiInfoKey: " + apiInfoKey.toString());

List<RawApi> messages = sampleMessageStore.fetchAllOriginalMessages(apiInfoKey);
if (messages.isEmpty()) return null;

OriginalHttpRequest originalHttpRequest = messages.get(0).getRequest();
return messages.get(0).getRequest();
}

public static String findHostFromOriginalHttpRequest(OriginalHttpRequest originalHttpRequest)
throws URISyntaxException {
String baseUrl = originalHttpRequest.getUrl();
if (baseUrl.startsWith("http")) {
URI uri = new URI(baseUrl);
String host = uri.getScheme() + "://" + uri.getHost();
return (uri.getPort() != -1) ? host + ":" + uri.getPort() : host;
return (uri.getPort() != -1) ? host + ":" + uri.getPort() : host;
} else {
return "https://" + originalHttpRequest.findHostFromHeader();
}
}

public static String findContentTypeFromOriginalHttpRequest(OriginalHttpRequest originalHttpRequest) {
Map<String, List<String>> headers = originalHttpRequest.getHeaders();
if (headers == null || headers.isEmpty()) {
return null;
}
final String CONTENT_TYPE = "content-type";
if (headers.containsKey(CONTENT_TYPE)) {
List<String> headerValues = headers.get(CONTENT_TYPE);
if (headerValues == null || headerValues.isEmpty()) {
return null;
}
return headerValues.get(0);
}
return null;
}


private LoginFlowResponse triggerLoginFlow(AuthMechanism authMechanism, int retries) {
LoginFlowResponse loginFlowResponse = null;
for (int i=0; i<retries; i++) {
Expand Down
55 changes: 52 additions & 3 deletions libs/dao/src/main/java/com/akto/util/HttpRequestResponseUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.akto.util;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.akto.dao.context.Context;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.types.CappedSet;
import com.akto.util.grpc.ProtoBufUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.BasicDBObject;
Expand All @@ -12,8 +14,6 @@
import java.util.*;

import static com.akto.dto.OriginalHttpRequest.*;
import static com.akto.util.grpc.ProtoBufUtils.DECODED_QUERY;
import static com.akto.util.grpc.ProtoBufUtils.RAW_QUERY;

public class HttpRequestResponseUtils {

Expand All @@ -22,6 +22,36 @@ public class HttpRequestResponseUtils {
public static final String FORM_URL_ENCODED_CONTENT_TYPE = "application/x-www-form-urlencoded";
public static final String GRPC_CONTENT_TYPE = "application/grpc";

public static List<SingleTypeInfo> generateSTIsFromPayload(int apiCollectionId, String url, String method,String body, int responseCode) {
int now = Context.now();
List<SingleTypeInfo> singleTypeInfos = new ArrayList<>();
Map<String, Set<Object>> respFlattened = extractValuesFromPayload(body);
for (String param: respFlattened.keySet()) {
// values is basically the type
Set<Object> values = respFlattened.get(param);
if (values == null || values.isEmpty()) continue;

ArrayList<Object> valuesList = new ArrayList<>(values);
String val = valuesList.get(0) == null ? null : valuesList.get(0).toString();
SingleTypeInfo.SubType subType = findSubType(val);
SingleTypeInfo.ParamId paramId = new SingleTypeInfo.ParamId(url, method,responseCode, false, param, subType, apiCollectionId, false);
SingleTypeInfo singleTypeInfo = new SingleTypeInfo(paramId, new HashSet<>(), new HashSet<>(), 0, now, 0, new CappedSet<>(), SingleTypeInfo.Domain.ANY, Long.MAX_VALUE, Long.MIN_VALUE);
singleTypeInfos.add(singleTypeInfo);
}

return singleTypeInfos;
}

public static SingleTypeInfo.SubType findSubType(String val) {
if (val == null) return SingleTypeInfo.GENERIC;
if (val.equalsIgnoreCase("short") || val.equalsIgnoreCase("int")) return SingleTypeInfo.INTEGER_32;
if (val.equalsIgnoreCase("long")) return SingleTypeInfo.INTEGER_64;
if (val.equalsIgnoreCase("float") || val.equalsIgnoreCase("double")) return SingleTypeInfo.FLOAT;
if (val.equalsIgnoreCase("boolean")) return SingleTypeInfo.TRUE;

return SingleTypeInfo.GENERIC;
}

public static Map<String, Set<Object>> extractValuesFromPayload(String body) {
if (body == null) return new HashMap<>();
if (body.startsWith("[")) body = "{\"json\": "+body+"}";
Expand Down Expand Up @@ -50,6 +80,25 @@ public static String rawToJsonString(String rawRequest, Map<String,List<String>>
return rawRequest;
}

public static String convertGRPCEncodedToJson(byte[] rawRequest) {
String base64 = Base64.getEncoder().encodeToString(rawRequest);

// empty grpc response, only headers present
if (rawRequest.length <= 5) {
return "{}";
}

try {
Map<Object, Object> map = ProtoBufUtils.getInstance().decodeProto(rawRequest);
if (map.isEmpty()) {
return base64;
}
return mapper.writeValueAsString(map);
} catch (Exception e) {
return base64;
}
}

public static String convertGRPCEncodedToJson(String rawRequest) {
try {
Map<Object, Object> map = ProtoBufUtils.getInstance().decodeProto(rawRequest);
Expand Down
Loading
Loading