Skip to content

Commit

Permalink
refactor: Separate queue service and implement factory for webhooks (#…
Browse files Browse the repository at this point in the history 
…599)

* refactor: Separate queue service and implement factory for webhooks

Signed-off-by: Oleg Kopysov <[email protected]>

* fix: Fix for the async start of threds

Signed-off-by: Oleg Kopysov <[email protected]>

* fix: Apply fix after code review

Signed-off-by: Oleg Kopysov <[email protected]>

---------

Signed-off-by: Oleg Kopysov <[email protected]>
  • Loading branch information
@o-kopysov
o-kopysov authored Sep 10, 2024
1 parent 5485f13 commit dce00d7
Show file tree
Hide file tree
Showing 11 changed files with 1,269 additions and 998 deletions.
11 changes: 11 additions & 0 deletions src/main/java/com/lpvs/entity/report/package-info.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* Copyright (c) 2024, Samsung Electronics Co., Ltd. All rights reserved.
*
* Use of this source code is governed by a MIT license that can be
* found in the LICENSE file.
*/

/**
* This package contains entity classes that represent various types of reports generated by the LPVS service.
*/
package com.lpvs.entity.report;
17 changes: 13 additions & 4 deletions src/main/java/com/lpvs/service/LPVSQueueProcessorService.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
/**
* Copyright (c) 2022, Samsung Electronics Co., Ltd. All rights reserved.
* Copyright (c) 2024, Samsung Electronics Co., Ltd. All rights reserved.
*
* Use of this source code is governed by a MIT license that can be
* found in the LICENSE file.
*/
package com.lpvs.service;

import com.lpvs.entity.LPVSQueue;
import com.lpvs.service.webhook.LPVSWebhookService;
import com.lpvs.service.webhook.LPVSWebhookServiceFactory;
import io.micrometer.common.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -41,13 +43,20 @@ public class LPVSQueueProcessorService {
@Value("${local.path:}")
private String localPath;

@Autowired private LPVSWebhookService webhookService;

/**
* Constructor for LPVSQueueProcessorService.
*
* @param queueService The LPVSQueueService to be injected.
* @param webhookServiceFactory Service for creating instance of the webhook service.
* @param isInternal Indicates the mode of LPVS operation.
*/
@Autowired
LPVSQueueProcessorService(LPVSQueueService queueService) {
LPVSQueueProcessorService(
LPVSQueueService queueService,
LPVSWebhookServiceFactory webhookServiceFactory,
@Value("${lpvs.mode.internal:false}") boolean isInternal) {
this.webhookService = webhookServiceFactory.createWebhookService(isInternal);
this.queueService = queueService;
}

Expand All @@ -72,7 +81,7 @@ protected void queueProcessor() throws Exception {
webhookConfig.setDate(new Date());

// Process the LPVSQueue element.
queueService.processWebHook(webhookConfig);
webhookService.processWebHook(webhookConfig);
}
}
}
194 changes: 6 additions & 188 deletions src/main/java/com/lpvs/service/LPVSQueueService.java
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,16 @@
/**
* Copyright (c) 2022, Samsung Electronics Co., Ltd. All rights reserved.
* Copyright (c) 2024, Samsung Electronics Co., Ltd. All rights reserved.
*
* Use of this source code is governed by a MIT license that can be
* found in the LICENSE file.
*/
package com.lpvs.service;

import com.lpvs.entity.LPVSFile;
import com.lpvs.entity.LPVSLicense;
import com.lpvs.entity.LPVSPullRequest;
import com.lpvs.entity.LPVSQueue;
import com.lpvs.entity.enums.LPVSPullRequestStatus;
import com.lpvs.repository.LPVSPullRequestRepository;
import com.lpvs.repository.LPVSQueueRepository;
import com.lpvs.service.scan.LPVSDetectService;
import com.lpvs.util.LPVSPayloadUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
Expand All @@ -33,35 +22,10 @@
@Slf4j
public class LPVSQueueService {

/**
* Service for interacting with GitHub.
*/
private LPVSGitHubService gitHubService;

/**
* Service for detecting licenses in scanned files.
*/
private LPVSDetectService detectService;

/**
* Service for managing licenses and license conflicts.
*/
private LPVSLicenseService licenseService;

/**
* Repository for storing LPVSPullRequest entities.
*/
private LPVSPullRequestRepository lpvsPullRequestRepository;

/**
* Repository for storing LPVSQueue entities.
*/
private LPVSQueueRepository queueRepository;

/**
* Maximum attempts for processing LPVSQueue elements.
*/
private int maxAttempts;
private final LPVSQueueRepository queueRepository;

/**
* BlockingDeque for managing LPVSQueue elements.
Expand All @@ -71,26 +35,11 @@ public class LPVSQueueService {
/**
* Constructor for LPVSQueueService.
*
* @param gitHubService Service for interacting with GitHub.
* @param detectService Service for detecting licenses in scanned files.
* @param licenseService Service for managing licenses and license conflicts.
* @param lpvsPullRequestRepository Repository for storing LPVSPullRequest entities.
* @param queueRepository Repository for storing LPVSQueue entities.
* @param maxAttempts Maximum attempts for processing LPVSQueue elements.
* @param queueRepository Repository for storing LPVSQueue entities.
*/
public LPVSQueueService(
LPVSGitHubService gitHubService,
LPVSDetectService detectService,
LPVSLicenseService licenseService,
LPVSPullRequestRepository lpvsPullRequestRepository,
LPVSQueueRepository queueRepository,
@Value("${lpvs.attempts:4}") int maxAttempts) {
this.gitHubService = gitHubService;
this.detectService = detectService;
this.licenseService = licenseService;
this.lpvsPullRequestRepository = lpvsPullRequestRepository;
@Autowired
public LPVSQueueService(LPVSQueueRepository queueRepository) {
this.queueRepository = queueRepository;
this.maxAttempts = maxAttempts;
}

/**
Expand Down Expand Up @@ -156,135 +105,4 @@ public void checkForQueue() throws InterruptedException {
QUEUE.putFirst(webhook);
}
}

/**
* Gets the LPVSQueue element with the latest scan date.
*
* @param webhookConfigList The list of LPVSQueue elements.
* @return The LPVSQueue element with the latest scan date.
*/
public LPVSQueue getLatestScan(List<LPVSQueue> webhookConfigList) {
LPVSQueue latestWebhookConfig = webhookConfigList.get(0);
for (LPVSQueue webhookConfig : webhookConfigList) {
if (latestWebhookConfig.getDate().compareTo(webhookConfig.getDate()) < 0) {
latestWebhookConfig = webhookConfig;
}
}
return latestWebhookConfig;
}

/**
* Asynchronously processes the LPVSQueue element, handling GitHub webhook events.
*
* @param webhookConfig The LPVSQueue element to be processed.
*/
@Async("threadPoolTaskExecutor")
public void processWebHook(LPVSQueue webhookConfig) {
Long id = webhookConfig.getId();
log.info(
"Processing webhook ID: "
+ id
+ ", attempt: "
+ (webhookConfig.getAttempts() + 1)
+ " for PR: "
+ webhookConfig.getPullRequestUrl());
LPVSPullRequest pullRequest =
lpvsPullRequestRepository.findLatestByPullRequestInfo(
webhookConfig.getUserId(),
LPVSPayloadUtil.getRepositoryOrganization(webhookConfig)
+ "/"
+ LPVSPayloadUtil.getRepositoryName(webhookConfig),
webhookConfig.getPullRequestFilesUrl(),
webhookConfig.getPullRequestHead(),
webhookConfig.getPullRequestBase(),
webhookConfig.getSender(),
LPVSPullRequestStatus.INTERNAL_ERROR.getPullRequestStatus());

if (pullRequest == null) {
pullRequest = new LPVSPullRequest();
pullRequest.setUser(webhookConfig.getUserId());
pullRequest.setRepositoryName(
LPVSPayloadUtil.getRepositoryOrganization(webhookConfig)
+ "/"
+ LPVSPayloadUtil.getRepositoryName(webhookConfig));
pullRequest.setPullRequestUrl(webhookConfig.getPullRequestUrl());
pullRequest.setPullRequestFilesUrl(webhookConfig.getPullRequestFilesUrl());
pullRequest.setPullRequestHead(webhookConfig.getPullRequestHead());
pullRequest.setPullRequestBase(webhookConfig.getPullRequestBase());
pullRequest.setSender(webhookConfig.getSender());
}

try {

pullRequest.setDate(webhookConfig.getDate());
pullRequest.setStatus(LPVSPullRequestStatus.SCANNING.toString());
pullRequest = lpvsPullRequestRepository.saveAndFlush(pullRequest);

String filePath = gitHubService.getPullRequestFiles(webhookConfig);
if (filePath != null && Files.list(Paths.get(filePath)).count() != 0) {
log.debug("Successfully downloaded files");

if (filePath.contains(":::::")) {
filePath = filePath.split(":::::")[0];
}
// check repository license
String repositoryLicense = gitHubService.getRepositoryLicense(webhookConfig);

if (repositoryLicense != null) {
LPVSLicense repoLicense =
licenseService.getLicenseBySpdxIdAndName(
repositoryLicense, Optional.empty());
webhookConfig.setRepositoryLicense(repoLicense.getSpdxId());
} else {
webhookConfig.setRepositoryLicense(null);
}
log.debug("Repository license: " + webhookConfig.getRepositoryLicense());

List<LPVSFile> files = detectService.runScan(webhookConfig, filePath);

// check license conflicts
List<LPVSLicenseService.Conflict<String, String>> detectedConflicts =
licenseService.findConflicts(webhookConfig, files);

log.debug("Creating comment");
gitHubService.commentResults(webhookConfig, files, detectedConflicts, pullRequest);
log.debug("Results posted on GitHub");
delete(webhookConfig);
} else {
log.warn("Files are not found. Probably pull request does not exist.");
throw new Exception(
"Files are not found. Probably pull request does not exist. Terminating.");
}
} catch (Exception | Error e) {
pullRequest.setStatus(LPVSPullRequestStatus.INTERNAL_ERROR.toString());
pullRequest = lpvsPullRequestRepository.saveAndFlush(pullRequest);
log.error("Can't authorize commentResults() " + e.getMessage());
int currentAttempts = webhookConfig.getAttempts() + 1;
if (currentAttempts < maxAttempts) {
webhookConfig.setAttempts(currentAttempts);
try {
addFirst(webhookConfig);
} catch (InterruptedException e1) {
log.warn("Failed to update Queue element");
}
queueRepository.save(webhookConfig);
} else {
log.warn(
"Maximum amount of processing webhook reached for pull request: "
+ pullRequest.getId()
+ " "
+ pullRequest.getPullRequestUrl());
try {
gitHubService.commentResults(webhookConfig, null, null, pullRequest);
} catch (Exception ex) {
log.warn("Failed to post FAIL result " + ex.getMessage());
}
delete(webhookConfig);
log.info(
"Webhook ID: "
+ id
+ " - removed from the queue because the number of attempts exceeded the max value");
}
}
}
}
22 changes: 22 additions & 0 deletions src/main/java/com/lpvs/service/webhook/LPVSWebhookService.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/**
* Copyright (c) 2024, Samsung Electronics Co., Ltd. All rights reserved.
*
* Use of this source code is governed by a MIT license that can be
* found in the LICENSE file.
*/
package com.lpvs.service.webhook;

import com.lpvs.entity.LPVSQueue;

/**
* The LPVSWebhookService interface defines the contract for processing LPVSQueue elements and handling webhook events.
*/
public interface LPVSWebhookService {

/**
* Processes the LPVSQueue element, handling webhook events.
*
* @param webhookConfig The LPVSQueue element to be processed.
*/
void processWebHook(LPVSQueue webhookConfig);
}
Loading

0 comments on commit dce00d7

Please sign in to comment.