Skip to content

Commit 0421f12

Browse files
sourabhsparkalasourabhsparkala
authored
Static analysis tool providers from Prospector Project (#866)
* Static analysis tool providers from Prospector Project Fixes #730
1 parent 38b8bbc commit 0421f12

File tree

51 files changed

+3303
-413
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+3303
-413
lines changed

‎src/main/java/com/sap/oss/phosphor/fosstars/data/AbstractStaticScanToolsDataProvider.java

Lines changed: 375 additions & 13 deletions
Large diffs are not rendered by default.

‎src/main/java/com/sap/oss/phosphor/fosstars/data/DataProviderSelector.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.sap.oss.phosphor.fosstars.data.github.IsEclipse;
3737
import com.sap.oss.phosphor.fosstars.data.github.LgtmDataProvider;
3838
import com.sap.oss.phosphor.fosstars.data.github.LicenseInfo;
39+
import com.sap.oss.phosphor.fosstars.data.github.MyPyDataProvider;
3940
import com.sap.oss.phosphor.fosstars.data.github.NumberOfCommits;
4041
import com.sap.oss.phosphor.fosstars.data.github.NumberOfContributors;
4142
import com.sap.oss.phosphor.fosstars.data.github.NumberOfDependentProjectOnGitHub;
@@ -44,6 +45,7 @@
4445
import com.sap.oss.phosphor.fosstars.data.github.OwaspSecurityLibraries;
4546
import com.sap.oss.phosphor.fosstars.data.github.PackageManagement;
4647
import com.sap.oss.phosphor.fosstars.data.github.ProgrammingLanguages;
48+
import com.sap.oss.phosphor.fosstars.data.github.PylintDataProvider;
4749
import com.sap.oss.phosphor.fosstars.data.github.ReadmeInfo;
4850
import com.sap.oss.phosphor.fosstars.data.github.ReleasesFromGitHub;
4951
import com.sap.oss.phosphor.fosstars.data.github.SecurityReviewsFromOpenSSF;
@@ -239,6 +241,8 @@ public DataProviderSelector(GitHubDataFetcher fetcher, NVD nvd) throws IOExcepti
239241
new VulnerabilitiesFromOwaspDependencyCheck(),
240242
new VulnerabilitiesFromNpmAudit(nvd),
241243
new HasExecutableBinaries(fetcher),
244+
new PylintDataProvider(fetcher),
245+
new MyPyDataProvider(fetcher),
242246
PROJECT_USAGE_PROVIDER,
243247
FUNCTIONALITY_PROVIDER,
244248
HANDLING_UNTRUSTED_DATA_LIKELIHOOD_PROVIDER,

‎src/main/java/com/sap/oss/phosphor/fosstars/data/StaticAnalysisScanTool.java

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎src/main/java/com/sap/oss/phosphor/fosstars/data/github/BanditDataProvider.java

Lines changed: 22 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -10,31 +10,36 @@
1010
import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
1111
import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
1212
import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
13-
import com.sap.oss.phosphor.fosstars.util.Yaml;
1413
import java.io.IOException;
15-
import java.io.InputStream;
16-
import java.nio.file.Files;
17-
import java.nio.file.Path;
14+
import java.util.Collections;
15+
import java.util.HashMap;
1816
import java.util.Map;
19-
import java.util.Optional;
17+
import java.util.function.Predicate;
2018
import java.util.regex.Pattern;
21-
import org.apache.commons.collections4.IteratorUtils;
2219

2320
/**
2421
* The data provider gathers info about how a project uses Bandit for static analysis. In
2522
* particular, it tries to fill out the following features:
2623
* <ul>
27-
* <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
28-
* <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
24+
* <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
25+
* <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
2926
* </ul>
3027
*/
3128
public class BanditDataProvider extends AbstractStaticScanToolsDataProvider {
3229

3330
/**
34-
* A step in a GitHub action that triggers analysis with Bandit.
31+
* A Predicate to check the any step in a GitHub action that triggers analysis with Bandit.
3532
*/
36-
private static final Pattern RUN_STEP_BANDIT_REGEX_PATTERN
37-
= Pattern.compile("^.*bandit .*$", Pattern.DOTALL);
33+
private static final Map<String, Predicate<String>> MATCH_BANDIT_PREDICATE = new HashMap<>();
34+
35+
static {
36+
{
37+
MATCH_BANDIT_PREDICATE.put("uses",
38+
step -> Pattern.compile(".*bandit.*$", Pattern.DOTALL).matcher(step).matches());
39+
MATCH_BANDIT_PREDICATE.put("run",
40+
step -> Pattern.compile("^.*bandit .*$", Pattern.DOTALL).matcher(step).matches());
41+
}
42+
}
3843

3944
/**
4045
* Initializes a data provider.
@@ -51,69 +56,14 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {
5156

5257
LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);
5358

54-
Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(false);
55-
Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(false);
56-
5759
// ideally, we're looking for a GitHub action that runs Bandit scan on pull requests
5860
// but if we just find an action that runs Bandit scans, that's also fine
59-
for (Path configPath : findGitHubActionsIn(repository)) {
60-
try (InputStream content = Files.newInputStream(configPath)) {
61-
Map<String, Object> githubAction = Yaml.readMap(content);
62-
if (triggersScan(githubAction)) {
63-
runsBandit = RUNS_BANDIT_SCANS.value(true);
64-
if (runsOnPullRequests(githubAction)) {
65-
usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(true);
66-
break;
67-
}
68-
}
69-
}
70-
}
61+
Visitor visitor = withVisitor();
62+
browse(repository, MATCH_BANDIT_PREDICATE, Collections.emptyMap(), visitor);
7163

72-
return ValueHashSet.from(runsBandit, usesBanditScanChecks);
73-
}
74-
75-
@Override
76-
public boolean triggersScan(Map<?, ?> githubAction) {
77-
return Optional.ofNullable(githubAction.get("jobs"))
78-
.filter(Map.class::isInstance)
79-
.map(Map.class::cast)
80-
.map(jobs -> jobs.values())
81-
.filter(Iterable.class::isInstance)
82-
.map(Iterable.class::cast)
83-
.map(BanditDataProvider::scanJobs)
84-
.orElse(false);
85-
}
64+
Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(visitor.runCheck);
65+
Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(visitor.usesCheck);
8666

87-
/**
88-
* Checks if any step in a collection of jobs triggers a Bandit scan.
89-
*
90-
* @param jobs The collection of jobs from GitHub action.
91-
* @return True if a step triggers a Bandit scan, false otherwise.
92-
*/
93-
private static boolean scanJobs(Iterable<?> jobs) {
94-
return IteratorUtils.toList(jobs.iterator()).stream()
95-
.filter(Map.class::isInstance)
96-
.map(Map.class::cast)
97-
.map(job -> job.get("steps"))
98-
.filter(Iterable.class::isInstance)
99-
.map(Iterable.class::cast)
100-
.anyMatch(BanditDataProvider::hasBanditRunStep);
101-
}
102-
103-
/**
104-
* Checks if a collection of steps from a GitHub action contains a step that triggers a Bandit
105-
* scan.
106-
*
107-
* @param steps The steps to be checked.
108-
* @return True if the steps contain a step that triggers a Bandit scan, false otherwise.
109-
*/
110-
private static boolean hasBanditRunStep(Iterable<?> steps) {
111-
return IteratorUtils.toList(steps.iterator()).stream()
112-
.filter(Map.class::isInstance)
113-
.map(Map.class::cast)
114-
.map(step -> step.get("run"))
115-
.filter(String.class::isInstance)
116-
.map(String.class::cast)
117-
.anyMatch(run -> RUN_STEP_BANDIT_REGEX_PATTERN.matcher(run).matches());
67+
return ValueHashSet.from(runsBandit, usesBanditScanChecks);
11868
}
119-
}
69+
}

‎src/main/java/com/sap/oss/phosphor/fosstars/data/github/CodeqlDataProvider.java

Lines changed: 21 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -10,31 +10,34 @@
1010
import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
1111
import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
1212
import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
13-
import com.sap.oss.phosphor.fosstars.util.Yaml;
1413
import java.io.IOException;
15-
import java.io.InputStream;
16-
import java.nio.file.Files;
17-
import java.nio.file.Path;
14+
import java.util.Collections;
15+
import java.util.HashMap;
1816
import java.util.Map;
19-
import java.util.Optional;
20-
import org.apache.commons.collections4.IteratorUtils;
17+
import java.util.function.Predicate;
2118

2219
/**
23-
* The data provider gathers info about how a project uses CodeQL for static analysis.
24-
* In particular, it tires to fill out the following features:
20+
* The data provider gathers info about how a project uses CodeQL for static analysis. In
21+
* particular, it tires to fill out the following features:
2522
* <ul>
26-
* <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
27-
* <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
23+
* <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
24+
* <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
2825
* </ul>
2926
*
3027
* @see LgtmDataProvider
3128
*/
3229
public class CodeqlDataProvider extends AbstractStaticScanToolsDataProvider {
3330

3431
/**
35-
* A step in a GitHub action that triggers analysis with CodeQL.
32+
* A predicate to check if any step in GitHub action that triggers analysis with CodeQL.
3633
*/
37-
private static final String CODEQL_ANALYZE_STEP_TASK = "github/codeql-action/analyze";
34+
private static final Map<String, Predicate<String>> MATCH_CODEQL_ANALYZE_PREDICATE =
35+
new HashMap<>();
36+
37+
static {
38+
MATCH_CODEQL_ANALYZE_PREDICATE.put("uses",
39+
uses -> uses.startsWith("github/codeql-action/analyze"));
40+
}
3841

3942
/**
4043
* Initializes a data provider.
@@ -51,56 +54,12 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {
5154

5255
LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);
5356

54-
Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(false);
55-
Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(false);
56-
57-
// ideally, we're looking for a GitHub action that runs CodeQL scan on pull requests
58-
// but if we just find an action that runs CodeQL scans, that's also fine
59-
for (Path configPath : findGitHubActionsIn(repository)) {
60-
try (InputStream content = Files.newInputStream(configPath)) {
61-
Map<String, Object> githubAction = Yaml.readMap(content);
62-
if (triggersScan(githubAction)) {
63-
runsCodeqlScans = RUNS_CODEQL_SCANS.value(true);
64-
if (runsOnPullRequests(githubAction)) {
65-
usesCodeqlChecks = USES_CODEQL_CHECKS.value(true);
66-
break;
67-
}
68-
}
69-
}
70-
}
71-
72-
return ValueHashSet.from(usesCodeqlChecks, runsCodeqlScans);
73-
}
57+
Visitor visitor = withVisitor();
58+
browse(repository, MATCH_CODEQL_ANALYZE_PREDICATE, Collections.emptyMap(), visitor);
7459

75-
@Override
76-
public boolean triggersScan(Map<?, ?> githubAction) {
77-
return Optional.ofNullable(githubAction.get("jobs"))
78-
.filter(Map.class::isInstance)
79-
.map(Map.class::cast)
80-
.map(jobs -> jobs.get("analyze"))
81-
.filter(Map.class::isInstance)
82-
.map(Map.class::cast)
83-
.map(jobs -> jobs.get("steps"))
84-
.filter(Iterable.class::isInstance)
85-
.map(Iterable.class::cast)
86-
.map(CodeqlDataProvider::hasCodeqlAnalyzeStep)
87-
.orElse(false);
88-
}
60+
Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(visitor.runCheck);
61+
Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(visitor.usesCheck);
8962

90-
/**
91-
* Checks if a collection of steps from a GitHub action contains a step that triggers
92-
* a CodeQL scan.
93-
*
94-
* @param steps The steps to be checked.
95-
* @return True if the steps contain a step that triggers a CodeQL scan, false otherwise.
96-
*/
97-
private static boolean hasCodeqlAnalyzeStep(Iterable<?> steps) {
98-
return IteratorUtils.toList(steps.iterator()).stream()
99-
.filter(Map.class::isInstance)
100-
.map(Map.class::cast)
101-
.map(step -> step.get("uses"))
102-
.filter(String.class::isInstance)
103-
.map(String.class::cast)
104-
.anyMatch(uses -> uses.startsWith(CODEQL_ANALYZE_STEP_TASK));
63+
return ValueHashSet.from(runsCodeqlScans, usesCodeqlChecks);
10564
}
106-
}
65+
}

0 commit comments

Comments
 (0)