[cli] Add exit code for processing errors

docs/pages/pmd/userdocs/cli_reference.md

@@ -5,6 +5,7 @@ tags: [userdocs]
 keywords: [command, line, options, help, formats, renderers]
 permalink: pmd_userdocs_cli_reference.html
 author: Tom Copeland <[email protected]>, Xavier Le Vourch <[email protected]>, Juan Martín Sotuyo Dodero <[email protected]>
+last_updated: May 2024 (7.2.0)
 ---
 
 
@@ -80,6 +81,11 @@ The tool comes with a rather extensive help text, simply running with `--help`!
  The valid values are the standard character sets of `java.nio.charset.Charset`."
  default="UTF-8"
  %}
+ {% include custom/cli_option_row.html options="--[no-]fail-on-processing-error"
+ description="Specifies whether PMD exits with non-zero status if processing errors occurred.
+ By default PMD exits with status 5 if processing errors or violations are found.
+ Disable this option with `--no-fail-on-processing-error` to exit with 0 instead and just write the report."
+ %}
  {% include custom/cli_option_row.html options="--[no-]fail-on-violation"
  description="Specifies whether PMD exits with non-zero status if violations are found.
  By default PMD exits with status 4 if violations are found.
@@ -208,16 +214,23 @@ Or you can set the environment variable `CLASSPATH` before starting PMD, e.g.
 
 ## Exit Status
 
-Please note that if PMD detects any violations, it will exit with status 4 (since 5.3).
+Please note that if PMD detects any violations, it will exit with status 4 (since 5.3) or 5 (since 7.2.0).
 This behavior has been introduced to ease PMD integration into scripts or hooks, such as SVN hooks.
 
 <table>
-<tr><td>0</td><td>Everything is fine, no violations found.</td></tr>
+<tr><td>0</td><td>Everything is fine, no violations found and no processing error occurred.</td></tr>
 <tr><td>1</td><td>PMD exited with an exception.</td></tr>
 <tr><td>2</td><td>Usage error. Command-line parameters are invalid or missing.</td></tr>
-<tr><td>4</td><td>At least one violation has been detected, unless <code>--no-fail-on-violation</code> is set.</td></tr>
+<tr><td>4</td><td>At least one violation has been detected, unless <code>--no-fail-on-violation</code> is set.<p>Since PMD 5.3.</p></td></tr>
+<tr><td>5</td><td>At least one processing error has occurred. There might be additionally zero or more violations detected.
+ To ignore processing errors, use <code>--no-fail-on-processing-error</code>.<p>Since PMD 7.2.0.</p></td></tr>
 </table>
 
+{%include note.html content="If PMD exits with 5, then PMD had either trouble parsing one or more files or a rule failed with an exception.
+That means, that either no violations for the entire file or for that rule are reported. These cases can be considered as false-negatives.
+In any case, the root cause should be investigated. If it's a problem in PMD itself, please create a bug report. Processing errors
+are usually part of the generated PMD report." %}
+
 ## Logging
 
 PMD internally uses [slf4j](https://www.slf4j.org/) and ships with slf4j-simple as the logging implementation.

docs/pages/pmd/userdocs/cpd/cpd.md

@@ -4,7 +4,7 @@ tags: [cpd, userdocs]
 summary: "Learn how to use CPD, the copy-paste detector shipped with PMD."
 permalink: pmd_userdocs_cpd.html
 author: Tom Copeland <[email protected]>
-last_updated: August 2023 (7.0.0)
+last_updated: May 2024 (7.2.0)
 ---
 
 ## Overview
@@ -150,6 +150,11 @@ exactly identical.
  If the root path is mentioned (e.g. \"/\" or \"C:\\\"), then the paths will be rendered
  as absolute."
  %}
+ {% include custom/cli_option_row.html options="--[no-]fail-on-processing-error"
+ description="Specifies whether CPD exits with non-zero status if processing errors occurred.
+ By default CPD exits with status 5 if processing errors or violations are found.
+ Disable this option with `--no-fail-on-processing-error` to exit with 0 instead and just write the report."
+ %}
  {% include custom/cli_option_row.html options="--[no-]fail-on-violation"
  description="Specifies whether CPD exits with non-zero status if violations are found.
  By default CPD exits with status 4 if violations are found.
@@ -279,16 +284,22 @@ If you specify a source directory but don't want to scan the sub-directories, yo
 
 ### Exit status
 
-Please note that if CPD detects duplicated source code, it will exit with status 4 (since 5.0).
+Please note that if CPD detects duplicated source code, it will exit with status 4 (since 5.0) or 5 (since 7.2.0).
 This behavior has been introduced to ease CPD integration into scripts or hooks, such as SVN hooks.
 
 <table>
-<tr><td>0</td><td>Everything is fine, no code duplications found.</td></tr>
+<tr><td>0</td><td>Everything is fine, no code duplications found and no processing errors occurred.</td></tr>
 <tr><td>1</td><td>CPD exited with an exception.</td></tr>
 <tr><td>2</td><td>Usage error. Command-line parameters are invalid or missing.</td></tr>
-<tr><td>4</td><td>At least one code duplication has been detected unless <code>--no-fail-on-violation</code> is set.</td></tr>
+<tr><td>4</td><td>At least one code duplication has been detected unless <code>--no-fail-on-violation</code> is set.<p>Since PMD 5.0.</p></td></tr>
+<tr><td>5</td><td>At least one processing error has occurred. There might be additionally zero or more duplications detected.
+ To ignore processing errors, use <code>--no-fail-on-processing-error</code>.<p>Since PMD 7.2.0.</p></td></tr>
 </table>
 
+{%include note.html content="If PMD exits with 5, then PMD had trouble lexing one or more files.
+That means, that no duplications for the entire file are reported. This can be considered as false-negative.
+In any case, the root cause should be investigated. If it's a problem in PMD itself, please create a bug report." %}
+
 ## Logging
 
 PMD internally uses [slf4j](https://www.slf4j.org/) and ships with slf4j-simple as the logging implementation.

docs/pages/pmd/userdocs/tools/ant.md

@@ -6,6 +6,7 @@ author: >
  David Dixon-Peugh <[email protected]>,
  Tom Copeland <[email protected]>,
  Xavier Le Vourch <[email protected]>
+last_updated: May 2024 (7.2.0)
 ---
 
 ## PMD
@@ -63,8 +64,8 @@ The examples below won't repeat this taskdef element, as this is always required
  <td>Yes, unless the ruleset nested element is used</td>
  </tr>
  <tr>
- <td>failonerror</td>
- <td>Whether or not to fail the build if any errors occur while processing the files</td>
+ <td>failOnError</td>
+ <td>Whether or not to fail the build if any processing errors occur while analyzing files.</td>
  <td>No</td>
  </tr>
  <tr>

docs/pages/release_notes.md

@@ -16,6 +16,8 @@ This is a {{ site.pmd.release_type }} release.
 
 ### 🐛 Fixed Issues
 
+* cli
+ * [#2827](https://github.com/pmd/pmd/issues/2827): \[cli] Consider processing errors in exit status
 * core
  * [#4978](https://github.com/pmd/pmd/issues/4978): \[core] Referenced Rulesets do not emit details on validation errors
  * [#4983](https://github.com/pmd/pmd/pull/4983): \[cpd] Fix CPD crashes about unicode escapes
@@ -33,6 +35,18 @@ This is a {{ site.pmd.release_type }} release.
 
 ### 🚨 API Changes
 
+#### CLI
+
+* New exit code 5 introduced. PMD and CPD will exit now by default with exit code 5, if any processing error
+ (e.g. parsing exception, lexing exception or rule exception) occurred. Such processing errors mean, that
+ either no violations or duplication for the entire file or for that rule are reported. These cases can be
+ considered as false-negatives.
+
+ In any case, the root cause should be investigated. If it's a problem in PMD itself, please create a bug report.
+
+* New CLI parameter `--no-fail-on-processing-error` to ignore processing errors and not exit with code 5. By default,
+ a build with processing errors will now fail and with that parameter, the previous behavior can be restored.
+
 #### Deprecated API
 
 * pmd-java

pmd-ant/src/main/java/net/sourceforge/pmd/ant/internal/PMDTaskImpl.java

@@ -144,7 +144,7 @@ private void doTask() {
  pmd.performAnalysis();
  stats = reportStatsListener.getResult();
  if (failOnError && pmd.getReporter().numErrors() > 0) {
- throw new BuildException("Some errors occurred while running PMD");
+ throw new BuildException("Some processing errors occurred while running PMD");
  }
  }
 

pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/AbstractAnalysisPmdSubcommand.java

@@ -48,6 +48,12 @@ public abstract class AbstractAnalysisPmdSubcommand<C extends AbstractConfigurat
  defaultValue = "true", negatable = true)
  protected boolean failOnViolation;
 
+ @Option(names = "--no-fail-on-processing-error",
+ description = "By default PMD exits with status 5 if processing errors or violations are found. "
+ + "Disable this option with '--no-fail-on-processing-error' to exit with 0 instead and just write the report.",
+ defaultValue = "true", negatable = true)
+ protected boolean failOnProcessingError;
+
  protected List<Path> relativizeRootPaths;
 
  @Option(names = { "--relativize-paths-with", "-z"}, description = "Path relative to which directories are rendered in the report. "

pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java

@@ -103,6 +103,7 @@ protected CPDConfiguration toConfiguration() {
  configuration.addRelativizeRoots(relativizeRootPaths);
  }
  configuration.setFailOnViolation(failOnViolation);
+ configuration.setFailOnProcessingError(failOnProcessingError);
  configuration.setInputFilePath(fileListPath);
  if (inputPaths != null) {
  configuration.setInputPathList(new ArrayList<>(inputPaths));
@@ -133,6 +134,11 @@ protected CPDConfiguration toConfiguration() {
  MutableBoolean hasViolations = new MutableBoolean();
  cpd.performAnalysis(report -> hasViolations.setValue(!report.getMatches().isEmpty()));
 
+ boolean hasProcessingErrors = configuration.getReporter().numErrors() > 0;
+ if (hasProcessingErrors && configuration.isFailOnProcessingError()) {
+ return CliExitCode.PROCESSING_ERRORS_OR_VIOLATIONS;
+ }
+
  if (hasViolations.booleanValue() && configuration.isFailOnViolation()) {
  return CliExitCode.VIOLATIONS_FOUND;
  }

pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/PmdCommand.java

@@ -270,6 +270,7 @@ protected PMDConfiguration toConfiguration() {
  configuration.setSuppressMarker(suppressMarker);
  configuration.setThreads(threads);
  configuration.setFailOnViolation(failOnViolation);
+ configuration.setFailOnProcessingError(failOnProcessingError);
  configuration.setAnalysisCacheLocation(cacheLocation != null ? cacheLocation.toString() : null);
  configuration.setIgnoreIncrementalAnalysis(noCache);
 
@@ -330,6 +331,8 @@ protected CliExitCode doExecute(PMDConfiguration configuration) {
  if (pmdReporter.numErrors() > 0) {
  // processing errors are ignored
  return CliExitCode.ERROR;
+ } else if (stats.getNumErrors() > 0 && configuration.isFailOnProcessingError()) {
+ return CliExitCode.PROCESSING_ERRORS_OR_VIOLATIONS;
  } else if (stats.getNumViolations() > 0 && configuration.isFailOnViolation()) {
  return CliExitCode.VIOLATIONS_FOUND;
  } else {

pmd-cli/src/main/java/net/sourceforge/pmd/cli/internal/CliExitCode.java

@@ -4,16 +4,16 @@
 
 package net.sourceforge.pmd.cli.internal;
 
-import net.sourceforge.pmd.PMDConfiguration;
+import net.sourceforge.pmd.AbstractConfiguration;
 
 /**
  * The execution result of any given command.
  */
 public enum CliExitCode {
- /** No errors, no violations. This is exit code {@code 0}. */
+ /** No errors, no processing errors, no violations. This is exit code {@code 0}. */
  OK(0),
  /**
- * Errors were detected, PMD may have not run to the end.
+ * Unexpected errors were detected, PMD may have not run to the end.
  * This is exit code {@code 1}.
  */
  ERROR(1),
@@ -23,11 +23,21 @@ public enum CliExitCode {
  */
  USAGE_ERROR(2),
  /**
- * No errors, but PMD found violations. This is exit code {@code 4}.
- * This is only returned if {@link PMDConfiguration#isFailOnViolation()}
- * is set (CLI flag {@code --failOnViolation}).
+ * No errors, but PMD found either duplications/violations or couldn't analyze all
+ * files due to parsing/lexing problems. This is exit code {@code 4}.
+ *
+ * <p>This is only returned if {@link AbstractConfiguration#isFailOnViolation()}
+ * is set. It can be disabled by using CLI flag {@code --no-fail-on-violation}.
  */
- VIOLATIONS_FOUND(4);
+ VIOLATIONS_FOUND(4),
+ /**
+ * PMD did run, but there was at least one processing error. There
+ * might be additionally duplications or violations. This is exit code {@code 5}.
+ *
+ * <p>This is only returned if {@link AbstractConfiguration#isFailOnProcessingError()}
+ * is set. It can be disabled by using CLI flag {@code --no-fail-on-processing-error}.
+ */
+ PROCESSING_ERRORS_OR_VIOLATIONS(5);
 
  private final int exitCode;
 
@@ -45,6 +55,7 @@ public static CliExitCode fromInt(int i) {
  case 1: return ERROR;
  case 2: return USAGE_ERROR;
  case 4: return VIOLATIONS_FOUND;
+ case 5: return PROCESSING_ERRORS_OR_VIOLATIONS;
  default:
  throw new IllegalArgumentException("Not a known exit code: " + i);
  }

pmd-cli/src/test/java/net/sourceforge/pmd/cli/CpdCliTest.java

@@ -5,6 +5,7 @@
 package net.sourceforge.pmd.cli;
 
 import static net.sourceforge.pmd.cli.internal.CliExitCode.OK;
+import static net.sourceforge.pmd.cli.internal.CliExitCode.PROCESSING_ERRORS_OR_VIOLATIONS;
 import static net.sourceforge.pmd.cli.internal.CliExitCode.VIOLATIONS_FOUND;
 import static net.sourceforge.pmd.util.CollectionUtil.listOf;
 import static org.hamcrest.CoreMatchers.startsWith;
@@ -128,14 +129,14 @@ void testWrongCliOptionsDoPrintUsage() throws Exception {
  @Test
  void testWrongCliOptionResultsInErrorLoggingAfterDir() throws Exception {
  // --ignore-identifiers doesn't take an argument anymore - it is interpreted as a file for inputPaths
- final CliExecutionResult result = runCli(VIOLATIONS_FOUND, "--minimum-tokens", "34", "--dir", SRC_DIR, "--ignore-identifiers", "false");
+ final CliExecutionResult result = runCli(PROCESSING_ERRORS_OR_VIOLATIONS, "--minimum-tokens", "34", "--dir", SRC_DIR, "--ignore-identifiers", "false");
  result.checkStdErr(containsString("No such file false"));
  }
 
  @Test
  void testWrongCliOptionResultsInErrorLoggingBeforeDir() throws Exception {
  // --ignore-identifiers doesn't take an argument anymore - it is interpreted as a file for inputPaths
- final CliExecutionResult result = runCli(VIOLATIONS_FOUND, "--minimum-tokens", "34", "--ignore-identifiers", "false", "--dir", SRC_DIR);
+ final CliExecutionResult result = runCli(PROCESSING_ERRORS_OR_VIOLATIONS, "--minimum-tokens", "34", "--ignore-identifiers", "false", "--dir", SRC_DIR);
  result.checkStdErr(containsString("No such file false"));
  }
 
@@ -152,7 +153,7 @@ void testFindJavaDuplication() throws Exception {
  */
  @Test
  void testIgnoreIdentifiers() throws Exception {
- runCli(VIOLATIONS_FOUND, "--minimum-tokens", "34", "--dir", SRC_DIR, "--ignore-identifiers", "false", "--debug")
+ runCli(VIOLATIONS_FOUND, "--minimum-tokens", "34", "--dir", SRC_DIR, "--ignore-identifiers", "--debug")
  .verify(result -> result.checkStdOut(containsString(
  "Found a 14 line (89 tokens) duplication"
  )));
@@ -220,7 +221,7 @@ void testEncodingOption() throws Exception {
  */
  @Test
  void testSkipLexicalErrors() throws Exception {
- runCli(VIOLATIONS_FOUND,
+ runCli(PROCESSING_ERRORS_OR_VIOLATIONS,
  "--minimum-tokens", "10",
  "-d", BASE_RES_PATH + "badandgood/",
  "--format", "text",
@@ -231,6 +232,43 @@ void testSkipLexicalErrors() throws Exception {
  });
  }
 
+ @Test
+ void testExitCodeWithLexicalErrors() throws Exception {
+ runCli(PROCESSING_ERRORS_OR_VIOLATIONS,
+ "--minimum-tokens", "10",
+ "-d", Paths.get(BASE_RES_PATH, "badandgood", "BadFile.java").toString(),
+ "--format", "text")
+ .verify(r -> {
+ r.checkStdErr(containsPattern("Error while tokenizing: Lexical error in file '.*?BadFile\\.java'"));
+ r.checkStdOut(emptyString());
+ });
+ }
+
+ @Test
+ void testExitCodeWithLexicalErrorsNoFail() throws Exception {
+ runCli(OK,
+ "--minimum-tokens", "10",
+ "-d", Paths.get(BASE_RES_PATH, "badandgood", "BadFile.java").toString(),
+ "--format", "text",
+ "--no-fail-on-processing-error")
+ .verify(r -> {
+ r.checkStdErr(containsPattern("Error while tokenizing: Lexical error in file '.*?BadFile\\.java'"));
+ r.checkStdOut(emptyString());
+ });
+ }
+
+ @Test
+ void testExitCodeWithLexicalErrorsAndSkipLexical() throws Exception {
+ runCli(PROCESSING_ERRORS_OR_VIOLATIONS,
+ "--minimum-tokens", "10",
+ "-d", Paths.get(BASE_RES_PATH, "badandgood", "BadFile.java").toString(),
+ "--format", "text",
+ "--skip-lexical-errors")
+ .verify(r -> {
+ r.checkStdErr(containsPattern("Skipping file: Lexical error in file .*?BadFile\\.java"));
+ r.checkStdOut(emptyString());
+ });
+ }
 
  @Test
  void jsShouldFindDuplicatesWithDifferentFileExtensions() throws Exception {

pmd-cli/src/test/java/net/sourceforge/pmd/cli/PmdCliTest.java

@@ -6,6 +6,7 @@
 
 import static net.sourceforge.pmd.cli.internal.CliExitCode.ERROR;
 import static net.sourceforge.pmd.cli.internal.CliExitCode.OK;
+import static net.sourceforge.pmd.cli.internal.CliExitCode.PROCESSING_ERRORS_OR_VIOLATIONS;
 import static net.sourceforge.pmd.cli.internal.CliExitCode.USAGE_ERROR;
 import static net.sourceforge.pmd.cli.internal.CliExitCode.VIOLATIONS_FOUND;
 import static net.sourceforge.pmd.util.CollectionUtil.listOf;
@@ -318,6 +319,27 @@ void exitStatusWithViolations() throws Exception {
  ));
  }
 
+ @Test
+ void exitStatusWithProcessingErrors() throws Exception {
+ runCli(PROCESSING_ERRORS_OR_VIOLATIONS, "--use-version", "dummy-parserThrows",
+ "-d", srcDir.toString(), "-f", "text", "-R", RULESET_WITH_VIOLATION)
+ .verify(r -> {
+ r.checkStdOut(containsString("someSource.dummy\t-\tParseException: Parse exception: ohio"));
+ r.checkStdErr(containsString("An error occurred while executing PMD."));
+ });
+ }
+
+ @Test
+ void exitStatusWithProcessingErrorsNoFail() throws Exception {
+ runCli(OK, "--use-version", "dummy-parserThrows",
+ "-d", srcDir.toString(), "-f", "text", "-R", RULESET_WITH_VIOLATION,
+ "--no-fail-on-processing-error")
+ .verify(r -> {
+ r.checkStdOut(containsString("someSource.dummy\t-\tParseException: Parse exception: ohio"));
+ r.checkStdErr(containsString("An error occurred while executing PMD."));
+ });
+ }
+
  @Test
  void testZipFileAsSource() throws Exception {
  Path zipArchive = createTemporaryZipArchive("sources.zip");

pmd-cli/src/test/resources/net/sourceforge/pmd/cli/RuleSetWithViolations.xml

@@ -3,12 +3,12 @@
  xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
 
  <description>
- Ruleset used by test RuleSetFactoryTest
+ Ruleset used by test net.sourceforge.pmd.cli.PmdCliTest
  </description>
 
  <rule name="ReportAllRootNodes" language="dummy" since="1.0" message="Violation from ReportAllRootNodes"
  class="net.sourceforge.pmd.lang.rule.xpath.XPathRule"
- externalInfoUrl="${pmd.website.baseurl}/rules/test/TestRuleset3.xml#Ruleset3Rule1">
+ externalInfoUrl="${pmd.website.baseurl}/rules/test/RuleSetWithViolations.xml#ReportAllRootNodes">
  <description>Just for test</description>
  <priority>3</priority>
  <properties>