Skip to content

Report logs only on failures in JUnit and ScalaTest #13721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hubertp wants to merge 19 commits into
base: develop
Choose a base branch
from
Open

Report logs only on failures in JUnit and ScalaTest #13721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
5c998fa
Report logs only on failures in JUnit and ScalaTest
hubertp Aug 1, 2025
86586d8
Further transition of JUnit tests
hubertp Aug 4, 2025
03f43da
Increase default log level for tests
hubertp Aug 4, 2025
681ad5a
DRY when creating context with test log handler
hubertp Aug 4, 2025
25a1f80
More DRY
hubertp Aug 4, 2025
3ae478a
Added appender rule to remaining tests
hubertp Aug 5, 2025
780dba2
Use customn log handler by default
hubertp Aug 5, 2025
6a2415d
Move remaining Scala tests to reporting logs on failure
hubertp Aug 5, 2025
34ec594
Drop `order` in Rule
hubertp Aug 5, 2025
212da4c
More coverage
hubertp Aug 5, 2025
f8c2148
fmt
hubertp Aug 5, 2025
0bb299e
fix benchmarks that expect output in stdout
hubertp Aug 6, 2025
37f1b8b
Merge branch 'develop' into wip/hubert/13308-logs-on-failure
hubertp Aug 6, 2025
25a5b7f
Merge branch 'develop' into wip/hubert/13308-logs-on-failure
hubertp Aug 11, 2025
c8ed19f
Avoid concurrent modification errors
hubertp Aug 11, 2025
62c6f5d
Address PR comments
hubertp Aug 11, 2025
f43f618
Merge branch 'develop' into wip/hubert/13308-logs-on-failure
hubertp Aug 12, 2025
cb9e5de
Merge branch 'develop' into wip/hubert/13308-logs-on-failure
hubertp Aug 15, 2025
5cd4c6e
Fix benchmark runs
hubertp Aug 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1694,7 +1694,7 @@ lazy val testkit = project
Compile / exportedModule := (Compile / exportedModuleBin).value,
Compile / exportedModuleBin := (Compile / packageBin).value
)
.dependsOn(`logging-service-logback`)
.dependsOn(`logging-service-logback` % "compile->test")
.dependsOn(`runtime-utils`)

lazy val searcher = project
Expand Down Expand Up @@ -2726,6 +2726,7 @@ lazy val `runtime-integration-tests` =
"-Dtck.inlineVerifierInstrument=false",
"-Dpolyglot.engine.AllowExperimentalOptions=true"
),
Test / javacOptions += "-implicit:none",
Copy link
Collaborator Author

@hubertp hubertp Aug 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not related to this PR but reduces warnings noise.

Test / javaOptions ++= testLogProviderOptions,
Test / moduleDependencies := {
GraalVM.modules ++ GraalVM.langsPkgs ++ GraalVM.insightPkgs ++ logbackPkg ++ helidon ++ scalaLibrary ++ scalaReflect ++ slf4jApi ++ Seq(
Expand Down
2 changes: 2 additions & 0 deletions ...marks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import org.enso.common.RuntimeOptions;
import org.enso.compiler.Compiler;
import org.enso.compiler.benchmarks.Utils;
Expand Down Expand Up @@ -51,6 +52,7 @@ public void setup() throws IOException {
ctx =
Utils.createDefaultContextBuilder()
.withModifiedContext(bldr -> bldr.option(RuntimeOptions.STRICT_ERRORS, "false"))
.withStdOutLogHandler(Level.WARNING)
.build();
var ensoCtx = ctx.ensoContext();
compiler = ensoCtx.getCompiler();
Expand Down
7 changes: 6 additions & 1 deletion ...runtime-integration-tests/src/test/java/org/enso/common/test/ContextInsightSetupTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@
import org.enso.common.ContextFactory;
import org.enso.common.LanguageInfo;
import org.enso.common.MethodNames.Module;
import org.enso.logger.JulHandler;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Value;
import org.hamcrest.core.AllOf;
import org.junit.AfterClass;
import org.junit.Rule;
import org.junit.Test;

/**
Expand All @@ -25,6 +28,8 @@ public class ContextInsightSetupTest {

public ContextInsightSetupTest() {}

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@AfterClass
public static void cleanupInsightProperty() {
System.getProperties().remove("enso.dev.insight");
Expand All @@ -45,7 +50,7 @@ public void initializeInsightViaProperty() throws Exception {
var out = new ByteArrayOutputStream();
// Need to initialize the Context via ContextFactory, so that ContextInsightSetup is
// triggered.
try (var ctx = ContextFactory.create().out(out).build()) {
try (var ctx = ContextFactory.create().logHandler(JulHandler.get()).out(out).build()) {

var fourtyTwo = evalModule(ctx, """
main = 42
Expand Down
4 changes: 4 additions & 0 deletions ...time-integration-tests/src/test/java/org/enso/compiler/dump/test/ApiModificationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,17 @@
import java.io.IOException;
import java.util.function.BiConsumer;
import org.enso.test.utils.ContextUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;

/** Tests recognitions of API changes in Enso code. */
public final class ApiModificationTest {
@ClassRule public static final ContextUtils ctxRule = ContextUtils.createDefault();

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void reorderingMethods_DoesNotModifyApi() throws IOException {
var prevSrc = """
Expand Down
3 changes: 3 additions & 0 deletions ...runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,16 @@
import org.enso.test.utils.ContextUtils;
import org.enso.test.utils.ProjectUtils;
import org.enso.test.utils.SourceModule;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

public class DocsGenerateTest {
@ClassRule public static final TemporaryFolder TEMP = new TemporaryFolder();
@ClassRule public static final ContextUtils ctxRule = ContextUtils.createDefault();
@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

public DocsGenerateTest() {}

Expand Down
2 changes: 2 additions & 0 deletions engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/IRDumpTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.enso.test.utils.ContextUtils;
import org.enso.test.utils.IRDumperTestWrapper;
import org.enso.test.utils.ProjectUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
Expand All @@ -34,6 +35,7 @@ public class IRDumpTest {
private static final Path irDumpsDir = Path.of(IRDumpFactoryService.DEFAULT_DUMP_DIR);
private ByteArrayOutputStream out;

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();
@Rule public PrintOutRule printOutRule = new PrintOutRule();

@BeforeClass
Expand Down
3 changes: 3 additions & 0 deletions ...ime-integration-tests/src/test/java/org/enso/compiler/test/BindingsMapResolutionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
import org.enso.test.utils.ContextUtils;
import org.enso.test.utils.ProjectUtils;
import org.enso.test.utils.SourceModule;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import scala.jdk.CollectionConverters;
Expand All @@ -33,6 +35,7 @@
public class BindingsMapResolutionTest {

@ClassRule public static final TemporaryFolder TMP_DIR = new TemporaryFolder();
@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void resolveSingleName_FromSingleImport() throws IOException {
Expand Down
8 changes: 6 additions & 2 deletions engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/ExecCompilerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.enso.compiler.test;

import static org.enso.compiler.test.ExecStrictCompilerTest.ctxRule;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
Expand All @@ -15,12 +14,14 @@
import org.enso.common.RuntimeOptions;
import org.enso.compiler.core.ir.expression.errors.Conversion.DeclaredAsPrivate$;
import org.enso.test.utils.ContextUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.PolyglotException;
import org.graalvm.polyglot.Source;
import org.hamcrest.core.AllOf;
import org.junit.After;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;

public class ExecCompilerTest {
Expand All @@ -30,6 +31,8 @@ public class ExecCompilerTest {
.withModifiedContext(ctxBldr -> ctxBldr.option(RuntimeOptions.STRICT_ERRORS, "false"))
.build();

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@After
public void cleanup() {
ctxRule.resetOut();
Expand Down Expand Up @@ -528,7 +531,8 @@ public void illegalPrivateConversion() {
var expectedErrMsg = DeclaredAsPrivate$.MODULE$.explain();
var runMethod = module.invokeMember(Module.EVAL_EXPRESSION, "run");
runMethod.execute(0);
assertThat(ctxRule.getOut(), containsString(expectedErrMsg));
assertThat(
String.join("\n", appenderRule.pendingLogMessages()), containsString(expectedErrMsg));
}

@Test
Expand Down
4 changes: 4 additions & 0 deletions ...untime-integration-tests/src/test/java/org/enso/compiler/test/ExecStrictCompilerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@
import org.enso.common.MethodNames;
import org.enso.common.RuntimeOptions;
import org.enso.test.utils.ContextUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.PolyglotException;
import org.graalvm.polyglot.Source;
import org.hamcrest.core.AllOf;
import org.junit.After;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;

public class ExecStrictCompilerTest {
Expand All @@ -28,6 +30,8 @@ public class ExecStrictCompilerTest {
.withModifiedContext(ctxBldr -> ctxBldr.option(RuntimeOptions.STRICT_ERRORS, "true"))
.build();

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@After
public void cleanMessages() {
ctxRule.resetOut();
Expand Down
34 changes: 12 additions & 22 deletions ...time-integration-tests/src/test/java/org/enso/compiler/test/ExportCycleDetectionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.enso.test.utils.ModuleUtils;
import org.enso.test.utils.ProjectUtils;
import org.enso.test.utils.SourceModule;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.PolyglotException;
import org.hamcrest.Matcher;
import org.junit.Rule;
Expand All @@ -24,6 +25,7 @@

public class ExportCycleDetectionTest {
@Rule public TemporaryFolder tempFolder = new TemporaryFolder();
@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void detectCycleInTwoModules() throws IOException {
Expand Down Expand Up @@ -103,38 +105,26 @@ public void detectCycleInThreeModules() throws IOException {

@Test
public void noCycleDetectedWhenExportingSymbolsFromItself_1() throws IOException {
var mainMod =
new SourceModule(
QualifiedName.fromString("Main"),
"""
expectNoCompilationErrors(
tempFolder.newFolder().toPath(),
"""
type Main_Type
export project.Main.Main_Type
""");
var projDir = tempFolder.newFolder().toPath();
ProjectUtils.createProject("Proj", Set.of(mainMod), projDir);
try (var ctx = ContextUtils.newBuilder().withProjectRoot(projDir).build()) {
var polyCtx = new PolyglotContext(ctx.context());
try {
polyCtx.getTopScope().compile(true);
} catch (PolyglotException e) {
fail("Compilation error not expected. But got: " + e);
}
var exportedSyms = ModuleUtils.getExportedSymbolsFromModule(ctx, "local.Proj.Main");
assertThat(exportedSyms.size(), is(1));
assertThat(exportedSyms, hasKey("Main_Type"));
}
}

@Test
public void noCycleDetectedWhenExportingSymbolsFromItself_2() throws IOException {
var mainMod =
new SourceModule(
QualifiedName.fromString("Main"),
"""
expectNoCompilationErrors(
tempFolder.newFolder().toPath(),
"""
type Main_Type
from project.Main export Main_Type
""");
var projDir = tempFolder.newFolder().toPath();
}

private void expectNoCompilationErrors(Path projDir, String code) throws IOException {
var mainMod = new SourceModule(QualifiedName.fromString("Main"), code);
ProjectUtils.createProject("Proj", Set.of(mainMod), projDir);
try (var ctx = ContextUtils.newBuilder().withProjectRoot(projDir).build()) {
var polyCtx = new PolyglotContext(ctx.context());
Expand Down
4 changes: 4 additions & 0 deletions ...e/runtime-integration-tests/src/test/java/org/enso/compiler/test/ExportedSymbolsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,18 @@
import org.enso.test.utils.ContextUtils;
import org.enso.test.utils.ProjectUtils;
import org.enso.test.utils.SourceModule;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import scala.jdk.javaapi.CollectionConverters;

public class ExportedSymbolsTest {
private Path projDir;

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Before
public void setup() throws IOException {
this.projDir = Files.createTempDirectory("exported-symbols-test");
Expand Down
3 changes: 3 additions & 0 deletions engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/IfThenElseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,19 @@
import com.oracle.truffle.api.library.ExportLibrary;
import com.oracle.truffle.api.library.ExportMessage;
import org.enso.test.utils.ContextUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.PolyglotException;
import org.graalvm.polyglot.Value;
import org.hamcrest.Matchers;
import org.hamcrest.core.AllOf;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;

public class IfThenElseTest {
@ClassRule public static final ContextUtils ctxRule = ContextUtils.createDefault();
@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void simpleIfThenElse() {
Expand Down
2 changes: 2 additions & 0 deletions ...-integration-tests/src/test/java/org/enso/compiler/test/ImportsAndFQNConsistencyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.enso.compiler.data.BindingsMap.ResolvedType;
import org.enso.interpreter.runtime.EnsoContext;
import org.enso.test.utils.ContextUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
Expand Down Expand Up @@ -47,6 +48,7 @@ public class ImportsAndFQNConsistencyTest {
.withModifiedContext(ctxBldr -> ctxBldr.option(RuntimeOptions.DISABLE_IR_CACHES, "false"))
.build();

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();
@Rule public final TestRule printCodeRule = new PrintCodeRule();

/**
Expand Down
9 changes: 7 additions & 2 deletions engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@
import org.enso.common.MethodNames;
import org.enso.common.RuntimeOptions;
import org.enso.interpreter.runtime.EnsoContext;
import org.enso.logger.JulHandler;
import org.enso.pkg.PackageManager;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.io.IOAccess;
import org.junit.Rule;
import org.junit.Test;

public class SerializerTest {
Expand All @@ -28,14 +31,16 @@ public Context ensoContextForPackage(String name, File pkgFile) throws IOExcepti
.option(
RuntimeOptions.LANGUAGE_HOME_OVERRIDE,
Paths.get("../../distribution/component").toFile().getAbsolutePath())
.option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName())
.logHandler(System.err)
.option(RuntimeOptions.LOG_LEVEL, Level.FINE.getName())
.logHandler(JulHandler.get())
.allowAllAccess(true)
.build();
assertNotNull("Enso language is supported", ctx.getEngine().getLanguages().get("enso"));
return ctx;
}

@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void testSerializationOfFQNs() throws Exception {
var testName = "Test_Serializer_FQN";
Expand Down
3 changes: 1 addition & 2 deletions ...ne/runtime-integration-tests/src/test/java/org/enso/compiler/test/StaticAnalysisTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ public abstract class StaticAnalysisTest {
(builder) ->
builder
.option(RuntimeOptions.ENABLE_STATIC_ANALYSIS, "true")
.option(RuntimeOptions.LOG_LEVEL, Level.INFO.getName())
.option(RuntimeOptions.LOG_LEVEL, Level.SEVERE.getName())
.option(RuntimeOptions.LOG_LEVEL, Level.FINE.getName())
.out(OutputStream.nullOutputStream())
.err(OutputStream.nullOutputStream()));

Expand Down
4 changes: 4 additions & 0 deletions engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/TypeInferenceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,16 @@
import org.enso.test.utils.ContextUtils;
import org.enso.test.utils.ModuleUtils;
import org.enso.test.utils.ProjectUtils;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.Source;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import scala.Option;

public class TypeInferenceTest extends StaticAnalysisTest {
@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void zeroAryCheck() throws Exception {
final URI uri = new URI("memory://zeroAryModuleMethodCheck.enso");
Expand Down
3 changes: 3 additions & 0 deletions ...ntime-integration-tests/src/test/java/org/enso/compiler/test/TypesFromSignaturesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,14 @@
import org.enso.compiler.pass.analyse.types.InferredType;
import org.enso.compiler.pass.analyse.types.TypeInferenceSignatures;
import org.enso.compiler.pass.analyse.types.TypeRepresentation;
import org.enso.testkit.ReportLogsOnFailureRule;
import org.graalvm.polyglot.Source;
import org.junit.Rule;
import org.junit.Test;
import scala.Option;

public class TypesFromSignaturesTest extends StaticAnalysisTest {
@Rule public ReportLogsOnFailureRule appenderRule = new ReportLogsOnFailureRule();

@Test
public void simpleCheck() throws Exception {
Expand Down
Loading
Loading