post process option to run commands after sim has finished (#1934)

Author: rakow

contribs/application/src/main/java/org/matsim/application/MATSimAppCommand.java

@@ -46,4 +46,18 @@ default void execute(String... args) {
 
 		}
 	}
+
+	/**
+	 * Apply the given command line arguments to this instance and return it.
+	 */
+	default MATSimAppCommand withArgs(String... args) {
+		CommandLine cli = new CommandLine(this);
+		CommandLine.ParseResult parseResult = cli.parseArgs(args);
+
+		if (!parseResult.errors().isEmpty())
+			throw new IllegalStateException("Error parsing arguments", parseResult.errors().get(0));
+
+		return cli.getCommand();
+	}
+
 }

contribs/application/src/main/java/org/matsim/application/MATSimApplication.java

@@ -35,7 +35,6 @@
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 
 /**
  * A helper class to define and execute MATSim scenarios, including a pipeline of prepare and analysis scripts.
@@ -101,6 +100,9 @@ public abstract class MATSimApplication implements Callable<Integer>, CommandLin
 	@CommandLine.Option(names = "--output", description = "Overwrite output folder defined by the application")
 	protected Path output;
 
+	@CommandLine.Option(names = "--post-processing", description = "Option for post-processing", defaultValue = "enabled")
+	protected PostProcessOption post;
+
 	/**
 	 * This Map will never contain anything, because this argument is not parsed correctly, but instead will go into remainingArgs field.
 	 *
@@ -191,7 +193,28 @@ public Integer call() throws Exception {
 
 		prepareControler(controler);
 
-		controler.run();
+		// Check if simulation needs to be run
+		if (post != PostProcessOption.post_process_only)
+			controler.run();
+
+		if (post != PostProcessOption.disabled) {
+
+			List<MATSimAppCommand> commands = preparePostProcessing(Path.of(config.controler().getOutputDirectory()), config.controler().getRunId());
+
+			for (MATSimAppCommand command : commands) {
+
+				try {
+					log.info("Running post-process command {}", command.getClass().getSimpleName());
+					command.call();
+				} catch (Exception e) {
+					log.error("Error running post-processing", e);
+				}
+
+			}
+
+		}
+
+
 		return 0;
 	}
 
@@ -318,10 +341,19 @@ protected void prepareScenario(Scenario scenario) {
 	protected void prepareControler(Controler controler) {
 	}
 
+	/**
+	 * Preparation of {@link MATSimAppCommand} to run after the simulation has finished. The instances have to be fully constructed in this method
+	 * no further arguments are passed down to them.
+	 * @return list of commands to run.
+	 */
+	protected List<MATSimAppCommand> preparePostProcessing(Path outputFolder, String runId) {
+		return List.of();
+	}
+
 	/**
 	 * Add and an option and value to run id and output folder.
 	 */
-	protected void addRunOption(Config config, String option, Object value) {
+	protected final void addRunOption(Config config, String option, Object value) {
 
 		String postfix;
 		if ("".equals(value))
@@ -342,7 +374,7 @@ protected void addRunOption(Config config, String option, Object value) {
 	/**
 	 * Add and an option to run id and output folder, delimited by "_".
 	 */
-	protected void addRunOption(Config config, String option) {
+	protected final void addRunOption(Config config, String option) {
 		addRunOption(config, option, "");
 	}
 
@@ -685,4 +717,22 @@ public Integer call() throws Exception {
 		Class<? extends MATSimAppCommand>[] value() default {};
 	}
 
+	/**
+	 * Option to switch post processing behavour
+	 */
+	public enum PostProcessOption {
+
+		enabled,
+
+		/**
+		 * Does not run the post-processing.
+		 */
+		disabled,
+
+		/**
+		 * Does not run the simulation, but only post-processing.
+		 */
+		post_process_only
+
+	}
 }

contribs/application/src/test/java/org/matsim/application/MATSimApplicationTest.java

@@ -1,30 +1,30 @@
 package org.matsim.application;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-
-import java.net.MalformedURLException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
 import org.junit.Assume;
 import org.junit.Rule;
 import org.junit.Test;
 import org.matsim.application.options.SampleOptions;
 import org.matsim.application.prepare.freight.tripExtraction.ExtractRelevantFreightTrips;
-import org.matsim.application.prepare.freight.toBeDeleted.GenerateGermanWideFreightTrips;
 import org.matsim.application.prepare.population.GenerateShortDistanceTrips;
 import org.matsim.application.prepare.population.MergePopulations;
 import org.matsim.application.prepare.population.TrajectoryToPlans;
 import org.matsim.core.config.Config;
 import org.matsim.core.config.ConfigUtils;
 import org.matsim.core.config.groups.PlanCalcScoreConfigGroup;
 import org.matsim.core.controler.Controler;
+import org.matsim.core.controler.OutputDirectoryHierarchy;
 import org.matsim.core.utils.io.IOUtils;
 import org.matsim.testcases.MatsimTestUtils;
-
 import picocli.CommandLine;
 
+import java.net.MalformedURLException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
 public class MATSimApplicationTest {
 
 	@Rule
@@ -69,8 +69,8 @@ public void yaml() {
 		assertThat(params.getOrCreateModeParams("car").getConstant())
 				.isEqualTo(-1);
 
-        assertThat(params.getOrCreateModeParams("bike").getConstant())
-                .isEqualTo(-2);
+		assertThat(params.getOrCreateModeParams("bike").getConstant())
+				.isEqualTo(-2);
 
 	}
 
@@ -158,9 +158,26 @@ public void freight() {
 
 	}
 
+	@Test
+	public void run() {
+
+		Config config = ConfigUtils.createConfig();
+		Path out = Path.of(utils.getOutputDirectory()).resolve("out");
+
+		config.controler().setOutputDirectory(out.toString());
+		config.controler().setOverwriteFileSetting(OutputDirectoryHierarchy.OverwriteFileSetting.deleteDirectoryIfExists);
+		config.controler().setLastIteration(1);
+
+		int ret = MATSimApplication.execute(TestScenario.class, config);
+
+		// Content defined in the post process section
+		assertThat(out.resolve("test.txt"))
+				.hasContent("Inhalt");
+
+	}
+
 	@MATSimApplication.Prepare({
-			TrajectoryToPlans.class, GenerateShortDistanceTrips.class,
-			GenerateGermanWideFreightTrips.class, ExtractRelevantFreightTrips.class, MergePopulations.class
+			TrajectoryToPlans.class, GenerateShortDistanceTrips.class, ExtractRelevantFreightTrips.class, MergePopulations.class
 	})
 	private static final class TestScenario extends MATSimApplication {
 
@@ -179,9 +196,14 @@ public TestScenario() {
 		protected Config prepareConfig(Config config) {
 
 			config.controler().setRunId(sample.adjustName("run-25pct"));
-
 			return config;
 		}
+
+
+		@Override
+		protected List<MATSimAppCommand> preparePostProcessing(Path outputFolder, String runId) {
+			return List.of(new TestCommand(outputFolder.resolve("test.txt"), "Inhalt"));
+		}
 	}
 
 }

contribs/application/src/test/java/org/matsim/application/TestCommand.java

@@ -0,0 +1,27 @@
+package org.matsim.application;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Test command that writes content to file.
+ */
+public class TestCommand implements MATSimAppCommand {
+
+	private final Path out;
+	private final String content;
+
+	public TestCommand(Path out, String content) {
+		this.out = out;
+		this.content = content;
+	}
+
+
+	@Override
+	public Integer call() throws Exception {
+
+		Files.writeString(out, content);
+
+		return 0;
+	}
+}