Merge pull request #120 from moia-oss/master

Update MATSim CW48

Author: mfrawley-moia

contribs/application/pom.xml

@@ -87,7 +87,7 @@
 		<dependency>
 			<groupId>com.github.matsim-org</groupId>
 			<artifactId>gtfs2matsim</artifactId>
-			<version>0bd5850fd6</version>
+			<version>47b0802a29</version>
 			<exclusions>
 				<!-- Exclude unneeded dependencies and these with known CVE -->
 				<exclusion>

contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java

@@ -66,7 +66,7 @@ public class NoiseAnalysis implements MATSimAppCommand {
 
 	@CommandLine.Option(names = "--consider-activities", split = ",", description = "Considered activities for noise calculation." +
 		" Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "home*,work*,educ*,leisure*")
-	private Set<String> considerActivities;
+	private Set<String> consideredActivities;
 
 	@CommandLine.Option(names = "--noise-barrier", description = "Path to the noise barrier File", defaultValue = "")
 	private String noiseBarrierFile;
@@ -86,19 +86,20 @@ public Integer call() throws Exception {
 		boolean overrideParameters = ! ConfigUtils.hasModule(config, NoiseConfigGroup.class);
 		NoiseConfigGroup noiseParameters = ConfigUtils.addOrGetModule(config, NoiseConfigGroup.class);
 
-		if(overrideParameters){
+		if (overrideParameters){
 			log.warn("no NoiseConfigGroup was configured before. Will set some standards. You should check the next lines in the log file and the output_config.xml!");
-			noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new));
-			noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new));
+			noiseParameters.setConsideredActivitiesForReceiverPointGridArray(consideredActivities.toArray(String[]::new));
+			noiseParameters.setConsideredActivitiesForDamageCalculationArray(consideredActivities.toArray(String[]::new));
 
 			{
-				Set<String> set = CollectionUtils.stringArrayToSet( new String[]{TransportMode.bike, TransportMode.walk, TransportMode.transit_walk, TransportMode.non_network_walk} );
-				noiseParameters.setNetworkModesToIgnoreSet( set );
-			}
-			{
-				String[] set = new String[]{"freight"};
-				noiseParameters.setHgvIdPrefixesArray( set );
+				//the default settings are now actually the same as what we 'override' here, but let's leave it here for clarity.
+				Set<String> ignoredNetworkModes = CollectionUtils.stringArrayToSet( new String[]{TransportMode.bike, TransportMode.walk, TransportMode.transit_walk, TransportMode.non_network_walk} );
+				noiseParameters.setNetworkModesToIgnoreSet( ignoredNetworkModes );
+
+				String[] hgvIdPrefixes = {"lkw", "truck", "freight"};
+				noiseParameters.setHgvIdPrefixesArray( hgvIdPrefixes );
 			}
+
 			//use actual speed and not freespeed
 			noiseParameters.setUseActualSpeedLevel(true);
 			//use the valid speed range (recommended by IK)

contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java

@@ -4,10 +4,7 @@
 import it.unimi.dsi.fastutil.ints.IntList;
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 import it.unimi.dsi.fastutil.ints.IntSet;
-import it.unimi.dsi.fastutil.objects.Object2IntLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.objects.Object2IntMap;
-import it.unimi.dsi.fastutil.objects.Object2LongMap;
-import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
+import it.unimi.dsi.fastutil.objects.*;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.math3.analysis.interpolation.LoessInterpolator;
@@ -27,6 +24,7 @@
 import org.matsim.core.utils.io.IOUtils;
 import picocli.CommandLine;
 import tech.tablesaw.api.*;
+import tech.tablesaw.columns.strings.AbstractStringColumn;
 import tech.tablesaw.io.csv.CsvReadOptions;
 import tech.tablesaw.joining.DataFrameJoiner;
 import tech.tablesaw.selection.Selection;
@@ -37,6 +35,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static tech.tablesaw.aggregate.AggregateFunctions.count;
@@ -46,8 +45,9 @@
 	requires = {"trips.csv", "persons.csv"},
 	produces = {
 		"mode_share.csv", "mode_share_per_dist.csv", "mode_users.csv", "trip_stats.csv",
-		"mode_share_per_%s.csv", "population_trip_stats.csv", "trip_purposes_by_hour.csv",
-		"mode_share_distance_distribution.csv", "mode_shift.csv",
+		"mode_share_per_purpose.csv", "mode_share_per_%s.csv",
+		"population_trip_stats.csv", "trip_purposes_by_hour.csv",
+		"mode_share_distance_distribution.csv", "mode_shift.csv", "mode_chains.csv",
 		"mode_choices.csv", "mode_choice_evaluation.csv", "mode_choice_evaluation_per_mode.csv",
 		"mode_confusion_matrix.csv", "mode_prediction_error.csv"
 	}
@@ -283,10 +283,15 @@ public Integer call() throws Exception {
 
 		joined.addColumns(dist_group);
 
+		TextColumn purpose = joined.textColumn("end_activity_type");
+
+		// Remove suffix durations like _345
+		purpose.set(Selection.withRange(0, purpose.size()), purpose.replaceAll("_[0-9]{2,}$", ""));
+
 		writeModeShare(joined, labels);
 
 		if (groups != null) {
-			groups.analyzeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g));
+			groups.writeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g));
 		}
 
 		if (persons.containsColumn(ATTR_REF_MODES)) {
@@ -305,17 +310,24 @@ public Integer call() throws Exception {
 
 		writePopulationStats(persons, joined);
 
-		writeTripStats(joined);
-
-		writeTripPurposes(joined);
-
-		writeTripDistribution(joined);
-
-		writeModeShift(joined);
+		tryRun(this::writeTripStats, joined);
+		tryRun(this::writeTripPurposes, joined);
+		tryRun(this::writeTripDistribution, joined);
+		tryRun(this::writeModeShift, joined);
+		tryRun(this::writeModeChains, joined);
+		tryRun(this::writeModeStatsPerPurpose, joined);
 
 		return 0;
 	}
 
+	private void tryRun(ThrowingConsumer<Table> f, Table df) {
+		try {
+			f.accept(df);
+		} catch (IOException e) {
+			log.error("Error while running method", e);
+		}
+	}
+
 	private void writeModeShare(Table trips, List<String> labels) {
 
 		Table aggr = trips.summarize("trip_id", count).by("dist_group", "main_mode");
@@ -502,11 +514,6 @@ private void writeTripPurposes(Table trips) {
 			IntColumn.create("arrival_h", arrival.intStream().toArray())
 		);
 
-		TextColumn purpose = trips.textColumn("end_activity_type");
-
-		// Remove suffix durations like _345
-		purpose.set(Selection.withRange(0, purpose.size()), purpose.replaceAll("_[0-9]{2,}$", ""));
-
 		Table tArrival = trips.summarize("trip_id", count).by("end_activity_type", "arrival_h");
 
 		tArrival.column(0).setName("purpose");
@@ -610,6 +617,89 @@ private void writeModeShift(Table trips) throws IOException {
 		aggr.write().csv(output.getPath("mode_shift.csv").toFile());
 	}
 
+	/**
+	 * Collects information about all modes used during one day.
+	 */
+	private void writeModeChains(Table trips) throws IOException {
+
+		Map<String, List<String>> modesPerPerson = new LinkedHashMap<>();
+
+		for (Row trip : trips) {
+			String id = trip.getString("person");
+			String mode = trip.getString("main_mode");
+			modesPerPerson.computeIfAbsent(id, s -> new LinkedList<>()).add(mode);
+		}
+
+		// Store other values explicitly
+		ObjectDoubleMutablePair<String> other = ObjectDoubleMutablePair.of("other", 0);
+		Object2DoubleMap<String> chains = new Object2DoubleOpenHashMap<>();
+		for (List<String> modes : modesPerPerson.values()) {
+			String key;
+			if (modes.size() == 1)
+				key = modes.getFirst();
+			else if (modes.size() > 6) {
+				other.right(other.rightDouble() + 1);
+				continue;
+			} else
+				key = String.join("-", modes);
+
+			chains.mergeDouble(key, 1, Double::sum);
+		}
+
+
+		List<ObjectDoubleMutablePair<String>> counts = chains.object2DoubleEntrySet().stream()
+			.map(e -> ObjectDoubleMutablePair.of(e.getKey(), (int) e.getDoubleValue()))
+			.sorted(Comparator.comparingDouble(p -> -p.rightDouble()))
+			.collect(Collectors.toList());
+
+		// Aggregate entries to prevent file from getting too large
+		for (int i = 250; i < counts.size(); i++) {
+			other.right(other.rightDouble() + counts.get(i).rightDouble());
+		}
+		counts = counts.subList(0, Math.min(counts.size(), 250));
+		counts.add(other);
+
+		counts.sort(Comparator.comparingDouble(p -> -p.rightDouble()));
+
+
+		try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mode_chains.csv")), CSVFormat.DEFAULT)) {
+
+			printer.printRecord("modes", "count", "share");
+
+			double total = counts.stream().mapToDouble(ObjectDoubleMutablePair::rightDouble).sum();
+			for (ObjectDoubleMutablePair<String> p : counts) {
+				printer.printRecord(p.left(), (int) p.rightDouble(), p.rightDouble() / total);
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	private void writeModeStatsPerPurpose(Table trips) {
+
+		Table aggr = trips.summarize("trip_id", count).by("end_activity_type", "main_mode");
+
+		Comparator<Row> cmp = Comparator.comparing(row -> row.getString("end_activity_type"));
+		aggr = aggr.sortOn(cmp.thenComparing(row -> row.getString("main_mode")));
+
+		aggr.doubleColumn(aggr.columnCount() - 1).setName("share");
+		aggr.column("end_activity_type").setName("purpose");
+
+		Set<String> purposes = (Set<String>) aggr.column("purpose").asSet();
+
+		// Norm each purpose to 1
+		// It was not clear if the purpose is a string or text colum, therefor this code uses the abstract version
+		for (String label : purposes) {
+			DoubleColumn all = aggr.doubleColumn("share");
+			Selection sel = ((AbstractStringColumn<?>) aggr.column("purpose")).isEqualTo(label);
+
+			double total = all.where(sel).sum();
+			if (total > 0)
+				all.set(sel, all.divide(total));
+		}
+
+		aggr.write().csv(output.getPath("mode_share_per_purpose.csv").toFile());
+	}
+
 	/**
 	 * How shape file filtering should be applied.
 	 */
@@ -619,4 +709,9 @@ enum LocationFilter {
 		home,
 		none
 	}
+
+	@FunctionalInterface
+	private interface ThrowingConsumer<T> {
+		void accept(T t) throws IOException;
+	}
 }

contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java

@@ -100,7 +100,7 @@ final class TripByGroupAnalysis {
 		}
 	}
 
-	void analyzeModeShare(Table trips, List<String> dists, List<String> modeOrder, Function<String, Path> output) {
+	void writeModeShare(Table trips, List<String> dists, List<String> modeOrder, Function<String, Path> output) {
 
 		for (Group group : groups) {
 

contribs/application/src/main/java/org/matsim/application/analysis/pt/PublicTransitAnalysis.java

@@ -0,0 +1,105 @@
+/* *********************************************************************** *
+ * project: org.matsim.*
+ *                                                                         *
+ * *********************************************************************** *
+ *                                                                         *
+ * copyright       : (C) 2024 by the members listed in the COPYING,        *
+ *                   LICENSE and WARRANTY file.                            *
+ * email           : info at matsim dot org                                *
+ *                                                                         *
+ * *********************************************************************** *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *   See also COPYING, LICENSE and WARRANTY file                           *
+ *                                                                         *
+ * *********************************************************************** */
+
+package org.matsim.application.analysis.pt;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.matsim.analysis.pt.stop2stop.PtStop2StopAnalysis;
+import org.matsim.api.core.v01.Scenario;
+import org.matsim.application.ApplicationUtils;
+import org.matsim.application.CommandSpec;
+import org.matsim.application.MATSimAppCommand;
+import org.matsim.application.analysis.emissions.AirPollutionAnalysis;
+import org.matsim.application.options.InputOptions;
+import org.matsim.application.options.OutputOptions;
+import org.matsim.application.options.SampleOptions;
+import org.matsim.core.api.experimental.events.EventsManager;
+import org.matsim.core.config.Config;
+import org.matsim.core.config.ConfigUtils;
+import org.matsim.core.events.EventsUtils;
+import org.matsim.core.events.MatsimEventsReader;
+import org.matsim.core.scenario.ScenarioUtils;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+	name = "transit", description = "General public transit analysis.",
+	mixinStandardHelpOptions = true, showDefaultValues = true
+)
+@CommandSpec(requireRunDirectory = true,
+	produces = {
+		"pt_pax_volumes.csv.gz",
+	}
+)
+public class PublicTransitAnalysis implements MATSimAppCommand {
+
+	private static final Logger log = LogManager.getLogger(PublicTransitAnalysis.class);
+
+	@CommandLine.Mixin
+	private final InputOptions input = InputOptions.ofCommand(PublicTransitAnalysis.class);
+	@CommandLine.Mixin
+	private final OutputOptions output = OutputOptions.ofCommand(PublicTransitAnalysis.class);
+	@CommandLine.Mixin
+	private SampleOptions sample;
+
+	public static void main(String[] args) {
+		new PublicTransitAnalysis().execute(args);
+	}
+
+	@Override
+	public Integer call() throws Exception {
+
+		Config config = prepareConfig();
+		Scenario scenario = ScenarioUtils.loadScenario(config);
+		EventsManager eventsManager = EventsUtils.createEventsManager();
+
+		String eventsFile = ApplicationUtils.matchInput("events", input.getRunDirectory()).toString();
+
+		PtStop2StopAnalysis ptStop2StopEventHandler = new PtStop2StopAnalysis(scenario.getTransitVehicles(), sample.getUpscaleFactor());
+		eventsManager.addHandler(ptStop2StopEventHandler);
+		eventsManager.initProcessing();
+		MatsimEventsReader matsimEventsReader = new MatsimEventsReader(eventsManager);
+		matsimEventsReader.readFile(eventsFile);
+
+		log.info("Done reading the events file.");
+		log.info("Finish processing...");
+		eventsManager.finishProcessing();
+
+		ptStop2StopEventHandler.writeStop2StopEntriesByDepartureCsv(output.getPath("pt_pax_volumes.csv.gz"),
+			",", ";");
+
+		return 0;
+	}
+
+	private Config prepareConfig() {
+		Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString());
+
+		config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("vehicles", input.getRunDirectory()).toAbsolutePath().toString());
+		config.network().setInputFile(ApplicationUtils.matchInput("network", input.getRunDirectory()).toAbsolutePath().toString());
+		config.transit().setTransitScheduleFile(ApplicationUtils.matchInput("transitSchedule", input.getRunDirectory()).toAbsolutePath().toString());
+		config.transit().setVehiclesFile(ApplicationUtils.matchInput("transitVehicles", input.getRunDirectory()).toAbsolutePath().toString());
+		config.plans().setInputFile(null);
+		config.facilities().setInputFile(null);
+		config.eventsManager().setNumberOfThreads(null);
+		config.eventsManager().setEstimatedNumberOfEvents(null);
+		config.global().setNumberOfThreads(1);
+
+		return config;
+	}
+}

contribs/application/src/main/java/org/matsim/application/prepare/network/CreateAvroNetwork.java

@@ -108,8 +108,11 @@ public Integer call() throws Exception {
 			filter = filter.and(link -> p.matcher(link.getId().toString()).matches());
 		}
 
+		// Strings that could have been added in the list, due to command line parsing
+		modes.removeIf(m -> m.isBlank() || m.equals("none") || m.equals("\"") || m.equals("\"\""));
+
 		// At least one of the specified modes needs to be contained
-		if (!modes.isEmpty() && !modes.equals(Set.of("none"))) {
+		if (!modes.isEmpty()) {
 			filter = filter.and(link -> modes.stream().anyMatch(m -> link.getAllowedModes().contains(m)));
 		}
 

contribs/application/src/main/java/org/matsim/application/prepare/network/params/ApplyNetworkParams.java

@@ -257,7 +257,10 @@ private boolean applyFreeSpeed(Link link, Feature ft) {
 			return false;
 
 		link.setFreespeed(freeSpeed);
-		link.getAttributes().putAttribute("speed_factor", freeSpeed);
+		link.getAttributes().putAttribute(
+			"speed_factor",
+			BigDecimal.valueOf(freeSpeed).setScale(5, RoundingMode.HALF_EVEN).doubleValue()
+		);
 
 		return modified;
 	}