fet(vdf): writing and reading VDF travel times (#284)

* feat(vdf): writing and reading VDF travel times

* chore: renamed writeTimeInterval attribute to writeTravelTimesInterval

* chore: making sure travel times remain the same outside updateExtent

core/src/main/java/org/eqasim/core/scenario/cutter/RunScenarioCutterV2.java

@@ -140,11 +140,11 @@ static public void main(String[] args)
         VDFConfigGroup vdfConfigGroup = VDFConfigGroup.getOrCreate(config);
         vdfConfigGroup.setUpdateAreaShapefile("extent/" + extentPath.getName());
         // We also set the VDF config to use the vdf.bin file for initial travel times
-        vdfConfigGroup.setInputFile("vdf.bin");
+        vdfConfigGroup.setInputTravelTimesFile("vdf_travel_times.bin");
 
         new ScenarioWriter(config, scenario, prefix).run(new File(outputPath).getAbsoluteFile());
 
-        FileUtils.copyFile(new File(cmd.getOptionStrict("vdf-travel-times-path")), new File(outputPath, "vdf.bin"));
+        FileUtils.copyFile(new File(cmd.getOptionStrict("vdf-travel-times-path")), new File(outputPath, "vdf_travel_times.bin"));
     }
 
     public static void findLargestFullyConnectedSubnetwork(Network network, String mode) {

core/src/main/java/org/eqasim/core/simulation/vdf/VDFConfigGroup.java

@@ -26,9 +26,11 @@ public class VDFConfigGroup extends ReflectiveConfigGroup {
 
 	static private final String HANDLER = "handler";
 	static private final String INPUT_FILE = "inputFile";
+	static private final String INPUT_TRAVEL_TIMES_FILE = "inputTravelTimesFile";
 	static private final String UPDATE_AREA_SHAPEFILE = "updateAreaShapefile";
 	static private final String WRITE_INTERVAL = "writeInterval";
 	static private final String WRITE_FLOW_INTERVAL = "writeFlowInterval";
+	static private final String WRITE_TRAVEL_TIMES_INTERVAL = "writeTravelTimesInterval";
 
 	private double startTime = 0.0 * 3600.0;
 	private double endTime = 24.0 * 3600.0;
@@ -43,10 +45,12 @@ public class VDFConfigGroup extends ReflectiveConfigGroup {
 
 	private double capacityFactor = 1.0;
 
-	private String inputFile = null;
+	private String inputFlowFile = null;
+	private String inputTravelTimesFile = null;
 	private String updateAreaShapefile = null;
 	private int writeInterval = 0;
 	private int writeFlowInterval = 0;
+	private int writeTravelTimesInterval = 0;
 
 	public enum HandlerType {
 		Horizon, Interpolation, SparseHorizon
@@ -179,7 +183,7 @@ public String getModesAsString() {
 	@StringSetter(MODES)
 	public void setModesAsString(String modes) {
 		this.modes.clear();
-		Arrays.asList(modes.split(",")).stream().map(String::trim).forEach(this.modes::add);
+		Arrays.stream(modes.split(",")).map(String::trim).forEach(this.modes::add);
 	}
 
 	@StringGetter(HANDLER)
@@ -193,13 +197,23 @@ public void setHandler(HandlerType handler) {
 	}
 
 	@StringGetter(INPUT_FILE)
-	public String getInputFile() {
-		return inputFile;
+	public String getInputFlowFile() {
+		return inputFlowFile;
 	}
 
 	@StringSetter(INPUT_FILE)
-	public void setInputFile(String inputFile) {
-		this.inputFile = inputFile;
+	public void setInputFlowFile(String inputFlowFile) {
+		this.inputFlowFile = inputFlowFile;
+	}
+
+	@StringGetter(INPUT_TRAVEL_TIMES_FILE)
+	public String getInputTravelTimesFile() {
+		return inputTravelTimesFile;
+	}
+
+	@StringSetter(INPUT_TRAVEL_TIMES_FILE)
+	public void setInputTravelTimesFile(String inputTravelTimesFile) {
+		this.inputTravelTimesFile = inputTravelTimesFile;
 	}
 
 	@StringGetter(UPDATE_AREA_SHAPEFILE)
@@ -232,6 +246,16 @@ public void setWriteFlowInterval(int val) {
 		this.writeFlowInterval = val;
 	}
 
+	@StringGetter(WRITE_TRAVEL_TIMES_INTERVAL)
+	public int getWriteTravelTimesInterval() {
+		return writeTravelTimesInterval;
+	}
+
+	@StringSetter(WRITE_TRAVEL_TIMES_INTERVAL)
+	public void setWriteTravelTimesInterval(int val) {
+		this.writeTravelTimesInterval = val;
+	}
+
 	public static VDFConfigGroup getOrCreate(Config config) {
 		VDFConfigGroup group = (VDFConfigGroup) config.getModules().get(GROUP_NAME);
 

core/src/main/java/org/eqasim/core/simulation/vdf/VDFModule.java

@@ -5,6 +5,8 @@
 import java.net.URL;
 import java.util.Optional;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.eqasim.core.components.config.EqasimConfigGroup;
 import org.eqasim.core.scenario.cutter.extent.ScenarioExtent;
 import org.eqasim.core.scenario.cutter.extent.ShapeScenarioExtent;
@@ -26,6 +28,9 @@
 import com.google.inject.Singleton;
 
 public class VDFModule extends AbstractEqasimExtension {
+
+	private static final Logger LOGGER = LogManager.getLogger(VDFModule.class);
+
 	@Override
 	protected void installEqasimExtension() {
 		VDFConfigGroup vdfConfig = VDFConfigGroup.getOrCreate(getConfig());
@@ -60,10 +65,10 @@ protected void installEqasimExtension() {
 	public VDFUpdateListener provideVDFUpdateListener(VDFScope scope, VDFTrafficHandler handler,
 			VDFTravelTime travelTime, VDFConfigGroup config, OutputDirectoryHierarchy outputHierarchy, Network network,
 			ControllerConfigGroup controllerConfig) {
-		URL inputFile = config.getInputFile() == null ? null
-				: ConfigGroup.getInputFileURL(getConfig().getContext(), config.getInputFile());
+		URL inputFile = config.getInputFlowFile() == null ? null
+				: ConfigGroup.getInputFileURL(getConfig().getContext(), config.getInputFlowFile());
 		return new VDFUpdateListener(network, scope, handler, travelTime, outputHierarchy, config.getWriteInterval(),
-				config.getWriteFlowInterval(), controllerConfig.getFirstIteration(), inputFile);
+				config.getWriteFlowInterval(), config.getWriteTravelTimesInterval(), controllerConfig.getFirstIteration(), inputFile);
 	}
 
 	@Provides
@@ -80,8 +85,15 @@ public VDFTravelTime provideVDFTravelTime(VDFConfigGroup config, VDFScope scope,
 				: new ShapeScenarioExtent.Builder(new File(ConfigGroup
 						.getInputFileURL(getConfig().getContext(), config.getUpdateAreaShapefile()).getPath()),
 						Optional.empty(), Optional.empty()).build();
-		return new VDFTravelTime(scope, config.getMinimumSpeed(), config.getCapacityFactor(),
+		VDFTravelTime vdfTravelTime = new VDFTravelTime(scope, config.getMinimumSpeed(), config.getCapacityFactor(),
 				eqasimConfig.getSampleSize(), network, vdf, eqasimConfig.getCrossingPenalty(), updateExtent);
+		if(config.getInputTravelTimesFile() != null) {
+			LOGGER.info("Reading VDF travel times");
+			URL inputTimeFile = ConfigGroup.getInputFileURL(getConfig().getContext(), config.getInputTravelTimesFile());
+			vdfTravelTime.readFrom(inputTimeFile);
+			LOGGER.info("  Done");
+		}
+		return vdfTravelTime;
 	}
 
 	@Provides

core/src/main/java/org/eqasim/core/simulation/vdf/VDFUpdateListener.java

@@ -28,13 +28,15 @@ public class VDFUpdateListener implements IterationEndsListener, StartupListener
 
 	private final String VDF_FILE = "vdf.bin";
 	private final String FLOW_FILE = "vdf_flow.csv";
+	private final String TRAVEL_TIMES_FILE = "vdf_travel_times.bin";
 
 	private final VDFScope scope;
 	private final VDFTrafficHandler handler;
 	private final VDFTravelTime travelTime;
 
 	private final int writeInterval;
 	private final int writeFlowInterval;
+	private final int writeTravelTimesInterval;
 	private final URL inputFile;
 
 	private final int firstIteration;
@@ -43,8 +45,8 @@ public class VDFUpdateListener implements IterationEndsListener, StartupListener
 	private final Network network;
 
 	public VDFUpdateListener(Network network, VDFScope scope, VDFTrafficHandler handler, VDFTravelTime travelTime,
-			OutputDirectoryHierarchy outputHierarchy, int writeInterval, int writeFlowInterval, int firstIteration,
-			URL inputFile) {
+							 OutputDirectoryHierarchy outputHierarchy, int writeInterval, int writeFlowInterval, int writeTravelTimesInterval, int firstIteration,
+							 URL inputFile) {
 		this.network = network;
 		this.scope = scope;
 		this.handler = handler;
@@ -54,6 +56,7 @@ public VDFUpdateListener(Network network, VDFScope scope, VDFTrafficHandler hand
 		this.outputHierarchy = outputHierarchy;
 		this.inputFile = inputFile;
 		this.firstIteration = firstIteration;
+		this.writeTravelTimesInterval = writeTravelTimesInterval;
 	}
 
 	@Override
@@ -83,6 +86,13 @@ public void notifyIterationEnds(IterationEndsEvent event) {
 			new FlowWriter(data, network, scope).write(flowFile);
 			logger.info("  Done");
 		}
+
+		if (writeTravelTimesInterval > 0 && (event.getIteration() % writeTravelTimesInterval == 0 || event.isLastIteration())) {
+			File travelTimesFile = new File(outputHierarchy.getIterationFilename(event.getIteration(), TRAVEL_TIMES_FILE));
+			logger.info("Writing travel time information to " + travelTimesFile + "...");
+			travelTime.write(travelTimesFile);
+			logger.info("  Done");
+		}
 	}
 
 	@Override
@@ -96,7 +106,7 @@ public void notifyStartup(StartupEvent event) {
 			// aggregate to have consistent travel times
 			IdMap<Link, List<Double>> data = handler.aggregate(true);
 			scope.verify(data, "Wrong flow format");
-			travelTime.update(data, true);
+			travelTime.update(data);
 
 			logger.info("  Done");
 		}
@@ -118,6 +128,13 @@ public void notifyShutdown(ShutdownEvent event) {
 			if (fromFlowFile.exists()) {
 				Files.copy(fromFlowFile, toFlowFile);
 			}
+
+			File fromTravelTimesFile = new File(outputHierarchy.getIterationFilename(event.getIteration(), TRAVEL_TIMES_FILE));
+			File toTravelTimesFile = new File(outputHierarchy.getOutputFilename(TRAVEL_TIMES_FILE));
+
+			if(fromTravelTimesFile.exists()) {
+				Files.copy(fromTravelTimesFile, toTravelTimesFile);
+			}
 		} catch (IOException e) {
 			throw new RuntimeException(e);
 		}

core/src/main/java/org/eqasim/core/simulation/vdf/travel_time/VDFTravelTime.java

@@ -1,10 +1,16 @@
 package org.eqasim.core.simulation.vdf.travel_time;
 
+import java.io.DataOutputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import com.google.common.base.Verify;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.eqasim.core.scenario.cutter.extent.ScenarioExtent;
@@ -16,6 +22,7 @@
 import org.matsim.api.core.v01.network.Network;
 import org.matsim.api.core.v01.population.Person;
 import org.matsim.core.router.util.TravelTime;
+import org.matsim.core.utils.io.IOUtils;
 import org.matsim.vehicles.Vehicle;
 
 public class VDFTravelTime implements TravelTime {
@@ -131,4 +138,41 @@ private double considerCrossingPenalty(Link link, double baseTravelTime) {
 			return baseTravelTime + crossingPenalty;
 		}
 	}
+
+	public void write(File outputFile) {
+        try {
+            DataOutputStream outputStream = new DataOutputStream(IOUtils.getOutputStream(outputFile.toURI().toURL(), false));
+			outputStream.writeInt(travelTimes.size());
+			outputStream.writeInt(scope.getIntervals());
+			for(Map.Entry<Id<Link>, List<Double>> entry : travelTimes.entrySet()) {
+				outputStream.writeUTF(entry.getKey().toString());
+				for(Double d : entry.getValue()) {
+					outputStream.writeDouble(d);
+				}
+			}
+			outputStream.flush();
+			outputStream.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+	public void readFrom(URL inputFile) {
+        try {
+            DataInputStream dataInputStream = new DataInputStream(IOUtils.getInputStream(inputFile));
+			Verify.verify(dataInputStream.readInt() == travelTimes.size());
+			Verify.verify(dataInputStream.readInt() == scope.getIntervals());
+			for(int i=0; i<travelTimes.size(); i++) {
+				Id<Link> linkId = Id.createLinkId(dataInputStream.readUTF());
+				for(int j=0; j<scope.getIntervals(); j++) {
+					double travelTime = dataInputStream.readDouble();
+					travelTimes.get(linkId).set(j, travelTime);
+				}
+			}
+			Verify.verify(dataInputStream.available() == 0);
+			dataInputStream.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

core/src/main/java/org/eqasim/core/simulation/vdf/utils/AdaptConfigForVDF.java

@@ -21,8 +21,9 @@ public static void adaptConfigForVDF(Config config, boolean engine) {
         config.qsim().setStorageCapFactor(1e9);
 
 
-        VDFConfigGroup.getOrCreate(config).setWriteInterval(1);
-        VDFConfigGroup.getOrCreate(config).setWriteFlowInterval(1);
+        VDFConfigGroup.getOrCreate(config).setWriteInterval(60);
+        VDFConfigGroup.getOrCreate(config).setWriteFlowInterval(60);
+        VDFConfigGroup.getOrCreate(config).setWriteTravelTimesInterval(60);
 
         VDFConfigGroup vdfConfigGroup = VDFConfigGroup.getOrCreate(config);
         vdfConfigGroup.setHandler(VDFConfigGroup.HandlerType.SparseHorizon);

core/src/main/java/org/eqasim/core/standalone_mode_choice/RunStandaloneModeChoice.java

@@ -1,7 +1,13 @@
 package org.eqasim.core.standalone_mode_choice;
 
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.File;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.nio.file.Paths;
@@ -68,7 +74,7 @@
  * - travel-times-factors-path: if provided, should point out to a csv file specifying the congestion levels on the network during the day as factors by which the free speed is divided. The file in question is a csv With a header timeUpperBound;travelTimeFactor in which the timeUpperBound should be ordered incrementally.
  * - recorded-travel-times-path: mutually exclusive with the travel-times-factors-path. Points to a RecordedTravelTime file.
  * - eqasim-configurator-class: The full name of a class extending the {@link org.eqasim.core.simulation.EqasimConfigurator} class, the provided configurator class will be instantiated and used to:
- *   - Detect optional config groups using the {@link org.eqasim.core.simulation.EqasimConfigurator#registerConfigGroup(ConfigGroup, boolean)} (Config)} method
+ *   - Detect optional config groups using the {@link org.eqasim.core.simulation.EqasimConfigurator#updateConfig(Config)} method
  *   - Configure the scenario using the {@link org.eqasim.core.simulation.EqasimConfigurator#configureScenario(Scenario)} before loading
  *   - Adjust the scenario using the {@link org.eqasim.core.simulation.EqasimConfigurator#adjustScenario(Scenario)} after loading
  * - mode-choice-configurator-class: The full name of a class the extending the {@link org.eqasim.core.standalone_mode_choice.StandaloneModeChoiceConfigurator} class.
@@ -226,7 +232,7 @@ RecordedTravelTime provideRecordedTravelTime() {
         }));
 
         boolean usingVdfTravelTime = false;
-        if(config.getModules().containsKey(VDFConfigGroup.GROUP_NAME) && VDFConfigGroup.getOrCreate(config).getInputFile() != null) {
+        if(config.getModules().containsKey(VDFConfigGroup.GROUP_NAME) && (VDFConfigGroup.getOrCreate(config).getInputFlowFile() != null || VDFConfigGroup.getOrCreate(config).getInputTravelTimesFile() != null)) {
             String usedTravelTimeArg = recordedTravelTimesPath.isPresent() ? CMD_RECORDED_TRAVEL_TIMES_PATH : travelTimesFactorsPath.isPresent() ? CMD_TRAVEL_TIMES_FACTORS_PATH : null;
             if(usedTravelTimeArg == null) {
                 usingVdfTravelTime = true;