ODRE import postprocessor fixes

iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/LineGeoData.java

@@ -15,45 +15,7 @@
  * @author Geoffroy Jamgotchian {@literal <geoffroy.jamgotchian at rte-france.com>}
  * @author Chamseddine Benhamed {@literal <chamseddine.benhamed at rte-france.com>}
  */
-public class LineGeoData {
+public record LineGeoData(String id, String country1, String country2, String substationStart, String substationEnd,
+ List<Coordinate> coordinates) {
 
- private String id;
- private String country1;
- private String country2;
- private String substationStart;
- private String substationEnd;
- private List<Coordinate> coordinates;
-
- public LineGeoData(String id, String country1, String country2, String substationStart, String substationEnd, List<Coordinate> coordinates) {
- this.id = id;
- this.country1 = country1;
- this.country2 = country2;
- this.substationStart = substationStart;
- this.substationEnd = substationEnd;
- this.coordinates = coordinates;
- }
-
- public String getId() {
- return id;
- }
-
- public String getCountry1() {
- return country1;
- }
-
- public String getCountry2() {
- return country2;
- }
-
- public String getSubstationStart() {
- return substationStart;
- }
-
- public String getSubstationEnd() {
- return substationEnd;
- }
-
- public List<Coordinate> getCoordinates() {
- return coordinates;
- }
 }

iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/GeographicDataParser.java

@@ -7,7 +7,6 @@
  */
 package com.powsybl.iidm.geodata.odre;
 
-import com.google.common.collect.Lists;
 import com.powsybl.iidm.geodata.elements.GeoShape;
 import com.powsybl.iidm.geodata.elements.LineGeoData;
 import com.powsybl.iidm.geodata.elements.SubstationGeoData;
@@ -20,8 +19,9 @@
 import org.apache.commons.lang3.time.StopWatch;
 import org.apache.commons.lang3.tuple.Pair;
 import org.jgrapht.Graph;
-import org.jgrapht.Graphs;
-import org.jgrapht.alg.connectivity.ConnectivityInspector;
+import org.jgrapht.event.ConnectedComponentTraversalEvent;
+import org.jgrapht.event.TraversalListenerAdapter;
+import org.jgrapht.event.VertexTraversalEvent;
 import org.jgrapht.traverse.BreadthFirstIterator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -92,7 +92,7 @@ public static Pair<String, String> substationOrder(Map<String, SubstationGeoData
  } else {
  boolean isStart = distanceCoordinate((geo1 != null ? geo1 : geo2).getCoordinate(), coordinates.get(0)) < distanceCoordinate((geo1 != null ? geo1 : geo2).getCoordinate(), coordinates.get(coordinates.size() - 1));
  String substation = geo1 != null ? substation1 : substation2;
- return Pair.of(isStart ? substation : "", isStart ? "" : substation);
+ return isStart ? Pair.of(substation, "") : Pair.of("", substation);
  }
  }
 
@@ -101,10 +101,10 @@ public static Map<String, LineGeoData> parseLines(Reader aerialLinesReader, Read
  StopWatch stopWatch = new StopWatch();
  stopWatch.start();
 
- Map<String, Graph<Coordinate, Object>> graphByLine = new HashMap<>();
+ Map<String, List<List<Coordinate>>> coordinatesListsByLine = new HashMap<>();
 
- parseLine(graphByLine, aerialLinesReader, odreConfig);
- parseLine(graphByLine, undergroundLinesReader, odreConfig);
+ parseLine(coordinatesListsByLine, aerialLinesReader, odreConfig);
+ parseLine(coordinatesListsByLine, undergroundLinesReader, odreConfig);
 
  Map<String, LineGeoData> lines = new HashMap<>();
 
@@ -114,51 +114,77 @@ public static Map<String, LineGeoData> parseLines(Reader aerialLinesReader, Read
  int oneConnectedSetDiscarded = 0;
  int twoOrMoreConnectedSetsDiscarded = 0;
 
- for (Map.Entry<String, Graph<Coordinate, Object>> e : graphByLine.entrySet()) {
+ for (Map.Entry<String, List<List<Coordinate>>> e : coordinatesListsByLine.entrySet()) {
  String lineId = e.getKey();
- Graph<Coordinate, Object> graph = e.getValue();
- List<Set<Coordinate>> connectedSets = new ConnectivityInspector<>(graph).connectedSets();
- if (connectedSets.size() == 1) {
- linesWithOneConnectedSet++;
- List<Coordinate> ends = getEnds(connectedSets.get(0), graph);
- if (ends.size() == 2) {
- List<Coordinate> coordinates = Lists.newArrayList(new BreadthFirstIterator<>(graph, ends.get(0)));
- Pair<String, String> substations = substationOrder(stringSubstationGeoDataMap, lineId, coordinates);
- LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), coordinates);
- lines.put(lineId, line);
- } else {
- oneConnectedSetDiscarded++;
- }
+
+ List<List<Coordinate>> coordinatesLists = e.getValue();
+ if (coordinatesLists.size() == 1) {
+ List<Coordinate> coordinates = coordinatesLists.get(0);
+ Pair<String, String> substations = substationOrder(stringSubstationGeoDataMap, lineId, coordinates);
+ LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), coordinates);
+ lines.put(lineId, line);
  } else {
- linesWithTwoOrMoreConnectedSets++;
- List<List<Coordinate>> coordinatesComponents = fillMultipleConnectedSetsCoordinatesList(connectedSets,
- graph);
+ LineGraph<Coordinate, Object> graph = new LineGraph<>(Object.class);
+ coordinatesLists.forEach(graph::addVerticesAndEdges);
+ List<ConnectedSet> connectedSets = getConnectedSets(graph);
+ if (connectedSets.size() == 1) {
+ linesWithOneConnectedSet++;
+ ConnectedSet connectedSet = connectedSets.get(0);
+ if (connectedSet.ends().size() == 2) {
+ List<Coordinate> coordinates = connectedSet.list();
+ Pair<String, String> substations = substationOrder(stringSubstationGeoDataMap, lineId, coordinates);
+ LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), coordinates);
+ lines.put(lineId, line);
+ } else {
+ oneConnectedSetDiscarded++;
+ }
+ } else {
+ linesWithTwoOrMoreConnectedSets++;
+ List<List<Coordinate>> coordinatesComponents = fillMultipleConnectedSetsCoordinatesList(connectedSets, graph);
 
- if (coordinatesComponents.size() != connectedSets.size()) {
- twoOrMoreConnectedSetsDiscarded++;
- continue;
- }
+  if (coordinatesComponents.size() != connectedSets.size()) {
+  twoOrMoreConnectedSetsDiscarded++;
+  continue;
+  }
 
- List<Coordinate> aggregatedCoordinates = aggregateCoordinates(coordinatesComponents);
- Pair<String, String> substations = substationOrder(stringSubstationGeoDataMap, lineId, aggregatedCoordinates);
- LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), aggregatedCoordinates);
- lines.put(lineId, line);
+ List<Coordinate> aggregatedCoordinates = aggregateCoordinates(coordinatesComponents);
+ Pair<String, String> substations = substationOrder(stringSubstationGeoDataMap, lineId, aggregatedCoordinates);
+ LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), aggregatedCoordinates);
+ lines.put(lineId, line);
+ }
  }
  }
 
  LOGGER.info("{} lines read in {} ms", lines.size(), stopWatch.getTime());
  LOGGER.info("{} lines have one Connected set, {} of them were discarded", linesWithOneConnectedSet, oneConnectedSetDiscarded);
  LOGGER.info("{} lines have two or more Connected sets, {} of them were discarded", linesWithTwoOrMoreConnectedSets, twoOrMoreConnectedSetsDiscarded);
 
- if (graphByLine.size() != lines.size()) {
+ if (coordinatesListsByLine.size() != lines.size()) {
  LOGGER.warn("Total discarded lines : {}/{} ",
- graphByLine.size() - lines.size(), graphByLine.size());
+ coordinatesListsByLine.size() - lines.size(), coordinatesListsByLine.size());
  }
 
  return lines;
  }
 
- private static void parseLine(Map<String, Graph<Coordinate, Object>> graphByLine, Reader reader, OdreConfig odreConfig) {
+ private static List<ConnectedSet> getConnectedSets(Graph<Coordinate, Object> graph) {
+ List<ConnectedSet> connectedSets = new ArrayList<>();
+ Set<Coordinate> vertexSet = graph.vertexSet();
+
+ Optional<Coordinate> endCoord = vertexSet.stream().filter(v -> graph.degreeOf(v) == 1).findFirst();
+ if (endCoord.isPresent()) {
+ var bfi = new BreadthFirstIterator<Coordinate, Object>(graph, () -> Stream.concat(Stream.of(endCoord.get()), vertexSet.stream()).iterator());
+ bfi.addTraversalListener(new GeoTraversalListener(graph, connectedSets));
+ while (bfi.hasNext()) {
+ bfi.next();
+ }
+ } else {
+ connectedSets = List.of(new ConnectedSet(vertexSet.stream().toList(), List.of()));
+ }
+ return connectedSets;
+ }
+
+ private static void parseLine(Map<String, List<List<Coordinate>>> coordinateListsByLine, Reader reader, OdreConfig odreConfig) {
  try {
  Iterable<CSVRecord> records = CSVParser.parse(reader, FileValidator.CSV_FORMAT);
  Map<String, String> idsColumnNames = odreConfig.idsColumnNames();
@@ -167,67 +193,44 @@ private static void parseLine(Map<String, Graph<Coordinate, Object>> graphByLine
  row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_2)),
  row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_3)),
  row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_4)),
- row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_5))).filter(Objects::nonNull).toList();
+ row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_5))).filter(Objects::nonNull).filter(s -> !s.isEmpty()).distinct().toList();
  GeoShape geoShape = GeoShapeDeserializer.read(row.get(odreConfig.geoShapeColumn()));
 
  if (ids.isEmpty() || geoShape.coordinates().isEmpty()) {
  continue;
  }
 
  for (String lineId : ids) {
- putLineGraph(lineId, graphByLine).addVerticesAndEdges(geoShape.coordinates());
+ coordinateListsByLine.computeIfAbsent(lineId, key -> new ArrayList<>())
+ .add(geoShape.coordinates());
  }
  }
  } catch (IOException e) {
  throw new UncheckedIOException(e);
  }
  }
 
- private static LineGraph<Coordinate, Object> putLineGraph(String lineId, Map<String, Graph<Coordinate, Object>> graphByLine) {
- return (LineGraph<Coordinate, Object>) graphByLine.computeIfAbsent(lineId, key -> new LineGraph<>(Object.class));
- }
-
- private static List<Coordinate> getEnds(Set<Coordinate> connectedSet, Graph<Coordinate, Object> graph) {
- List<Coordinate> ends = new ArrayList<>();
- for (Coordinate coordinate : connectedSet) {
- if (Graphs.neighborListOf(graph, coordinate).size() == 1) {
- ends.add(coordinate);
- }
- }
- return ends;
- }
-
  private static double getBranchLength(List<Coordinate> coordinatesComponent) {
- return DistanceCalculator.distance(coordinatesComponent.get(0).getLatitude(), coordinatesComponent.get(0).getLongitude(),
- coordinatesComponent.get(coordinatesComponent.size() - 1).getLatitude(), coordinatesComponent.get(coordinatesComponent.size() - 1).getLongitude());
+ return distanceCoordinate(coordinatesComponent.get(0), coordinatesComponent.get(coordinatesComponent.size() - 1));
  }
 
  private static List<Coordinate> aggregateCoordinates(List<List<Coordinate>> coordinatesComponents) {
- coordinatesComponents.sort((comp1, comp2) -> (int) (getBranchLength(comp2) - getBranchLength(comp1)));
- return aggregateCoordinates(coordinatesComponents.get(0), coordinatesComponents.get(1));
- }
-
- private static List<Coordinate> aggregateCoordinates(List<Coordinate> coordinatesComponent1, List<Coordinate> coordinatesComponent2) {
  List<Coordinate> aggregatedCoordinates;
 
+ List<Coordinate> coordinatesComponent1 = coordinatesComponents.get(0);
+ List<Coordinate> coordinatesComponent2 = coordinatesComponents.get(1);
+
  double l1 = getBranchLength(coordinatesComponent1);
  double l2 = getBranchLength(coordinatesComponent2);
 
  if (100 * l2 / l1 < THRESHOLD) {
- return coordinatesComponent1;
+ return l1 > l2 ? coordinatesComponent1 : coordinatesComponent2;
  }
 
- double d1 = DistanceCalculator.distance(coordinatesComponent1.get(0).getLatitude(), coordinatesComponent1.get(0).getLongitude(),
- coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLatitude(), coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLongitude());
-
- double d2 = DistanceCalculator.distance(coordinatesComponent1.get(0).getLatitude(), coordinatesComponent1.get(0).getLongitude(),
- coordinatesComponent2.get(0).getLatitude(), coordinatesComponent2.get(0).getLongitude());
-
- double d3 = DistanceCalculator.distance(coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLatitude(), coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLongitude(),
- coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLatitude(), coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLongitude());
-
- double d4 = DistanceCalculator.distance(coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLatitude(), coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLongitude(),
- coordinatesComponent2.get(0).getLatitude(), coordinatesComponent2.get(0).getLongitude());
+ double d1 = distanceCoordinate(coordinatesComponent1.get(0), coordinatesComponent2.get(coordinatesComponent2.size() - 1));
+ double d2 = distanceCoordinate(coordinatesComponent1.get(0), coordinatesComponent2.get(0));
+ double d3 = distanceCoordinate(coordinatesComponent1.get(coordinatesComponent1.size() - 1), coordinatesComponent2.get(coordinatesComponent2.size() - 1));
+ double d4 = distanceCoordinate(coordinatesComponent1.get(coordinatesComponent1.size() - 1), coordinatesComponent2.get(0));
 
  List<Double> distances = Arrays.asList(d1, d2, d3, d4);
  double min = min(distances);
@@ -266,18 +269,52 @@ private static Pair<String, String> findStartAndEndSubstationsOfLine(String line
  return Pair.of(sub1pil1 < sub2pil1 ? substation1 : substation2, sub1pil1 < sub2pil1 ? substation2 : substation1);
  }
 
- private static List<List<Coordinate>> fillMultipleConnectedSetsCoordinatesList(List<Set<Coordinate>> connectedSets,
+ private static List<List<Coordinate>> fillMultipleConnectedSetsCoordinatesList(List<ConnectedSet> connectedSets,
  Graph<Coordinate, Object> graph) {
  List<List<Coordinate>> coordinatesComponents = new ArrayList<>();
- for (Set<Coordinate> connectedSet : connectedSets) {
- List<Coordinate> endsComponent = getEnds(connectedSet, graph);
+ for (ConnectedSet connectedSet : connectedSets) {
+ List<Coordinate> endsComponent = connectedSet.ends();
  if (endsComponent.size() == 2) {
- List<Coordinate> coordinatesComponent = Lists.newArrayList(new BreadthFirstIterator<>(graph, endsComponent.get(0)));
- coordinatesComponents.add(coordinatesComponent);
+ coordinatesComponents.add(connectedSet.list());
  } else {
  break;
  }
  }
  return coordinatesComponents;
  }
+
+ private record ConnectedSet(List<Coordinate> list, List<Coordinate> ends) {
+ }
+
+ private static class GeoTraversalListener extends TraversalListenerAdapter<Coordinate, Object> {
+ private final List<ConnectedSet> connectedSets;
+ private final Graph<Coordinate, Object> graph;
+ private List<Coordinate> currentConnectedSet;
+ private List<Coordinate> currentConnectedSetEnds;
+
+ public GeoTraversalListener(Graph<Coordinate, Object> graph, List<ConnectedSet> connectedSets) {
+ this.graph = graph;
+ this.connectedSets = connectedSets;
+ }
+
+ @Override
+ public void connectedComponentFinished(ConnectedComponentTraversalEvent e) {
+ connectedSets.add(new ConnectedSet(currentConnectedSet, currentConnectedSetEnds));
+ }
+
+ @Override
+ public void connectedComponentStarted(ConnectedComponentTraversalEvent e) {
+ currentConnectedSet = new ArrayList<>();
+ currentConnectedSetEnds = new ArrayList<>();
+ }
+
+ @Override
+ public void vertexTraversed(VertexTraversalEvent<Coordinate> e) {
+ Coordinate v = e.getVertex();
+ currentConnectedSet.add(v);
+ if (graph.degreeOf(v) == 1) {
+ currentConnectedSetEnds.add(v);
+ }
+ }
+ }
 }

iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderPostProcessor.java

@@ -78,13 +78,13 @@ public String getName() {
  @Override
  public void process(Network network, ComputationManager computationManager) {
  if (Files.exists(substationsFilePath)) {
- OdreGeoDataAdder.fillNetworkSubstationsGeoDataFromFile(network, substationsFilePath, odreConfig);
  boolean aerialLinesPresent = Files.exists(aerialLinesFilePath);
  boolean undergroundLinesPresent = Files.exists(undergroundLinesFilePath);
  if (aerialLinesPresent && undergroundLinesPresent) {
  OdreGeoDataAdder.fillNetworkLinesGeoDataFromFiles(network,
  aerialLinesFilePath, undergroundLinesFilePath, substationsFilePath, odreConfig);
  } else {
+ OdreGeoDataAdder.fillNetworkSubstationsGeoDataFromFile(network, substationsFilePath, odreConfig);
  String missingAerialFiles = aerialLinesPresent ? "" : aerialLinesFilePath + " ";
  String missingFiles = missingAerialFiles.concat(undergroundLinesPresent ? "" : undergroundLinesFilePath.toString());
  LOGGER.warn("Could not load lines geographical data, file(s) not found : {}", missingFiles);

iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineGraph.java

@@ -7,22 +7,20 @@
  */
 package com.powsybl.iidm.geodata.utils;
 
-import org.jgrapht.graph.Pseudograph;
+import org.jgrapht.graph.SimpleGraph;
 
 /**
  * @author Hugo Marcellin {@literal <hugo.marcelin at rte-france.com>}
  */
-public class LineGraph<V, E> extends Pseudograph<V, E> {
+public class LineGraph<V, E> extends SimpleGraph<V, E> {
  public LineGraph(Class<? extends E> edgeClass) {
  super(edgeClass);
  }
 
  public void addVerticesAndEdges(Iterable<V> vertices) {
  V previousVertex = null;
  for (V vertex : vertices) {
- if (!containsVertex(vertex)) {
- addVertex(vertex);
- }
+ addVertex(vertex);
  if (previousVertex != null) {
  addEdge(previousVertex, vertex);
  }