Datasources - Part 2: main extension + existStrict + builder

cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java

@@ -189,6 +189,11 @@ public boolean exists(String suffix, String ext) throws IOException {
  return ds.exists(suffix, ext) && filter.test(DataSourceUtil.getFileName(getBaseName(), suffix, ext));
  }
 
+ @Override
+ public boolean isMainExtension(String ext) {
+ return ds.isMainExtension(ext);
+ }
+
  @Override
  public boolean exists(String fileName) throws IOException {
  return ds.exists(fileName) && filter.test(fileName);

cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java

@@ -14,6 +14,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
@@ -24,6 +25,8 @@
  * @author Luma Zamarreño {@literal <zamarrenolm at aia.es>}
  */
 public class CgmesOnDataSource {
+ private static final String[] EXTENSIONS = {"xml"};
+
  public CgmesOnDataSource(ReadOnlyDataSource ds) {
  this.dataSource = ds;
  }
@@ -32,29 +35,53 @@ public ReadOnlyDataSource dataSource() {
  return dataSource;
  }
 
+ private boolean checkIfMainFileNotWithCgmesData(boolean isCim14) {
+ if (dataSource.getMainExtension() != null && Arrays.asList(EXTENSIONS).contains(dataSource.getMainExtension())) {
+ try (InputStream is = dataSource.newInputStream(null, dataSource.getMainExtension())) {
+ return isCim14 ? !existsNamespacesCim14(NamespaceReader.namespaces(is)) : !existsNamespaces(NamespaceReader.namespaces(is));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ return true;
+ }
+
  public boolean exists() {
+ // Check that the main file is a CGMES file
+ if (dataSource.getMainExtension() != null && !dataSource.getMainExtension().isEmpty() && checkIfMainFileNotWithCgmesData(false)) {
+ return false;
+ }
  // check that RDF and CIM16 are defined as namespaces in the data source
- Set<String> foundNamespaces = namespaces();
- if (!foundNamespaces.contains(RDF_NAMESPACE)) {
+ return existsNamespaces(namespaces());
+ }
+
+ private boolean existsNamespaces(Set<String> namespaces) {
+ if (!namespaces.contains(RDF_NAMESPACE)) {
  return false;
  }
  // FIXME(Luma) This is legacy behaviour, we do not consider CIM14 valid in this check
  // But I think we do not need to support 14 separately?
- return foundNamespaces.contains(CIM_16_NAMESPACE) || foundNamespaces.contains(CIM_100_NAMESPACE);
+ return namespaces.contains(CIM_16_NAMESPACE) || namespaces.contains(CIM_100_NAMESPACE);
  }
 
  public boolean existsCim14() {
+ // Check that the main file is a CGMES file
+ if (dataSource.getMainExtension() != null && !dataSource.getMainExtension().isEmpty() && checkIfMainFileNotWithCgmesData(true)) {
+ return false;
+ }
  // check that RDF and CIM16 are defined as namespaces in the data source
- Set<String> foundNamespaces = namespaces();
- if (!foundNamespaces.contains(RDF_NAMESPACE)) {
+ return existsNamespacesCim14(namespaces());
+ }
+
+ private boolean existsNamespacesCim14(Set<String> namespaces) {
+ if (!namespaces.contains(RDF_NAMESPACE)) {
  return false;
  }
  // FIXME(Luma) This is legacy behaviour, we do not consider CIM14 valid in this check
  // But I think we do not need to support 14 separately?
- if (!foundNamespaces.contains(CIM_14_NAMESPACE)) {
+ if (!namespaces.contains(CIM_14_NAMESPACE)) {
  return false;
  }
-
  return names().stream().anyMatch(CgmesSubset.EQUIPMENT::isValidName);
  }
 

commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java

@@ -17,8 +17,8 @@ public abstract class AbstractArchiveDataSource extends AbstractFileSystemDataSo
  private final String archiveFileName;
  final ArchiveFormat archiveFormat;
 
- AbstractArchiveDataSource(Path directory, String archiveFileName, String baseName, CompressionFormat compressionFormat, ArchiveFormat archiveFormat, DataSourceObserver observer) {
- super(directory, baseName, compressionFormat, observer);
+ AbstractArchiveDataSource(Path directory, String archiveFileName, String baseName, String mainExtension, CompressionFormat compressionFormat, ArchiveFormat archiveFormat, DataSourceObserver observer) {
+ super(directory, baseName, mainExtension, compressionFormat, observer);
  this.archiveFileName = archiveFileName;
  this.archiveFormat = archiveFormat;
  }

commons/src/main/java/com/powsybl/commons/datasource/AbstractFileSystemDataSource.java

@@ -17,6 +17,7 @@
 abstract class AbstractFileSystemDataSource implements DataSource {
  final Path directory;
  final String baseName;
+ final String mainExtension;
  final CompressionFormat compressionFormat;
  final DataSourceObserver observer;
 
@@ -27,14 +28,21 @@ abstract class AbstractFileSystemDataSource implements DataSource {
  * @param observer Data source observer
  */
  AbstractFileSystemDataSource(Path directory, String baseName,
+ String mainExtension,
  CompressionFormat compressionFormat,
  DataSourceObserver observer) {
  this.directory = Objects.requireNonNull(directory);
  this.baseName = Objects.requireNonNull(baseName);
+ this.mainExtension = mainExtension;
  this.compressionFormat = compressionFormat;
  this.observer = observer;
  }
 
+ @Override
+ public boolean isMainExtension(String ext) {
+ return mainExtension == null || mainExtension.isEmpty() || mainExtension.equals(ext);
+ }
+
  public Path getDirectory() {
  return this.directory;
  }
@@ -44,10 +52,19 @@ public String getBaseName() {
  return baseName;
  }
 
+ @Override
+ public String getMainExtension() {
+ return mainExtension;
+ }
+
  public CompressionFormat getCompressionFormat() {
  return compressionFormat;
  }
 
+ String getCompressionExtension() {
+ return compressionFormat == null ? "" : "." + compressionFormat.getExtension();
+ }
+
  public DataSourceObserver getObserver() {
  return observer;
  }

commons/src/main/java/com/powsybl/commons/datasource/Bzip2DirectoryDataSource.java

@@ -18,12 +18,8 @@
  */
 public class Bzip2DirectoryDataSource extends DirectoryDataSource {
 
- public Bzip2DirectoryDataSource(Path directory, String baseName, DataSourceObserver observer) {
- super(directory, baseName, CompressionFormat.BZIP2, observer);
- }
-
- public Bzip2DirectoryDataSource(Path directory, String baseName) {
- super(directory, baseName, CompressionFormat.BZIP2, null);
+ public Bzip2DirectoryDataSource(Path directory, String baseName, String mainExtension, DataSourceObserver observer) {
+ super(directory, baseName, mainExtension, CompressionFormat.BZIP2, observer);
  }
 
  @Override

commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2024, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package com.powsybl.commons.datasource;
+
+import com.powsybl.commons.PowsyblException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * @author Nicolas Rol {@literal <nicolas.rol at rte-france.com>}
+ */
+class DataSourceBuilder {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceBuilder.class);
+ private Path directory;
+ private String baseName;
+ private String archiveFileName;
+ private CompressionFormat compressionFormat;
+ private ArchiveFormat archiveFormat;
+ private String mainExtension = "";
+ private DataSourceObserver observer;
+
+ DataSourceBuilder withDirectory(Path directory) {
+ this.directory = directory;
+ return this;
+ }
+
+ DataSourceBuilder withBaseName(String baseName) {
+ this.baseName = baseName;
+ return this;
+ }
+
+ DataSourceBuilder withArchiveFileName(String archiveFileName) {
+ this.archiveFileName = archiveFileName;
+ return this;
+ }
+
+ DataSourceBuilder withCompressionFormat(CompressionFormat compressionFormat) {
+ this.compressionFormat = compressionFormat;
+ return this;
+ }
+
+ DataSourceBuilder withArchiveFormat(ArchiveFormat archiveFormat) {
+ this.archiveFormat = archiveFormat;
+ return this;
+ }
+
+ DataSourceBuilder withMainExtension(String mainExtension) {
+ this.mainExtension = mainExtension;
+ return this;
+ }
+
+ DataSourceBuilder withObserver(DataSourceObserver observer) {
+ this.observer = observer;
+ return this;
+ }
+
+ DataSource build() {
+ // Check the mandatory parameters
+ buildChecks();
+
+ // Create the datasource
+ if (compressionFormat == CompressionFormat.ZIP || archiveFormat == ArchiveFormat.ZIP) {
+ return buildZip();
+ } else if (compressionFormat == null) {
+ return new DirectoryDataSource(directory, baseName, mainExtension, observer);
+ } else {
+ return switch (compressionFormat) {
+ case BZIP2 -> new Bzip2DirectoryDataSource(directory, baseName, mainExtension, observer);
+ case GZIP -> new GzDirectoryDataSource(directory, baseName, mainExtension, observer);
+ case XZ -> new XZDirectoryDataSource(directory, baseName, mainExtension, observer);
+ case ZSTD -> new ZstdDirectoryDataSource(directory, baseName, mainExtension, observer);
+ default -> {
+ LOGGER.warn("Unsupported compression format {}", compressionFormat);
+ yield new DirectoryDataSource(directory, baseName, mainExtension, observer);
+ }
+ };
+ }
+ }
+
+ private void buildChecks() {
+ if (directory == null) {
+ throw new PowsyblException("Datasource directory cannot be null");
+ }
+ if (!Files.isDirectory(directory)) {
+ throw new PowsyblException("Datasource directory has to be a directory");
+ }
+ if (baseName == null) {
+ throw new PowsyblException("Datasource baseName cannot be null");
+ }
+ }
+
+ private DataSource buildZip() {
+ if (compressionFormat != null && archiveFormat != null && !(compressionFormat == CompressionFormat.ZIP && archiveFormat == ArchiveFormat.ZIP)) {
+ throw new PowsyblException(String.format("Incoherence between compression format %s and archive format %s", compressionFormat, archiveFormat));
+ }
+ return archiveFileName == null ?
+ new ZipArchiveDataSource(directory, baseName, mainExtension, observer) :
+ new ZipArchiveDataSource(directory, archiveFileName, baseName, mainExtension, observer);
+ }
+}

commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java

@@ -40,45 +40,44 @@ static String getBaseName(String fileName) {
  }
 
  static DataSource createDataSource(Path directory, String basename, CompressionFormat compressionExtension, DataSourceObserver observer) {
+ return createDataSource(directory, basename, null, compressionExtension, observer);
+ }
+
+ static DataSource createDataSource(Path directory, String basename, String mainExtension, CompressionFormat compressionFormat, DataSourceObserver observer) {
  Objects.requireNonNull(directory);
  Objects.requireNonNull(basename);
 
- if (compressionExtension == null) {
- return new DirectoryDataSource(directory, basename, observer);
- } else {
- switch (compressionExtension) {
- case BZIP2:
- return new Bzip2DirectoryDataSource(directory, basename, observer);
- case GZIP:
- return new GzDirectoryDataSource(directory, basename, observer);
- case XZ:
- return new XZDirectoryDataSource(directory, basename, observer);
- case ZIP:
- return new ZipArchiveDataSource(directory, basename, observer);
- case ZSTD:
- return new ZstdDirectoryDataSource(directory, basename, observer);
- default:
- throw new IllegalStateException("Unexpected CompressionFormat value: " + compressionExtension);
- }
+ DataSourceBuilder dataSourceBuilder = new DataSourceBuilder()
+ .withDirectory(directory)
+ .withBaseName(basename)
+ .withMainExtension(mainExtension)
+ .withCompressionFormat(compressionFormat)
+ .withObserver(observer);
+
+ // If a zip compression is asked
+ if (compressionFormat == CompressionFormat.ZIP) {
+ dataSourceBuilder.withArchiveFormat(ArchiveFormat.ZIP);
  }
+
+ return dataSourceBuilder.build();
  }
 
  static DataSource createDataSource(Path directory, String fileNameOrBaseName, DataSourceObserver observer) {
  Objects.requireNonNull(directory);
  Objects.requireNonNull(fileNameOrBaseName);
 
- if (fileNameOrBaseName.endsWith(".zst")) {
-  return new ZstdDirectoryDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer);
- } else if (fileNameOrBaseName.endsWith(".zip")) {
-  return new ZipArchiveDataSource(directory, fileNameOrBaseName, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer);
- } else if (fileNameOrBaseName.endsWith(".xz")) {
- return new XZDirectoryDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3)), observer);
- } else if (fileNameOrBaseName.endsWith(".gz")) {
- return new GzDirectoryDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3)), observer);
- } else if (fileNameOrBaseName.endsWith(".bz2")) {
- return new Bzip2DirectoryDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer);
- } else {
- return new DirectoryDataSource(directory, getBaseName(fileNameOrBaseName), observer);
- }
+ // Get the file information
+ FileInformation fileInformation = new FileInformation(fileNameOrBaseName);
+
+ DataSourceBuilder dataSourceBuilder = new DataSourceBuilder()
+  .withDirectory(directory)
+ .withArchiveFileName(fileNameOrBaseName)
+  .withBaseName(fileInformation.getBaseName())
+ .withMainExtension(fileInformation.getMainExtension())
+  .withCompressionFormat(fileInformation.getCompressionFormat())
+ .withArchiveFormat(fileInformation.getArchiveFormat())
+  .withObserver(observer);
+
+ return dataSourceBuilder.build();
  }
 }