Fix s3 ncml

cdm/core/src/main/java/thredds/filesystem/MFileOS.java

@@ -53,7 +53,7 @@ public MFileOS(java.io.File file) {
   }
 
   public MFileOS(String filename) {
-    this.file = new File(filename);
+    this.file = new File(filename.replaceFirst("^file:", ""));
     this.lastModified = file.lastModified();
   }
 

cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java

@@ -13,6 +13,8 @@
 
 import com.google.common.collect.Multimap;
 import thredds.client.catalog.ServiceType;
+import thredds.inventory.MFile;
+import thredds.inventory.MFiles;
 import ucar.httpservices.HTTPFactory;
 import ucar.httpservices.HTTPMethod;
 import ucar.nc2.util.EscapeStrings;
@@ -59,9 +61,11 @@ public static List<String> getProtocols(String url) {
     StringBuilder buf = new StringBuilder(url);
     // If there are any leading protocols, then they must stop at the first '/'.
     int slashpos = buf.indexOf("/");
-    // Check special case of file:<path> with no slashes after file:
+    // Check special cases of file:<path> or cdms3:<path> with no slashes after:
     if (url.startsWith("file:") && "/\\".indexOf(url.charAt(5)) < 0) {
       allprotocols.add("file");
+    } else if (url.startsWith("cdms3:") && "/\\".indexOf(url.charAt(6)) < 0) {
+      allprotocols.add("cdms3");
     } else if (slashpos >= 0) {
       // Remove everything after the first slash
       buf.delete(slashpos + 1, buf.length());
@@ -128,7 +132,7 @@ public static DatasetUrl findDatasetUrl(String orgLocation) throws IOException {
     }
     pos = location.lastIndexOf('?');
     String query = null;
-    if (pos >= 0) {
+    if (pos >= 0 && !leadProtocol.equals("cdms3")) {
       query = trueUrl.substring(pos + 1);
       trueUrl = trueUrl.substring(0, pos);
     }
@@ -147,9 +151,9 @@ public static DatasetUrl findDatasetUrl(String orgLocation) throws IOException {
       // - we have file://<path> or file:<path>; we need to see if
       // the extension can help, otherwise, start defaulting.
       // - we have a simple url: e.g. http://... ; contact the server
-      if (leadProtocol.equals("file")) {
+      if (leadProtocol.equals("file") || leadProtocol.equals("cdms3")) {
         serviceType = decodePathExtension(trueUrl); // look at the path extension
-        if (serviceType == null && checkIfNcml(new File(location))) {
+        if (serviceType == null && checkIfNcml(MFiles.create(location))) {
           serviceType = ServiceType.NCML;
         }
       } else {
@@ -163,11 +167,10 @@ public static DatasetUrl findDatasetUrl(String orgLocation) throws IOException {
         }
       }
     }
-
     if (serviceType == ServiceType.NCML) { // ??
       // If lead protocol was null, then pretend it was a file
       // Note that technically, this should be 'file://'
-      trueUrl = (allProtocols.isEmpty() ? "file:" + trueUrl : location);
+      trueUrl = (allProtocols.isEmpty() ? "file:" + trueUrl : trueUrl);
     }
 
     // Add back the query and fragment (if any)
@@ -511,12 +514,12 @@ private static boolean checkIfRemoteNcml(String location) throws IOException {
     return false;
   }
 
-  private static boolean checkIfNcml(File file) throws IOException {
-    if (!file.exists()) {
+  private static boolean checkIfNcml(MFile mFile) throws IOException {
+    if (!mFile.exists()) {
       return false;
     }
 
-    try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file), NUM_BYTES_TO_DETERMINE_NCML)) {
+    try (BufferedInputStream in = new BufferedInputStream(mFile.getInputStream(), NUM_BYTES_TO_DETERMINE_NCML)) {
       byte[] bytes = new byte[NUM_BYTES_TO_DETERMINE_NCML];
       int bytesRead = in.read(bytes);
 

cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java

@@ -18,13 +18,14 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.StringTokenizer;
-import java.net.URL;
 import javax.annotation.Nullable;
 import org.jdom2.Element;
 import org.jdom2.JDOMException;
 import org.jdom2.Namespace;
 import org.jdom2.input.SAXBuilder;
 import org.jdom2.output.XMLOutputter;
+import thredds.inventory.MFile;
+import thredds.inventory.MFiles;
 import ucar.ma2.Array;
 import ucar.ma2.DataType;
 import ucar.nc2.Attribute;
@@ -346,26 +347,21 @@ public static NetcdfDataset.Builder readNcml(Reader r, String ncmlLocation, Canc
    */
   public static NetcdfDataset.Builder readNcml(String ncmlLocation, String referencedDatasetUri, CancelTask cancelTask)
       throws IOException {
-    URL url = new URL(ncmlLocation);
+    MFile mFile = MFiles.create(ncmlLocation);
 
     if (debugURL) {
       System.out.println(" NcmlReader open " + ncmlLocation);
-      System.out.println("   URL = " + url);
-      System.out.println("   external form = " + url.toExternalForm());
-      System.out.println("   protocol = " + url.getProtocol());
-      System.out.println("   host = " + url.getHost());
-      System.out.println("   path = " + url.getPath());
-      System.out.println("  file = " + url.getFile());
+      System.out.println("   Path = " + mFile.getPath());
     }
 
     org.jdom2.Document doc;
     try {
       SAXBuilder builder = new SAXBuilder();
       builder.setExpandEntities(false);
       if (debugURL) {
-        System.out.println(" NetcdfDataset URL = <" + url + ">");
+        System.out.println(" NetcdfDataset path = <" + mFile.getPath() + ">");
       }
-      doc = builder.build(url);
+      doc = builder.build(mFile.getInputStream());
     } catch (JDOMException e) {
       throw new IOException(e.getMessage());
     }

cdm/core/src/main/java/ucar/nc2/util/URLnaming.java

@@ -4,6 +4,9 @@
  */
 package ucar.nc2.util;
 
+import java.io.IOException;
+import thredds.inventory.MFile;
+import thredds.inventory.MFiles;
 import ucar.unidata.util.StringUtil2;
 import java.io.File;
 import java.net.URI;
@@ -86,6 +89,20 @@ public static String resolve(String baseUri, String relativeUri) {
       }
     }
 
+    if (baseUri.startsWith("cdms3:")) {
+      if (relativeUri.startsWith("cdms3:")) {
+        return relativeUri;
+      } else {
+        MFile absoluteMFile;
+        try {
+          absoluteMFile = MFiles.create(baseUri).getParent().getChild(relativeUri);
+        } catch (IOException e) {
+          return relativeUri;
+        }
+        return absoluteMFile == null ? relativeUri : absoluteMFile.getPath();
+      }
+    }
+
     // non-file URLs
 
     // relativeUri = canonicalizeRead(relativeUri);
@@ -116,12 +133,18 @@ public static String resolveFile(String baseDir, String filepath) {
     if (baseDir.startsWith("file:"))
       baseDir = baseDir.substring(5);
 
-    File base = new File(baseDir);
-    if (!base.isDirectory())
-      base = base.getParentFile();
+    MFile base = MFiles.create(baseDir);
+    if (!base.isDirectory()) {
+      try {
+        base = base.getParent();
+      } catch (IOException e) {
+        return filepath;
+      }
+    }
     if (base == null)
       return filepath;
-    return base.getAbsolutePath() + "/" + filepath;
+    MFile absoluteFile = base.getChild(filepath);
+    return absoluteFile == null ? filepath : absoluteFile.getPath();
   }
 
 }

cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java

@@ -21,7 +21,6 @@
  */
 public class TestDatasetUrl {
   private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  static final boolean show = true;
 
   protected void protocheck(String path, String expected) {
     if (expected == null)
@@ -32,11 +31,6 @@ protected void protocheck(String path, String expected) {
     StringBuffer buff = new StringBuffer();
     protocols.forEach(p -> buff.append(p).append(":"));
     String result = buff.toString();
-    boolean ok = expected.equals(result);
-    if (show || !ok)
-      System.out.printf(" %s <- %s%n", result, path);
-    if (!ok)
-      System.out.printf("  !!!EXPECTED '%s'%n", expected);
     assertThat(result).isEqualTo(expected);
   }
 
@@ -76,11 +70,6 @@ public void testGetProtocols() {
 
   protected void testFind(String path, ServiceType expected) throws IOException {
     DatasetUrl result = DatasetUrl.findDatasetUrl(path);
-    boolean ok = (expected == null) ? result.serviceType == null : expected == result.serviceType;
-    if (show || !ok)
-      System.out.printf(" %s <- %s%n", result.serviceType, path);
-    if (!ok)
-      System.out.printf("  !!!EXPECTED '%s'%n", expected);
     assertThat(result.serviceType).isEqualTo(expected);
   }
 
@@ -123,5 +112,16 @@ public void testFindDatasetUrl() throws IOException {
     testFind(
         "dynamic:http://thredds.ucar.edu:8080/thredds/fmrc/NCEP/GFS/CONUS_95km/files/GFS_CONUS_95km_20070319_0600.grib1",
         null);
+
+    testFind("file:/path/to/file.ncml", ServiceType.NCML);
+    testFind("/path/to/file.ncml", ServiceType.NCML);
+    testFind("file:/path/to/file.xml", ServiceType.NCML);
+    testFind("/path/to/file.xml", ServiceType.NCML);
+
+    testFind("cdms3:thredds-test-data?testStandalone.ncml#delimiter=/", ServiceType.NCML);
+    testFind("cdms3:thredds-test-data?testStandalone.ncml", ServiceType.NCML);
+    testFind("cdms3:thredds-test-data?ncml/testStandalone.ncml#delimiter=/", ServiceType.NCML);
+    testFind("cdms3:thredds-test-data?ncml/testStandalone.ncml", ServiceType.NCML);
+    testFind("cdms3://profile_name@my.endpoint.edu/bucket-name?super/long/key.ncml#delimiter=/", ServiceType.NCML);
   }
 }