From f0d7388e08c34aefcb4fa0fce3b9294df349ed3d Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 13 Sep 2024 14:28:38 -0600 Subject: [PATCH 1/2] Add S3 Radar Server tests --- .../thredds/server/radar/TestRadarServer.java | 10 +- .../server/radar/TestRadarServerQuery.java | 41 +++-- .../thredds/radar/radarCollections.xml | 163 ++++++++++++++++++ 3 files changed, 200 insertions(+), 14 deletions(-) diff --git a/tds/src/integrationTests/java/thredds/server/radar/TestRadarServer.java b/tds/src/integrationTests/java/thredds/server/radar/TestRadarServer.java index 1440f453b9..8200585835 100644 --- a/tds/src/integrationTests/java/thredds/server/radar/TestRadarServer.java +++ b/tds/src/integrationTests/java/thredds/server/radar/TestRadarServer.java @@ -31,7 +31,15 @@ public static java.util.Collection getTestParameters() { // {"/radar/radarCollections.xml"}, {"/radarServer/nexrad/level2/IDD/dataset.xml"}, {"/radarServer/nexrad/level2/IDD/stations.xml"}, {"/radarServer/nexrad/level2/IDD?stn=KDGX&time_start=2014-06-05T12:47:17&time_end=2014-06-05T16:07:17"}, - {"/radarServer/nexrad/level3/IDD/stations.xml"}, {"/radarServer/terminal/level3/IDD/stations.xml"},}); + {"/radarServer/nexrad/level3/IDD/stations.xml"}, {"/radarServer/terminal/level3/IDD/stations.xml"}, + + + // s3 tests + {"/radarServer/s3/nexrad/level2/IDD/dataset.xml"}, {"/radarServer/s3/nexrad/level2/IDD/stations.xml"}, + {"/radarServer/s3/nexrad/level2/IDD?stn=KDGX&time_start=2014-06-05T12:47:17&time_end=2014-06-05T16:07:17"}, + {"/radarServer/s3/nexrad/level3/IDD/stations.xml"}, + + }); } private static final String expectedContentType = "application/xml"; diff --git a/tds/src/integrationTests/java/thredds/server/radar/TestRadarServerQuery.java b/tds/src/integrationTests/java/thredds/server/radar/TestRadarServerQuery.java index a3fb8bd386..cf44184113 100644 --- a/tds/src/integrationTests/java/thredds/server/radar/TestRadarServerQuery.java +++ b/tds/src/integrationTests/java/thredds/server/radar/TestRadarServerQuery.java @@ -10,62 +10,77 @@ import org.jdom2.xpath.XPathFactory; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import thredds.test.util.TestOnLocalServer; import thredds.util.xml.XmlUtil; import ucar.unidata.util.test.category.NeedsCdmUnitTest; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import static com.google.common.truth.Truth.assertThat; @Category(NeedsCdmUnitTest.class) +@RunWith(Parameterized.class) public class TestRadarServerQuery { private static final Namespace NS = Namespace.getNamespace("ns", "http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0"); - private static final String L2_URL = "/radarServer/nexrad/level2/IDD"; - private static final String L3_URL = "/radarServer/nexrad/level3/IDD"; + + private final String l2Url; + private final String l3Url; + + @Parameterized.Parameters(name = "{0}") + public static List getTestParameters() { + return Arrays.asList("", "s3/"); + } + + public TestRadarServerQuery(String datasetPathPrefix) { + l2Url = "/radarServer/" + datasetPathPrefix + "nexrad/level2/IDD"; + l3Url = "/radarServer/" + datasetPathPrefix + "nexrad/level3/IDD"; + } @Test public void shouldReturnAllDatasetsForStation() throws IOException, JDOMException { - String endpoint = L2_URL + "?stn=KDGX&temporal=all"; + String endpoint = l2Url + "?stn=KDGX&temporal=all"; verifyNumberOfDatasets(endpoint, 3); } @Test public void shouldReturnZeroDatasetsForNonOverlappingTimeRange() throws IOException, JDOMException { - String endpoint = L2_URL + "?stn=KDGX&time_start=2000-01-01T12:00:00&time_end=2001-01-01T12:00:00"; + String endpoint = l2Url + "?stn=KDGX&time_start=2000-01-01T12:00:00&time_end=2001-01-01T12:00:00"; verifyNumberOfDatasets(endpoint, 0); } @Test public void shouldReturnOneDatasetForOverlappingTimeRange() throws IOException, JDOMException { - String endpoint = L2_URL + "?stn=KDGX&time_start=2014-06-02T23:52:00&time_end=2014-06-02T23:53:00"; + String endpoint = l2Url + "?stn=KDGX&time_start=2014-06-02T23:52:00&time_end=2014-06-02T23:53:00"; verifyNumberOfDatasets(endpoint, 1); } @Test public void shouldReturnOneDatasetForOverlappingTimeDuration() throws IOException, JDOMException { - String endpoint = L2_URL + "?stn=KDGX&time_start=2014-06-02T23:52:00&time_duration=PT1M"; + String endpoint = l2Url + "?stn=KDGX&time_start=2014-06-02T23:52:00&time_duration=PT1M"; verifyNumberOfDatasets(endpoint, 1); } @Test public void shouldReturnOneDatasetForTime() throws IOException, JDOMException { - String endpoint = L2_URL + "?stn=KDGX&time=2014-06-02T23:52:00"; + String endpoint = l2Url + "?stn=KDGX&time=2014-06-02T23:52:00"; verifyNumberOfDatasets(endpoint, 1); } @Test public void shouldReturnZeroDatasetsForNonExistentStation() throws IOException, JDOMException { - String endpoint = L2_URL + "?stn=ABCD&temporal=all"; + String endpoint = l2Url + "?stn=ABCD&temporal=all"; verifyNumberOfDatasets(endpoint, 0); } @Test public void shouldReturnErrorForNonOverlappingBox() throws IOException, JDOMException { - String endpoint = L2_URL + "?north=10&south=0&west=-100&east=-80&temporal=all"; + String endpoint = l2Url + "?north=10&south=0&west=-100&east=-80&temporal=all"; byte[] result = TestOnLocalServer.getContent(TestOnLocalServer.withHttpPath(endpoint), HttpServletResponse.SC_OK, "text/plain;charset=iso-8859-1"); assertThat(new String(result, StandardCharsets.UTF_8)).isEqualTo("No stations found for specified coordinates."); @@ -73,25 +88,25 @@ public void shouldReturnErrorForNonOverlappingBox() throws IOException, JDOMExce @Test public void shouldReturnAllDatasetsForOverlappingBox() throws IOException, JDOMException { - String endpoint = L2_URL + "?north=50&south=30&west=-100&east=-80&temporal=all"; + String endpoint = l2Url + "?north=50&south=30&west=-100&east=-80&temporal=all"; verifyNumberOfDatasets(endpoint, 3); } @Test public void shouldReturnAllDatasetsForLonLat() throws IOException, JDOMException { - String endpoint = L2_URL + "?latitude=30&longitude=-90&temporal=all"; + String endpoint = l2Url + "?latitude=30&longitude=-90&temporal=all"; verifyNumberOfDatasets(endpoint, 3); } @Test public void shouldReturnAllLevel3Datasets() throws IOException, JDOMException { - String endpoint = L3_URL + "?temporal=all&var=N0R&stn=UDX"; + String endpoint = l3Url + "?temporal=all&var=N0R&stn=UDX"; verifyNumberOfDatasets(endpoint, 329); } @Test public void shouldReturnErrorWithoutVar() throws IOException, JDOMException { - String endpoint = L3_URL + "?temporal=all&stn=UDX"; + String endpoint = l3Url + "?temporal=all&stn=UDX"; byte[] result = TestOnLocalServer.getContent(TestOnLocalServer.withHttpPath(endpoint), HttpServletResponse.SC_OK, "text/plain;charset=iso-8859-1"); assertThat(new String(result, StandardCharsets.UTF_8)).isEqualTo("One or more variables required."); diff --git a/tds/src/test/content/thredds/radar/radarCollections.xml b/tds/src/test/content/thredds/radar/radarCollections.xml index e0e14e35f0..eece1ab041 100644 --- a/tds/src/test/content/thredds/radar/radarCollections.xml +++ b/tds/src/test/content/thredds/radar/radarCollections.xml @@ -47,6 +47,46 @@ + + + + Radial + NEXRAD2 + radarServer + NEXRAD Level II Radar WSR-88D for Case Study CCS039. + + 1998-06-29T18:00:00 + 1998-06-29T23:00:00 + + + + + + + + + nexrad/level2/CCS039 + + + 37.0 + 9.0 + degrees_north + + + -100.0 + 16.0 + degrees_east + + + 0.0 + 21.0 + km + + + + + @@ -169,6 +209,129 @@ + + + + Radial + NIDS + radarServer + + The NIDS data feed provides roughly 20 radar products sent every 5-10 minutes from 154 sites over NOAAPORT broadcast. These "derived" products include base reflectivity and velocity, composite reflectivity, precipitation estimates, echo tops and VAD winds + + + + + DOC/NOAA/NWS + + + + + 20.0 + 40.0 + degrees_north + + + -135.0 + 70.0 + degrees_east + + + 0.0 + 0.0 + km + + + + present + 14 days + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 27ac865037b090a80aae6f8fb73360c5773c1b39 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 18 Sep 2024 13:35:12 -0600 Subject: [PATCH 2/2] Use MFile instead of Path to get radar server to work for S3 --- .../radarServer2/RadarDataInventory.java | 146 ++++++++++-------- .../radarServer2/RadarServerConfig.java | 67 +------- .../radarServer2/RadarServerController.java | 6 +- 3 files changed, 87 insertions(+), 132 deletions(-) diff --git a/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java b/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java index 83e7e84087..84c38e23ec 100644 --- a/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java +++ b/tds/src/main/java/thredds/server/radarServer2/RadarDataInventory.java @@ -5,6 +5,10 @@ package thredds.server.radarServer2; +import thredds.inventory.CollectionConfig; +import thredds.inventory.MController; +import thredds.inventory.MControllers; +import thredds.inventory.MFile; import ucar.nc2.constants.FeatureType; import ucar.nc2.dt.RadialDatasetSweep; import ucar.nc2.ft.FeatureDatasetFactoryManager; @@ -33,7 +37,7 @@ public enum DirType { private static final long updateIntervalMsec = 600000; private EnumMap> items; - private Path collectionDir; + private MFile collectionDir; private DirectoryStructure structure; private String fileTimeFmt, dataFormat; private java.util.regex.Pattern fileTimeRegex; @@ -46,7 +50,7 @@ public enum DirType { private DateRange timeCoverage; private RadarServerConfig.RadarConfigEntry.GeoInfo geoCoverage; - public RadarDataInventory(Path datasetRoot, int numCrawl) { + public RadarDataInventory(MFile datasetRoot, int numCrawl) { items = new EnumMap<>(DirType.class); collectionDir = datasetRoot; structure = new DirectoryStructure(collectionDir); @@ -56,7 +60,7 @@ public RadarDataInventory(Path datasetRoot, int numCrawl) { nearestWindow = CalendarPeriod.of(1, CalendarPeriod.Field.Hour); } - public Path getCollectionDir() { + public MFile getCollectionDir() { return collectionDir; } @@ -159,11 +163,11 @@ public SimpleDateFormat getFormat() { return sdf; } - public Date getDate(Path path) { - Path relPath = base.relativize(path); + public Date getDate(MFile mFile) { + String relPath = base.relativize(mFile); StringBuilder sb = new StringBuilder(""); for (Integer l : levels) { - sb.append(relPath.getName(l)); + sb.append(Paths.get(relPath).getName(l)); } try { SimpleDateFormat fmt = getFormat(); @@ -174,12 +178,12 @@ public Date getDate(Path path) { } } - private Path base; + private MFile base; private List order; private List keyIndices; - public DirectoryStructure(Path dir) { + public DirectoryStructure(MFile dir) { base = dir; order = new ArrayList<>(); keyIndices = new ArrayList<>(); @@ -194,11 +198,12 @@ public void addSubDir(DirType type, String fmt) { } // Get a key for a path based on station/var - public String getKey(Path path) { - Path relPath = base.relativize(path); + public String getKey(MFile mFile) { + String relPath = base.relativize(mFile); StringBuilder sb = new StringBuilder(""); + for (int ind : keyIndices) { - sb.append(relPath.getName(ind)); + sb.append(Paths.get(relPath).getName(ind)); } return sb.toString(); } @@ -251,7 +256,7 @@ public void addFileTime(String regex, String fmt) { fileTimeFmt = fmt; } - private void findItems(Path start, int level) { + private void findItems(MFile start, int level) { // Add each entry from this level to the appropriate item box // and recurse if (level >= structure.order.size() || level >= structure.maxCrawlDepth) @@ -268,23 +273,17 @@ private void findItems(Path start, int level) { } int crawled = 0; - try (DirectoryStream dirStream = Files.newDirectoryStream(start)) { - for (Path p : dirStream) { - if (Files.isDirectory(p)) { - String item = p.getFileName().toString(); - values.add(item); - // Try to grab station info from some file - // TODO: Fix or remove - // if (entry.type == DirType.Station) - // updateStations(item, p); - if (crawled < maxCrawlItems) { - findItems(p, level + 1); - ++crawled; - } - } + for (MFile subDir : getSubDirs(start)) { + String item = subDir.getName(); + values.add(item); + // Try to grab station info from some file + // TODO: Fix or remove + // if (entry.type == DirType.Station) + // updateStations(item, p); + if (crawled < maxCrawlItems) { + findItems(subDir, level + 1); + ++crawled; } - } catch (IOException e) { - System.out.println("findItems(): Error reading directory: " + start.toString()); } } @@ -354,12 +353,12 @@ public Query newQuery() { public class Query { public class QueryResultItem { - private QueryResultItem(Path f, CalendarDate cd) { + private QueryResultItem(MFile f, CalendarDate cd) { file = f; time = cd; } - public Path file; + public MFile file; public CalendarDate time; } @@ -412,8 +411,8 @@ private boolean checkDate(CalendarDateRange range, CalendarDate d) { return range == null || range.includes(d); } - public Collection results() { - List results = new ArrayList<>(); + public Collection results() throws IOException { + List results = new ArrayList<>(); DirectoryStructure.DirectoryDateMatcher matcher = structure.matcher(); results.add(structure.base); @@ -432,7 +431,7 @@ public Collection results() { // exists. For dates, add the items that are within the filter for (int i = 0; i < structure.order.size(); ++i) { DirectoryStructure.DirEntry entry = structure.order.get(i); - List newResults = new ArrayList<>(); + List newResults = new ArrayList<>(); List queryItem = q.get(entry.type); switch (entry.type) { // Loop over results and add subdirs that are within the @@ -443,15 +442,11 @@ public Collection results() { SimpleDateFormat fmt = matcher.getFormat(); CalendarDateRange dirRange = rangeFromFormat(fmt, range); - for (Path p : results) - try (DirectoryStream dirStream = Files.newDirectoryStream(p)) { - for (Path sub : dirStream) { - Date d = matcher.getDate(sub); - if (d != null && checkDate(dirRange, CalendarDate.of(d))) - newResults.add(sub); - } - } catch (IOException e) { - System.out.println("results(): Error reading dir: " + p.toString()); + for (MFile mFile : results) + for (MFile sub : getSubDirs(mFile)) { + Date d = matcher.getDate(sub); + if (d != null && checkDate(dirRange, CalendarDate.of(d))) + newResults.add(sub); } break; @@ -461,10 +456,11 @@ public Collection results() { case Variable: default: for (Object next : queryItem) { - for (Path p : results) { - Path nextPath = p.resolve(next.toString()); - if (Files.exists(nextPath)) - newResults.add(nextPath); + for (MFile mFile : results) { + MFile nextMFile = mFile.getChild(next.toString() + "/"); + if (nextMFile != null && nextMFile.exists()) { + newResults.add(nextMFile); + } } } } @@ -473,28 +469,24 @@ public Collection results() { // Now get the contents of the remaining directories Collection filteredFiles = new ArrayList<>(); - for (Path p : results) { - try (DirectoryStream dirStream = Files.newDirectoryStream(p)) { - for (Path f : dirStream) { - java.util.regex.Matcher regexMatcher = fileTimeRegex.matcher(f.toString()); - if (!regexMatcher.find()) - continue; - - try { - SimpleDateFormat fmt = new SimpleDateFormat(fileTimeFmt); - fmt.setTimeZone(TimeZone.getTimeZone("UTC")); - Date d = fmt.parse(regexMatcher.group()); - if (d != null) { - CalendarDate cd = CalendarDate.of(d); - if (checkDate(range, cd)) - filteredFiles.add(new QueryResultItem(f, cd)); - } - } catch (ParseException e) { - // Ignore file + for (MFile dir : results) { + for (MFile mFile : getFiles(dir)) { + java.util.regex.Matcher regexMatcher = fileTimeRegex.matcher(mFile.getName()); + if (!regexMatcher.find()) + continue; + + try { + SimpleDateFormat fmt = new SimpleDateFormat(fileTimeFmt); + fmt.setTimeZone(TimeZone.getTimeZone("UTC")); + Date d = fmt.parse(regexMatcher.group()); + if (d != null) { + CalendarDate cd = CalendarDate.of(d); + if (checkDate(range, cd)) + filteredFiles.add(new QueryResultItem(mFile, cd)); } + } catch (ParseException e) { + // Ignore file } - } catch (IOException e) { - System.out.println("results(): Error getting files for: " + p.toString()); } } @@ -524,4 +516,26 @@ public Collection results() { return filteredFiles; } } + + private List getSubDirs(MFile directory) { + final MController mController = MControllers.create(directory.getPath()); + final List mFiles = new ArrayList<>(); + + final CollectionConfig dirs = new CollectionConfig("dirs", directory.getPath(), false, null, null); + final Iterator dirIterator = mController.getSubdirs(dirs, true); + dirIterator.forEachRemaining(mFiles::add); + + return mFiles; + } + + private List getFiles(MFile directory) throws IOException { + final MController mController = MControllers.create(directory.getPath()); + final List mFiles = new ArrayList<>(); + + final CollectionConfig files = new CollectionConfig("files", directory.getPath(), false, null, null); + final Iterator fileIterator = mController.getInventoryTop(files, true); + fileIterator.forEachRemaining(mFiles::add); + + return mFiles; + } } diff --git a/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java b/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java index cd5559f8be..8ddb6bb273 100644 --- a/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java +++ b/tds/src/main/java/thredds/server/radarServer2/RadarServerConfig.java @@ -10,6 +10,8 @@ import org.jdom2.Namespace; import org.jdom2.JDOMException; import org.jdom2.input.SAXBuilder; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.constants.CDM; import ucar.nc2.units.DateRange; import ucar.nc2.units.DateType; @@ -18,20 +20,13 @@ import java.io.File; import java.io.IOException; -import java.net.URI; -import java.nio.file.*; -import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; /** * Handle configuration for the Radar Server */ public class RadarServerConfig { - static ConcurrentHashMap fsproviders = new ConcurrentHashMap<>(); - static public List readXML(String filename) { List configs = new ArrayList<>(); @@ -61,7 +56,7 @@ static public List readXML(String filename) { Element meta = dataset.getChild("metadata", catNS); conf.name = dataset.getAttributeValue("name"); conf.urlPath = dataset.getAttributeValue("path"); - conf.dataPath = getPath(AliasTranslator.translateAlias(dataset.getAttributeValue("location"))); + conf.mFile = MFiles.create(AliasTranslator.translateAlias(dataset.getAttributeValue("location"))); conf.dataFormat = meta.getChild("dataFormat", catNS).getValue(); conf.stationFile = meta.getChild("stationFile", catNS).getAttributeValue("path"); conf.doc = meta.getChild("documentation", catNS).getValue(); @@ -88,60 +83,6 @@ static public List readXML(String filename) { return configs; } - private static Path getPath(String location) throws IOException { - FileSystem fs; - - // If we're given an actual URI, use that to find the file system. - // Otherwise, use the default. - if (location.contains(":")) { - URI uri = URI.create(location); - - // Fix parsing of s3:// (note two '/') style paths - if (uri.getPath().isEmpty()) { - uri = URI.create(location.replace("//", "///")); - } - - location = uri.getPath(); - fs = getFS(uri); - } else { - fs = FileSystems.getDefault(); - } - return fs.getPath(location); - } - - private static FileSystem getFS(URI uri) throws IOException { - FileSystem fs; - - try { - fs = getProvider(uri).getFileSystem(uri); - } catch (ProviderNotFoundException e) { - System.out.println("Provider not found: " + e.getMessage()); - System.out.println("Using default file system."); - fs = FileSystems.getDefault(); - } - return fs; - } - - // This is to work around the fact that when we *get* a filesystem, we - // cannot pass in the class loader. This results in custom providers (say - // S3) not being found. However, the filesystem already exists, so the - // filesystem can't be re-created either. - private static FileSystemProvider getProvider(URI uri) throws IOException { - if (fsproviders.containsKey(uri.getScheme())) { - return fsproviders.get(uri.getScheme()); - } else { - FileSystem fs; - try { - fs = FileSystems.newFileSystem(uri, new HashMap(), - Thread.currentThread().getContextClassLoader()); - } catch (FileSystemAlreadyExistsException e) { - fs = FileSystems.getFileSystem(uri); - } - fsproviders.put(uri.getScheme(), fs.provider()); - return fs.provider(); - } - } - protected static RadarConfigEntry.GeoInfo readGeospatialCoverage(Element gcElem) { if (gcElem == null) return null; @@ -232,7 +173,7 @@ protected static TimeDuration readDuration(Element elem) { } static public class RadarConfigEntry { - public Path dataPath; + public MFile mFile; public String name, urlPath, dataFormat, stationFile, doc; public String dateParseRegex, dateFmt, layout; public DateRange timeCoverage; diff --git a/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java b/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java index 5cc1dc197f..1f42e4e573 100644 --- a/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java +++ b/tds/src/main/java/thredds/server/radarServer2/RadarServerController.java @@ -148,7 +148,7 @@ public void init(ContextRefreshedEvent event) { List configs = RadarServerConfig.readXML(contentPath + "/radar/radarCollections.xml"); for (RadarServerConfig.RadarConfigEntry conf : configs) { - RadarDataInventory di = new RadarDataInventory(conf.dataPath, conf.crawlItems); + RadarDataInventory di = new RadarDataInventory(conf.mFile, conf.crawlItems); di.setName(conf.name); di.setDescription(conf.doc); @@ -495,14 +495,14 @@ private HttpEntity makeCatalog(String dataset, RadarDataInventory inv, R for (RadarDataInventory.Query.QueryResultItem i : res) { DatasetBuilder fileDB = new DatasetBuilder(mainDB); - fileDB.setName(i.file.getFileName().toString()); + fileDB.setName(i.file.getName()); fileDB.put(Dataset.Id, String.valueOf(i.file.hashCode())); fileDB.put(Dataset.Dates, new DateType(i.time.toString(), null, "start of ob", i.time.getCalendar())); // TODO: Does this need to be converted from the on-disk path // to a mapped url path? - fileDB.put(Dataset.UrlPath, inv.getCollectionDir().relativize(i.file).toString()); + fileDB.put(Dataset.UrlPath, inv.getCollectionDir().relativize(i.file)); mainDB.addDataset(fileDB); }