From 94d772773bcf89e6f4cc3da5a7a0b7ee9c471978 Mon Sep 17 00:00:00 2001 From: Furqaan Khan <46216254+furqaankhan@users.noreply.github.com> Date: Fri, 10 Nov 2023 02:11:14 -0500 Subject: [PATCH] [SEDONA-422] Add a feature in RS_SetBandNoDataValue and fix NoDataValue in RS_Clip (#1081) --- .../common/raster/RasterBandEditors.java | 35 +++++++++++++++--- .../common/raster/RasterBandEditorsTest.java | 36 +++++++++++-------- docs/api/sql/Raster-operators.md | 4 +-- .../apache/sedona/sql/rasteralgebraTest.scala | 7 ++-- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/raster/RasterBandEditors.java b/common/src/main/java/org/apache/sedona/common/raster/RasterBandEditors.java index da37429c7c..3c2f36be9a 100644 --- a/common/src/main/java/org/apache/sedona/common/raster/RasterBandEditors.java +++ b/common/src/main/java/org/apache/sedona/common/raster/RasterBandEditors.java @@ -40,10 +40,28 @@ import java.util.Collections; public class RasterBandEditors { - public static GridCoverage2D setBandNoDataValue(GridCoverage2D raster, int bandIndex, double noDataValue) { + /** + * Adds no-data value to the raster. + * @param raster Source raster to add no-data value + * @param bandIndex Band index to add no-data value + * @param noDataValue Value to set as no-data value, if null then remove existing no-data value + * @return Raster with no-data value + */ + public static GridCoverage2D setBandNoDataValue(GridCoverage2D raster, int bandIndex, Double noDataValue) { RasterUtils.ensureBand(raster, bandIndex); Double rasterNoData = RasterBandAccessors.getBandNoDataValue(raster, bandIndex); - if ( !(rasterNoData == null) && rasterNoData == noDataValue) { + + // Remove no-Data if it is null + if (noDataValue == null) { + if (RasterBandAccessors.getBandNoDataValue(raster) == null) { + return raster; + } + GridSampleDimension[] sampleDimensions = raster.getSampleDimensions(); + sampleDimensions [bandIndex - 1] = RasterUtils.removeNoDataValue(sampleDimensions[bandIndex - 1]); + return RasterUtils.create(raster.getRenderedImage(), raster.getGridGeometry(), sampleDimensions, null); + } + + if ( !(rasterNoData == null) && rasterNoData.equals(noDataValue)) { return raster; } GridSampleDimension[] bands = raster.getSampleDimensions(); @@ -60,7 +78,13 @@ public static GridCoverage2D setBandNoDataValue(GridCoverage2D raster, int bandI return RasterUtils.create(raster.getRenderedImage(), gridGeometry2D, bands, null); } - public static GridCoverage2D setBandNoDataValue(GridCoverage2D raster, double noDataValue) { + /** + * Adds no-data value to the raster. + * @param raster Source raster to add no-data value + * @param noDataValue Value to set as no-data value, if null then remove existing no-data value + * @return Raster with no-data value + */ + public static GridCoverage2D setBandNoDataValue(GridCoverage2D raster, Double noDataValue) { return setBandNoDataValue(raster, 1, noDataValue); } @@ -223,7 +247,10 @@ public static GridCoverage2D clip(GridCoverage2D raster, int band, Geometry geom } } } - newRaster = RasterUtils.create(resultRaster, raster.getGridGeometry(), newRaster.getSampleDimensions()); + newRaster = RasterUtils.create(resultRaster, raster.getGridGeometry(), newRaster.getSampleDimensions(), noDataValue); + } else { + // to add no-data value + newRaster = RasterUtils.create(newRaster.getRenderedImage(), newRaster.getGridGeometry(), newRaster.getSampleDimensions(), noDataValue); } return newRaster; diff --git a/common/src/test/java/org/apache/sedona/common/raster/RasterBandEditorsTest.java b/common/src/test/java/org/apache/sedona/common/raster/RasterBandEditorsTest.java index 0901593bca..362a99d322 100644 --- a/common/src/test/java/org/apache/sedona/common/raster/RasterBandEditorsTest.java +++ b/common/src/test/java/org/apache/sedona/common/raster/RasterBandEditorsTest.java @@ -33,31 +33,39 @@ import java.util.Arrays; import java.util.List; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class RasterBandEditorsTest extends RasterTestBase{ @Test public void testSetBandNoDataValueWithRaster() throws IOException { GridCoverage2D raster = rasterFromGeoTiff(resourceFolder + "raster/test1.tiff"); - GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(raster, 1,3); + GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(raster, 1,3d); double actual = RasterBandAccessors.getBandNoDataValue(grid); double expected = 3; assertEquals(expected, actual, 0.1d); assert(Arrays.equals(MapAlgebra.bandAsArray(raster, 1), MapAlgebra.bandAsArray(grid, 1))); - grid = RasterBandEditors.setBandNoDataValue(raster, -999); + grid = RasterBandEditors.setBandNoDataValue(raster, -999d); actual = RasterBandAccessors.getBandNoDataValue(grid); expected = -999; assertEquals(expected, actual, 0.1d); assert(Arrays.equals(MapAlgebra.bandAsArray(raster, 1), MapAlgebra.bandAsArray(grid, 1))); } + @Test + public void testSetBandNoDataValueWithNull() throws IOException { + GridCoverage2D raster = rasterFromGeoTiff(resourceFolder + "raster/raster_with_no_data/test5.tiff"); + GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(raster, 1,null); + String actual = Arrays.toString(grid.getSampleDimensions()); + String expected = "[RenderedSampleDimension[\"PALETTE_INDEX\"]]"; + assertEquals(expected, actual); + } + @Test public void testSetBandNoDataValueWithEmptyRaster() throws FactoryException { GridCoverage2D emptyRaster = RasterConstructors.makeEmptyRaster(1, 20, 20, 0, 0, 8, 8, 0.1, 0.1, 4326); - GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(emptyRaster, 1, 999); + GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(emptyRaster, 1, 999d); double actual = RasterBandAccessors.getBandNoDataValue(grid); double expected = 999; assertEquals(expected, actual, 0.1d); @@ -71,8 +79,8 @@ public void testSetBandNoDataValueWithEmptyRaster() throws FactoryException { @Test public void testSetBandNoDataValueWithEmptyRasterMultipleBand() throws FactoryException { GridCoverage2D emptyRaster = RasterConstructors.makeEmptyRaster(2, 20, 20, 0, 0, 8, 8, 0.1, 0.1, 0); - GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(emptyRaster, -9999); - grid = RasterBandEditors.setBandNoDataValue(grid, 2, 444); + GridCoverage2D grid = RasterBandEditors.setBandNoDataValue(emptyRaster, -9999d); + grid = RasterBandEditors.setBandNoDataValue(grid, 2, 444d); assertEquals(-9999, (double) RasterBandAccessors.getBandNoDataValue(grid), 0.1d); assertEquals(444, (double) RasterBandAccessors.getBandNoDataValue(grid, 2), 0.1d); } @@ -89,7 +97,7 @@ public void testClip() throws IOException, FactoryException, TransformException, assertArrayEquals(originalMetadata, clippedMetadata, 0.01d); String actual = String.valueOf(clippedRaster.getSampleDimensions()[0]); - String expected = String.valueOf(raster.getSampleDimensions()[0]); + String expected = "RenderedSampleDimension(\"RED_BAND\":[200.0 ... 200.0])\n ‣ Category(\"No data\":[200...200])\n"; assertEquals(expected, actual); List points = new ArrayList<>(); @@ -98,9 +106,9 @@ public void testClip() throws IOException, FactoryException, TransformException, points.add(Constructors.geomFromWKT("POINT(237201 4.20429e+06)", 26918)); points.add(Constructors.geomFromWKT("POINT(237919 4.20357e+06)", 26918)); points.add(Constructors.geomFromWKT("POINT(254668 4.21769e+06)", 26918)); - double[] actualValues = PixelFunctions.values(clippedRaster, points, 1).stream().mapToDouble(d -> d).toArray(); - double[] expectedValues = new double[] {200.0, 200.0, 0.0, 0.0, 200.0}; - assertArrayEquals(expectedValues, actualValues, 0.001d); + Double[] actualValues = PixelFunctions.values(clippedRaster, points, 1).toArray(new Double[0]); + Double[] expectedValues = new Double[] {null, null, 0.0, 0.0, null}; + assertTrue(Arrays.equals(expectedValues, actualValues)); GridCoverage2D croppedRaster = RasterBandEditors.clip(raster, 1, geom, 200, true); points = new ArrayList<>(); @@ -109,9 +117,9 @@ public void testClip() throws IOException, FactoryException, TransformException, points.add(Constructors.geomFromWKT("POINT(237201 4.20429e+06)", 26918)); points.add(Constructors.geomFromWKT("POINT(237919 4.20357e+06)", 26918)); points.add(Constructors.geomFromWKT("POINT(223802 4.20465e+06)", 26918)); - actualValues = PixelFunctions.values(croppedRaster, points, 1).stream().mapToDouble(d -> d).toArray(); - expectedValues = new double[] {0.0, 0.0, 0.0, 0.0, 200.0}; - assertArrayEquals(expectedValues, actualValues, 0.001d); + actualValues = PixelFunctions.values(croppedRaster, points, 1).toArray(new Double[0]); + expectedValues = new Double[] {0.0, 0.0, 0.0, 0.0, null}; + assertTrue(Arrays.equals(expectedValues, actualValues)); } @Test diff --git a/docs/api/sql/Raster-operators.md b/docs/api/sql/Raster-operators.md index 02ee063b3b..8eef3a2bea 100644 --- a/docs/api/sql/Raster-operators.md +++ b/docs/api/sql/Raster-operators.md @@ -1069,8 +1069,6 @@ Introduction: Returns a raster that is clipped by the given geometry. If `crop` is not specified then it will default to `true`, meaning it will make the resulting raster shrink to the geometry's extent and if `noDataValue` is not specified then the resulting raster will have the minimum possible value for the band pixel data type. -!!!note - When `crop` is `true`, no new raster is created - instead a viewpoint to the source raster is returned, and when `false`, a new raster is generated. Format: @@ -1291,7 +1289,7 @@ Output: ### RS_SetBandNoDataValue -Introduction: Sets no data value for given band. If band index not specified then band 1 is assumed. +Introduction: This sets the no data value for a specified band in the raster. If the band index is not provided, band 1 is assumed by default. Passing a `null` value for `noDataValue` will remove the no data value and that will ensure all pixels are included in functions rather than excluded as no data. Format: `RS_SetBandNoDataValue(raster: Raster, bandIndex: Integer = 1, noDataValue: Double)` diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala index eb37f14258..6e77a1ba74 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala @@ -451,6 +451,9 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen val actual = df.selectExpr("RS_BandNoDataValue(RS_SetBandNoDataValue(raster, 1, -999))").first().getDouble(0) val expected = -999 assertEquals(expected, actual, 0.001d) + + val actualNull = df.selectExpr("RS_BandNoDataValue(RS_SetBandNoDataValue(raster, 1, null))").first().get(0) + assertNull(actualNull) } it("Passed RS_SetBandNoDataValue with empty raster") { @@ -540,7 +543,7 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen "ST_GeomFromWKT('POINT(237201 4.20429e+06)'),ST_GeomFromWKT('POINT(237919 4.20357e+06)')," + "ST_GeomFromWKT('POINT(254668 4.21769e+06)')), 1)" ).first().get(0) - var expectedValues = Seq(200.0, 200.0, 0.0, 0.0, 200.0) + var expectedValues = Seq(null, null, 0.0, 0.0, null) assertTrue(expectedValues.equals(actualValues)) val croppedDf = df.selectExpr("RS_Clip(raster, 1, geom, 200, false) as cropped") @@ -550,7 +553,7 @@ class rasteralgebraTest extends TestBaseScala with BeforeAndAfter with GivenWhen "ST_GeomFromWKT('POINT(237201 4.20429e+06)'),ST_GeomFromWKT('POINT(237919 4.20357e+06)')," + "ST_GeomFromWKT('POINT(223802 4.20465e+06)')), 1)" ).first().get(0) - expectedValues = Seq(0.0, 0.0, 0.0, 0.0, 200.0) + expectedValues = Seq(0.0, 0.0, 0.0, 0.0, null) assertTrue(expectedValues.equals(actualValues)) }