Merge pull request #336 from j3r3m1/lczTest0

Lcz test0

.gitignore

@@ -16,4 +16,5 @@ myh2gisbdtopodb.mv.db
 
 site.xml
 
-bdtopofolder
+dataForTests
+lczDebug.json

bdtopo_v2/src/main/groovy/org/orbisgis/orbisprocess/geoclimate/bdtopo_v2/WorkflowBDTopo_V2.groovy

@@ -76,9 +76,9 @@ import java.sql.SQLException
  *         "mapOfWeights":
  *         {"sky_view_factor": 1,
  *             "aspect_ratio": 1,
- *             "building_surface_fraction": 1,
- *             "impervious_surface_fraction" : 1,
- *             "pervious_surface_fraction": 1,
+ *             "building_surface_fraction": 2,
+ *             "impervious_surface_fraction" : 0,
+ *             "pervious_surface_fraction": 0,
  *             "height_of_roughness_elements": 1,
  *             "terrain_roughness_length": 1},
  *         "hLevMin": 3,
@@ -805,8 +805,8 @@ def findIDZones(def h2gis_datasource, def id_zones){
 def extractProcessingParameters(def processing_parameters){
     def defaultParameters = [distance: 1000,indicatorUse: ["LCZ", "URBAN_TYPOLOGY", "TEB"],
                              svfSimplified:false, prefixName: "",
-                             mapOfWeights : ["sky_view_factor" : 1, "aspect_ratio": 1, "building_surface_fraction": 1,
-                                             "impervious_surface_fraction" : 1, "pervious_surface_fraction": 1,
+                             mapOfWeights : ["sky_view_factor" : 1, "aspect_ratio": 1, "building_surface_fraction": 2,
+                                             "impervious_surface_fraction" : 0, "pervious_surface_fraction": 0,
                                              "height_of_roughness_elements": 1, "terrain_roughness_length": 1],
                              hLevMin : 3, hLevMax: 15, hThresholdLev2: 10]
     if(processing_parameters){
@@ -1554,8 +1554,8 @@ def GeoIndicators() {
                 roadTable: String, railTable: String, vegetationTable: String,
                 hydrographicTable: String, surface_vegetation: 100000, surface_hydro: 2500,
                 distance: 0.01, indicatorUse: ["LCZ", "URBAN_TYPOLOGY", "TEB"], svfSimplified:false, prefixName: "",
-                mapOfWeights: ["sky_view_factor" : 1, "aspect_ratio": 1, "building_surface_fraction": 1,
-                               "impervious_surface_fraction" : 1, "pervious_surface_fraction": 1,
+                mapOfWeights: ["sky_view_factor" : 1, "aspect_ratio": 1, "building_surface_fraction": 2,
+                               "impervious_surface_fraction" : 0, "pervious_surface_fraction": 0,
                                "height_of_roughness_elements": 1, "terrain_roughness_length": 1]
         outputs outputTableBuildingIndicators: String, outputTableBlockIndicators: String,
                 outputTableRsuIndicators: String, outputTableRsuLcz:String, outputTableZone:String

bdtopo_v2/src/test/groovy/org/orbisgis/orbisprocess/geoclimate/bdtopo_v2/ProcessingChainBDTopoTest.groovy

@@ -15,9 +15,11 @@ import static org.junit.jupiter.api.Assertions.*
 
 class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
 
+    public static communeToTest = "12174"
+
     public static Logger logger = LoggerFactory.getLogger(ProcessingChainBDTopoTest.class)
     public static def h2db = "./target/myh2gisbdtopodb"
-    public static def bdtopoFoldName = "bdtopofolder"
+    public static def bdtopoFoldName = "processingChain/dataForTests"
     public static def listTables = ["IRIS_GE", "BATI_INDIFFERENCIE", "BATI_INDUSTRIEL", "BATI_REMARQUABLE",
     "ROUTE", "SURFACE_EAU", "ZONE_VEGETATION", "TRONCON_VOIE_FERREE", "TERRAIN_SPORT", "CONSTRUCTION_SURFACIQUE",
     "SURFACE_ROUTE", "SURFACE_ACTIVITE"]
@@ -44,17 +46,17 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
 
         // Load parameter files
         paramTables.each{
-            h2GISDatabase.load(ProcessingChain.class.getResource(it+".csv"), it, true)
+            h2GISDatabase.load(getClass().getResource(it+".csv"), it, true)
         }
 
         def relativePath = bdtopoFoldName + File.separator + inseeCode
 
         // Test whether there is a folder containing .shp files for the corresponding INSEE code
-        if(ProcessingChain.class.getResource(relativePath)){
+        if(getClass().getResource(relativePath)){
             // Test is the URL is a folder
-            if(new File(ProcessingChain.class.getResource(relativePath).toURI()).isDirectory()){
+            if(new File(getClass().getResource(relativePath).toURI()).isDirectory()){
                 listTables.each {
-                    def filePath = ProcessingChain.class.getResource(relativePath + File.separator + it + ".shp")
+                    def filePath = getClass().getResource(relativePath + File.separator + it + ".shp")
                     // If some layers are missing, do not try to load them...
                     if (filePath) {
                         h2GISDatabase.load(filePath, it, true)
@@ -79,7 +81,7 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
     @DisabledIfSystemProperty(named = "data.bd.topo", matches = "false")
     void prepareBDTopoTest(){
         def dbSuffixName = "_prepare"
-        def inseeCode = "01306"
+        def inseeCode = communeToTest
         H2GIS h2GISDatabase = loadFiles(inseeCode, dbSuffixName)
         def process = BDTopo_V2.prepareData
         assertTrue process.execute([datasource: h2GISDatabase,
@@ -89,7 +91,7 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
                                     tableHydroName: 'SURFACE_EAU', tableVegetName: 'ZONE_VEGETATION',
                                     tableImperviousSportName: 'TERRAIN_SPORT', tableImperviousBuildSurfName: 'CONSTRUCTION_SURFACIQUE',
                                     tableImperviousRoadSurfName: 'SURFACE_ROUTE', tableImperviousActivSurfName: 'SURFACE_ACTIVITE',
-                                    distBuffer: 500, expand: 1000, idZone: '56260',
+                                    distBuffer: 500, expand: 1000, idZone: communeToTest,
                                     hLevMin: 3, hLevMax : 15, hThresholdLev2 : 10
         ])
         process.getResults().each {entry ->
@@ -139,17 +141,16 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
     @DisabledIfSystemProperty(named = "data.bd.topo", matches = "false")
     void bdtopoLczFromTestFiles() {
         def dbSuffixName = "_lcz"
-        String inseeCode = "01306"
-        H2GIS datasource = loadFiles(inseeCode, dbSuffixName)
-        def process = ProcessingChain.PrepareBDTopo.prepareBDTopo()
+        H2GIS datasource = loadFiles(communeToTest, dbSuffixName)
+        def process = BDTopo_V2.prepareData
         assertTrue process.execute([datasource: datasource,
                                     tableIrisName: 'IRIS_GE', tableBuildIndifName: 'BATI_INDIFFERENCIE',
                                     tableBuildIndusName: 'BATI_INDUSTRIEL', tableBuildRemarqName: 'BATI_REMARQUABLE',
                                     tableRoadName: 'ROUTE', tableRailName: 'TRONCON_VOIE_FERREE',
                                     tableHydroName: 'SURFACE_EAU', tableVegetName: 'ZONE_VEGETATION',
                                     tableImperviousSportName: 'TERRAIN_SPORT', tableImperviousBuildSurfName: 'CONSTRUCTION_SURFACIQUE',
                                     tableImperviousRoadSurfName: 'SURFACE_ROUTE', tableImperviousActivSurfName: 'SURFACE_ACTIVITE',
-                                    distBuffer: 500, expand: 1000, idZone: inseeCode,
+                                    distBuffer: 500, expand: 1000, idZone: communeToTest,
                                     hLevMin: 3, hLevMax : 15, hThresholdLev2 : 10
         ])
         def abstractTables = process.getResults()
@@ -183,8 +184,7 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
     @DisabledIfSystemProperty(named = "data.bd.topo", matches = "false")
     void bdtopoGeoIndicatorsFromTestFiles() {
         def dbSuffixName = "_geoIndicators"
-        String inseeCode = "01306"
-        H2GIS h2GISDatabase = loadFiles(inseeCode, dbSuffixName)
+        H2GIS h2GISDatabase = loadFiles(communeToTest, dbSuffixName)
         def process = BDTopo_V2.prepareData
         assertTrue process.execute([datasource: h2GISDatabase,
                                     tableIrisName: 'IRIS_GE', tableBuildIndifName: 'BATI_INDIFFERENCIE',
@@ -193,7 +193,7 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
                                     tableHydroName: 'SURFACE_EAU', tableVegetName: 'ZONE_VEGETATION',
                                     tableImperviousSportName: 'TERRAIN_SPORT', tableImperviousBuildSurfName: 'CONSTRUCTION_SURFACIQUE',
                                     tableImperviousRoadSurfName: 'SURFACE_ROUTE', tableImperviousActivSurfName: 'SURFACE_ACTIVITE',
-                                    distBuffer: 500, expand: 1000, idZone: inseeCode,
+                                    distBuffer: 500, expand: 1000, idZone: communeToTest,
                                     hLevMin: 3, hLevMax : 15, hThresholdLev2 : 10
         ])
         def abstractTables = process.getResults()
@@ -219,14 +219,13 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
     @Disabled
     void testBDTOPO_V2Workflow() {
         String directory ="./target/geoclimate_chain/bdtopo_config/"
-        def inseeCode = "01306"
         File dirFile = new File(directory)
         dirFile.delete()
         dirFile.mkdir()
         IProcess processBDTopo = BDTopo_V2.workflow
         assertTrue(processBDTopo.execute(configurationFile: getClass().getResource("config/bdtopo_workflow_folderinput_folderoutput_id_zones.json").toURI()))mm
         assertNotNull(processBDTopo.getResults().outputFolder)
-        def baseNamePathAndFileOut = processBDTopo.getResults().outputFolder + File.separator + "zone_" + inseeCode + "_"
+        def baseNamePathAndFileOut = processBDTopo.getResults().outputFolder + File.separator + "zone_" + communeToTest + "_"
         assertTrue(new File(baseNamePathAndFileOut + "rsu_indicators.geojson").exists())
         assertFalse(new File(baseNamePathAndFileOut + "rsu_lcz.geojson").exists())
         assertFalse(new File(baseNamePathAndFileOut + "block_indicators.geojson").exists())
@@ -244,16 +243,6 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
         assertTrue(process.execute(configurationFile: configFile))
     }
 
-    @Test //Integration tests
-    @Disabled
-    void lczProduceTestData() {
-        def configFile = getClass().getResource("bdtopofolder/lczTests/lczDebug.json").toURI()
-
-        // Execute several cities where we have tests
-        IProcess process = BDTopo_V2.workflow
-        assertTrue(process.execute(configurationFile: configFile))
-    }
-
     @Disabled
     @Test
     void workflowFolderToDatabase() {
@@ -268,7 +257,7 @@ class ProcessingChainBDTopoTest extends ChainProcessAbstractTest{
                         "delete" :true
                 ],
                 "input" : [
-                        "folder": ["path" :".../bdtopofolder",
+                        "folder": ["path" :".../processingChain",
                             "id_zones":["-", "-"]]],
                 "output" :[
                         "database" :

geoindicators/src/main/groovy/org/orbisgis/orbisprocess/geoclimate/geoindicators/RsuIndicators.groovy

@@ -1165,7 +1165,8 @@ IProcess smallestCommunGeometry() {
                 CREATE INDEX IF NOT EXISTS ids_$roadTable_zindex0_buffer ON $roadTable_zindex0_buffer USING RTREE(the_geom);
                 CREATE TABLE $road_tmp AS SELECT ST_CollectionExtract(st_intersection(st_union(st_accum(a.the_geom)),b.the_geom),3) AS the_geom, b.id_rsu FROM
                 $roadTable_zindex0_buffer AS a, $rsuTable AS b WHERE a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom) GROUP BY b.id_rsu;
-                DROP TABLE IF EXISTS $roadTable_zindex0_buffer;"""
+                DROP TABLE IF EXISTS $roadTable_zindex0_buffer;
+                """
                     tablesToMerge+= ["$road_tmp": "select ST_ToMultiLine(the_geom) as the_geom, id_rsu from $road_tmp WHERE ST_ISEMPTY(THE_GEOM)=false"]
                 }
 
@@ -1403,14 +1404,17 @@ IProcess surfaceFractions() {
                         i += 1
             }
 
+
             def query = """DROP TABLE IF EXISTS $outputTableName; CREATE TABLE $outputTableName AS SELECT a.ID_RSU """
             def end_query = """ FROM $spatialRelationsTable AS a RIGHT JOIN $rsuTable b 
                                 ON a.ID_RSU=b.ID_RSU GROUP BY b.ID_RSU;"""
             // Calculates the fraction of overlapped layers according to "superpositionsWithPriorities"
             superpositions.each{key, values ->
                 // Calculating the overlaying layer when it has no overlapped layer
                 def tempoLayers = LAYERS.minus([key])
-                query += ", SUM(CASE WHEN a.$key =1 AND a.${tempoLayers.join(" =0 AND a.")} =0 THEN a.area ELSE 0 END)/st_area(b.the_geom) AS ${key}_fraction "
+
+                query += ", COALESCE(SUM(CASE WHEN a.$key =1 AND a.${tempoLayers.join(" =0 AND a.")} =0 THEN a.area ELSE 0 END),0)/st_area(b.the_geom) AS ${key}_fraction "
+
                 // Calculate each combination of overlapped layer for the current overlaying layer
                 def notOverlappedLayers = priorities.minus(values).minus([key])
                 // If an non overlapped layer is prioritized, its number should be 0 for the overlapping to happen
@@ -1431,7 +1435,8 @@ IProcess surfaceFractions() {
                         if(!var2Zero.isEmpty()){
                             var2ZeroQuery = " AND a." + var2Zero.join("=0 AND a.") + " =0 "
                         }
-                        query += ", SUM(CASE WHEN a.$key =1 AND a.$val =1 $var2ZeroQuery $nonOverlappedQuery THEN a.area ELSE 0 END)/st_area(b.the_geom) AS ${key}_${val}_fraction "
+                        query += ", COALESCE(SUM(CASE WHEN a.$key =1 AND a.$val =1 $var2ZeroQuery $nonOverlappedQuery THEN a.area ELSE 0 END),0)/st_area(b.the_geom) AS ${key}_${val}_fraction "
+
                     }
                     var2Zero.add(val)
                 }
@@ -1458,7 +1463,9 @@ IProcess surfaceFractions() {
                             nonOverlappedQuery += " AND a.$key =0 "
                         }
                     }
-                    query += ", SUM(CASE WHEN a.$val =1 $var2ZeroQuery $varAlreadyUsedQuery $nonOverlappedQuery THEN a.area ELSE 0 END)/st_area(b.the_geom) AS ${val}_fraction "
+
+                    query += ", COALESCE(SUM(CASE WHEN a.$val =1 $var2ZeroQuery $varAlreadyUsedQuery $nonOverlappedQuery THEN a.area ELSE 0 END),0)/st_area(b.the_geom) AS ${val}_fraction "
+
                 }
             }
             datasource.execute query + end_query

geoindicators/src/main/groovy/org/orbisgis/orbisprocess/geoclimate/geoindicators/TypologyClassification.groovy

@@ -93,6 +93,7 @@ IProcess identifyLczType() {
                 def classifiedUrbanLcz = "classifiedUrbanLcz$uuid"
                 def classifiedLcz = "classifiedLcz$uuid"
 
+
                 // The LCZ definitions are created in a Table of the DataBase (note that the "terrain_roughness_class"
                 // is replaced by the "terrain_roughness_length" in order to have a continuous interval of values)
                 datasource.execute "DROP TABLE IF EXISTS $LCZ_classes; " +
@@ -128,6 +129,7 @@ IProcess identifyLczType() {
                 // I. Rural LCZ types are classified according to a "manual" decision tree
                 datasource.getTable(rsuAllIndicators).BUILDING_FRACTION_LCZ.createIndex()
                 datasource.getTable(rsuAllIndicators).ASPECT_RATIO.createIndex()
+
                 datasource.execute """DROP TABLE IF EXISTS $ruralLCZ; 
                                     CREATE TABLE $ruralLCZ 
                                             AS SELECT   $ID_FIELD_RSU, IMPERVIOUS_FRACTION_LCZ, 
@@ -139,7 +141,7 @@ IProcess identifyLczType() {
                                                          AS HIGH_ALL_VEGETATION,
                                                         LOW_VEGETATION_FRACTION_LCZ+HIGH_VEGETATION_FRACTION_LCZ AS ALL_VEGETATION
                                             FROM        $rsuAllIndicators
-                                            WHERE       BUILDING_FRACTION_LCZ < 0.1 AND ASPECT_RATIO < 0.1;"""
+                                            WHERE       (BUILDING_FRACTION_LCZ < 0.1 OR BUILDING_FRACTION_LCZ IS NULL) AND (ASPECT_RATIO < 0.1 OR ASPECT_RATIO IS NULL);"""
 
                 datasource.getTable(ruralLCZ).IMPERVIOUS_FRACTION_LCZ.createIndex()
                 datasource.getTable(ruralLCZ).PERVIOUS_FRACTION_LCZ.createIndex()
@@ -152,11 +154,13 @@ IProcess identifyLczType() {
                                                                 THEN 105
                                                                 ELSE CASE WHEN ALL_VEGETATION<WATER_FRACTION_LCZ
                                                                         THEN 107
-                                                                        ELSE CASE WHEN HIGH_ALL_VEGETATION<0.1
-                                                                                THEN 104
-                                                                                ELSE CASE WHEN HIGH_ALL_VEGETATION<0.75
-                                                                                        THEN 102
-                                                                                        ELSE 101 END END END END AS LCZ1,
+                                                                        ELSE CASE WHEN HIGH_ALL_VEGETATION IS NULL
+                                                                                THEN 999
+                                                                                ELSE CASE WHEN HIGH_ALL_VEGETATION<0.1
+                                                                                        THEN 104
+                                                                                        ELSE CASE WHEN HIGH_ALL_VEGETATION<0.75
+                                                                                                THEN 102
+                                                                                                ELSE 101 END END END END END AS LCZ1,
                                                         null AS LCZ2, null AS min_distance, null AS PSS 
                                             FROM $ruralLCZ"""
 

geoindicators/src/test/groovy/org/orbisgis/orbisprocess/geoclimate/geoindicators/BuildingIndicatorsTests.groovy

@@ -14,7 +14,7 @@ class BuildingIndicatorsTests {
 
     @BeforeAll
     static void init(){
-        h2GIS = H2GIS.open('./target/buildingdb;AUTO_SERVER=TRUE')
+        h2GIS = H2GIS.open("./target/${BuildingIndicatorsTests.getName()};AUTO_SERVER=TRUE")
     }
 
     @BeforeEach

geoindicators/src/test/groovy/org/orbisgis/orbisprocess/geoclimate/geoindicators/GenericIndicatorsTests.groovy

@@ -14,7 +14,7 @@ class GenericIndicatorsTests {
 
     @BeforeAll
     static void init(){
-        h2GIS = H2GIS.open('./target/buildingdb;AUTO_SERVER=TRUE')
+        h2GIS = H2GIS.open("./target/${GenericIndicatorsTests.getName()};AUTO_SERVER=TRUE")
     }
 
     @BeforeEach

geoindicators/src/test/groovy/org/orbisgis/orbisprocess/geoclimate/geoindicators/RsuIndicatorsTests.groovy

@@ -397,7 +397,7 @@ class RsuIndicatorsTests {
 
     @Test
     void surfaceFractionTest() {
-        // Only the RSU 5 is conserved for the test
+        // Only the RSU 4 is conserved for the test
         h2GIS.execute "DROP TABLE IF EXISTS rsu_tempo;" +
                 "CREATE TABLE rsu_tempo AS SELECT * " +
                 "FROM rsu_test WHERE id_rsu = 4"
@@ -463,7 +463,6 @@ class RsuIndicatorsTests {
         assertEquals(0, result2["building_fraction"])
     }
 
-
     @Test
     void surfaceFractionTest3() {
         // Test whether the road fraction is taken into account...
@@ -480,8 +479,6 @@ class RsuIndicatorsTests {
                 prefixName: "test", datasource: h2GIS])
         def tempoTable = p.results.outputTableName
 
-        println h2GIS.firstRow("SELECT * FROM $tempoTable")
-
         // Apply the surface fractions for different combinations
         // combination 1
         def  p0 =  Geoindicators.RsuIndicators.surfaceFractions()