enable subsetting access model processing by test watershed groups

smnorris · Jan 8, 2024 · 71de82b · 71de82b
1 parent bf80608
commit 71de82b
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 16 deletions.
diff --git a/db/tables/parameters.sql b/db/tables/parameters.sql
@@ -30,3 +30,11 @@ create table bcfishpass.parameters_habitat_thresholds (
   rear_mad_max numeric,
   rear_lake_ha_min integer
 );
+
+
+-- --------------
+-- list of watershed groups to process in access model/crossings
+-- defaults to all
+-- --------------
+drop table if exists bcfishpass.watershed_groups_access;
+create table bcfishpass.watershed_groups_access as select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly;
diff --git a/model/01_access/Makefile b/model/01_access/Makefile
@@ -3,10 +3,8 @@
 
 PSQL=psql $(DATABASE_URL) -v ON_ERROR_STOP=1          # point psql to db and stop on errors
 
-# process all watershed groups
-WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly")
-# test with just watershed groups noted in param (misses cross-wsg barreriers)
-#WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.parameters_habitat_method")
+# watershed groups to process are defined by bcfishpass.watershed_groups_access
+WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access")
 
 # define barrier table targets
 NATURAL_BARRIERS = .make/barriers_falls \
@@ -23,17 +21,19 @@ ANTH_BARRIERS = .make/barriers_anthropogenic \
 	.make/barriers_dams_hydro \
 	.make/barriers_pscis
 
-# SOURCE_BARRIERS = $(patsubst sql/%.sql, .make/%, $(wildcard sql/barriers_*.sql))
-
 # access models - combine barriers into tables of per-species scenarios
 ACCESS_MODELS = $(patsubst sql/%.sql, .make/%, $(wildcard sql/model_access_*.sql))
 
-
 all: .make/qa
 
 clean: 
 	rm -Rf .make
 
+# for access model processing that is per-watershed group, use just watershed groups noted in parameters file
+# (for testing only)
+test:
+	$(PSQL) -c "truncate bcfishpass.watershed_groups_access"
+	$(PSQL) -c "insert into bcfishpass.watershed_groups_access select watershed_group_code from bcfishpass.parameters_habitat_method"
 
 # ------
 # FALLS

diff --git a/model/01_access/gradient_barriers/gradient_barriers.sh b/model/01_access/gradient_barriers/gradient_barriers.sh
@@ -2,7 +2,7 @@
 set -euxo pipefail
 
 PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1"
-WSGS=$($PSQL -AXt -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly")
+WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access")
 
 # create table
 $PSQL -f sql/gradient_barriers.sql

diff --git a/model/01_access/modelled_stream_crossings/Makefile b/model/01_access/modelled_stream_crossings/Makefile
@@ -1,7 +1,5 @@
 PSQL = psql $(DATABASE_URL) -v ON_ERROR_STOP=1 # Ensure psql stops on error
-GROUPS = $(shell psql $(DATABASE_URL) -AtX -c "SELECT watershed_group_code \
-	FROM whse_basemapping.fwa_watershed_groups_poly \
-	ORDER BY watershed_group_code")
+WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access")
 
 # download latest from s3
 .make/download_archive:
@@ -82,11 +80,11 @@ GROUPS = $(shell psql $(DATABASE_URL) -AtX -c "SELECT watershed_group_code \
 	$(PSQL) -f sql/01_create_output_table.sql
 
 	# load preliminary crossings, iterating through watershed groups for each data source
-	parallel $(PSQL) -f sql/02_intersect_dra.sql -v wsg={1} ::: $(GROUPS)
-	parallel $(PSQL) -f sql/03_intersect_ften.sql -v wsg={1} ::: $(GROUPS)
-	parallel $(PSQL) -f sql/04_intersect_ogc.sql -v wsg={1} ::: $(GROUPS)
-	parallel $(PSQL) -f sql/05_intersect_ogcpre06.sql -v wsg={1} ::: $(GROUPS)
-	parallel $(PSQL) -f sql/06_intersect_railway.sql -v wsg={1} ::: $(GROUPS)
+	parallel $(PSQL) -f sql/02_intersect_dra.sql -v wsg={1} ::: $(WSGS)
+	parallel $(PSQL) -f sql/03_intersect_ften.sql -v wsg={1} ::: $(WSGS)
+	parallel $(PSQL) -f sql/04_intersect_ogc.sql -v wsg={1} ::: $(WSGS)
+	parallel $(PSQL) -f sql/05_intersect_ogcpre06.sql -v wsg={1} ::: $(WSGS)
+	parallel $(PSQL) -f sql/06_intersect_railway.sql -v wsg={1} ::: $(WSGS)
 
 	# remove duplicate crossings introduced by using multiple sources
 	$(PSQL) -f sql/07_remove_duplicates.sql