From 71de82b65a414661bc97233d04d4ef5054bd84e7 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Mon, 8 Jan 2024 15:09:45 -0800 Subject: [PATCH] enable subsetting access model processing by test watershed groups --- db/tables/parameters.sql | 8 ++++++++ model/01_access/Makefile | 14 +++++++------- .../gradient_barriers/gradient_barriers.sh | 2 +- model/01_access/modelled_stream_crossings/Makefile | 14 ++++++-------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/db/tables/parameters.sql b/db/tables/parameters.sql index 5cf3253f..89e2ddf6 100644 --- a/db/tables/parameters.sql +++ b/db/tables/parameters.sql @@ -30,3 +30,11 @@ create table bcfishpass.parameters_habitat_thresholds ( rear_mad_max numeric, rear_lake_ha_min integer ); + + +-- -------------- +-- list of watershed groups to process in access model/crossings +-- defaults to all +-- -------------- +drop table if exists bcfishpass.watershed_groups_access; +create table bcfishpass.watershed_groups_access as select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly; \ No newline at end of file diff --git a/model/01_access/Makefile b/model/01_access/Makefile index c88de5f1..00c2516d 100644 --- a/model/01_access/Makefile +++ b/model/01_access/Makefile @@ -3,10 +3,8 @@ PSQL=psql $(DATABASE_URL) -v ON_ERROR_STOP=1 # point psql to db and stop on errors -# process all watershed groups -WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly") -# test with just watershed groups noted in param (misses cross-wsg barreriers) -#WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.parameters_habitat_method") +# watershed groups to process are defined by bcfishpass.watershed_groups_access +WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access") # define barrier table targets NATURAL_BARRIERS = .make/barriers_falls \ @@ -23,17 +21,19 @@ ANTH_BARRIERS = .make/barriers_anthropogenic \ .make/barriers_dams_hydro \ .make/barriers_pscis -# SOURCE_BARRIERS = $(patsubst sql/%.sql, .make/%, $(wildcard sql/barriers_*.sql)) - # access models - combine barriers into tables of per-species scenarios ACCESS_MODELS = $(patsubst sql/%.sql, .make/%, $(wildcard sql/model_access_*.sql)) - all: .make/qa clean: rm -Rf .make +# for access model processing that is per-watershed group, use just watershed groups noted in parameters file +# (for testing only) +test: + $(PSQL) -c "truncate bcfishpass.watershed_groups_access" + $(PSQL) -c "insert into bcfishpass.watershed_groups_access select watershed_group_code from bcfishpass.parameters_habitat_method" # ------ # FALLS diff --git a/model/01_access/gradient_barriers/gradient_barriers.sh b/model/01_access/gradient_barriers/gradient_barriers.sh index c98e8295..158c3c53 100755 --- a/model/01_access/gradient_barriers/gradient_barriers.sh +++ b/model/01_access/gradient_barriers/gradient_barriers.sh @@ -2,7 +2,7 @@ set -euxo pipefail PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" -WSGS=$($PSQL -AXt -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly") +WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access") # create table $PSQL -f sql/gradient_barriers.sql diff --git a/model/01_access/modelled_stream_crossings/Makefile b/model/01_access/modelled_stream_crossings/Makefile index 8cef2050..49fbd1d7 100644 --- a/model/01_access/modelled_stream_crossings/Makefile +++ b/model/01_access/modelled_stream_crossings/Makefile @@ -1,7 +1,5 @@ PSQL = psql $(DATABASE_URL) -v ON_ERROR_STOP=1 # Ensure psql stops on error -GROUPS = $(shell psql $(DATABASE_URL) -AtX -c "SELECT watershed_group_code \ - FROM whse_basemapping.fwa_watershed_groups_poly \ - ORDER BY watershed_group_code") +WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access") # download latest from s3 .make/download_archive: @@ -82,11 +80,11 @@ GROUPS = $(shell psql $(DATABASE_URL) -AtX -c "SELECT watershed_group_code \ $(PSQL) -f sql/01_create_output_table.sql # load preliminary crossings, iterating through watershed groups for each data source - parallel $(PSQL) -f sql/02_intersect_dra.sql -v wsg={1} ::: $(GROUPS) - parallel $(PSQL) -f sql/03_intersect_ften.sql -v wsg={1} ::: $(GROUPS) - parallel $(PSQL) -f sql/04_intersect_ogc.sql -v wsg={1} ::: $(GROUPS) - parallel $(PSQL) -f sql/05_intersect_ogcpre06.sql -v wsg={1} ::: $(GROUPS) - parallel $(PSQL) -f sql/06_intersect_railway.sql -v wsg={1} ::: $(GROUPS) + parallel $(PSQL) -f sql/02_intersect_dra.sql -v wsg={1} ::: $(WSGS) + parallel $(PSQL) -f sql/03_intersect_ften.sql -v wsg={1} ::: $(WSGS) + parallel $(PSQL) -f sql/04_intersect_ogc.sql -v wsg={1} ::: $(WSGS) + parallel $(PSQL) -f sql/05_intersect_ogcpre06.sql -v wsg={1} ::: $(WSGS) + parallel $(PSQL) -f sql/06_intersect_railway.sql -v wsg={1} ::: $(WSGS) # remove duplicate crossings introduced by using multiple sources $(PSQL) -f sql/07_remove_duplicates.sql