No more phase 1 (#1587)

- Implements logic to request all dailies from WFWX and marshall results into `NoonForecast`s - Only persists records from WFWX that have `recordType.id` set as `FORECAST` - Refactor `bcfw_p1*` to `wfwx*` and remove `bcfw_p1` test fixture artifacts - Refactor `bot` -> `job` since they are cronjobs and make this consistent in codebase - Remove ambiguous bot names - Remove jenkins - Adds `wfwx_update_date` column to `noon_forecasts` and updates whole row unique constraint to store new records of forecasts - Migration first adds this column as nullable, since existing data does not have values here, then sets the column to be equal to the `creation_date` for existing records, then enforces a non-nullable constraint. All new records will have `wfwx_update_date` set to the `updateDate` that WFWX promises to return. - Tested the migration with local data -- switched to main branch with existing db schema, copied partial data from pods, switched to this branch, ran migrations without issue, ran `noon_forecasts` job with start timestamp 1638301981 to verify old and new data records work. - Tested with dummy forecasts live.
bcgov · Jan 17, 2022 · d202a32 · d202a32
1 parent 764dd0c
commit d202a32
Show file tree

Hide file tree

Showing 47 changed files with 1,248 additions and 2,940 deletions.
diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml
@@ -92,31 +92,31 @@ jobs:
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
           SECOND_LEVEL_DOMAIN="apps.silver.devops.gov.bc.ca" VANITY_DOMAIN="${SUFFIX}-dev-psu.nrs.gov.bc.ca" bash openshift/scripts/oc_deploy.sh ${SUFFIX} apply
 
-      - name: Hourly actuals cronjob (Marvin)
+      - name: Hourly actuals cronjob
         shell: bash
         run: |
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
-          bash openshift/scripts/oc_provision_bcfw_p1_hourly_actuals_cronjob.sh ${SUFFIX} apply
+          bash openshift/scripts/oc_provision_wfwx_hourly_actuals_cronjob.sh ${SUFFIX} apply
 
-      - name: Noon forecasts cronjob (Bender)
+      - name: Noon forecasts cronjob
         shell: bash
         run: |
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
-          bash openshift/scripts/oc_provision_bcfw_p1_forecasts_cronjob.sh ${SUFFIX} apply
+          bash openshift/scripts/oc_provision_wfwx_noon_forecasts_cronjob.sh ${SUFFIX} apply
 
-      - name: Environment Canada GDPS cronjob (Donald)
+      - name: Environment Canada GDPS cronjob
         shell: bash
         run: |
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
           PROJ_DEV="e1e498-dev" bash openshift/scripts/oc_provision_ec_gdps_cronjob.sh ${SUFFIX} apply
 
-      - name: Environment Canada HRDPS cronjob (Donald)
+      - name: Environment Canada HRDPS cronjob
         shell: bash
         run: |
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"
           bash openshift/scripts/oc_provision_ec_hrdps_cronjob.sh ${SUFFIX} apply
 
-      - name: Environment Canada RDPS cronjob (Donald)
+      - name: Environment Canada RDPS cronjob
         shell: bash
         run: |
           oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_DEV_TOKEN }}"

diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml
@@ -45,8 +45,8 @@ jobs:
           PROJ_TARGET=e1e498-prod bash openshift/scripts/oc_provision_c_haines_cronjob.sh prod apply
           echo BC FireWeather cronjobs
           echo "Run forecast at 8h30 PDT and 16h30 PDT (so before and after noon)"
-          PROJ_TARGET=e1e498-prod SCHEDULE="30 15,23 * * *" bash openshift/scripts/oc_provision_bcfw_p1_forecasts_cronjob.sh prod apply
-          PROJ_TARGET=e1e498-prod SCHEDULE="15 * * * *" bash openshift/scripts/oc_provision_bcfw_p1_hourly_actuals_cronjob.sh prod apply
+          PROJ_TARGET=e1e498-prod SCHEDULE="30 * * * *" bash openshift/scripts/oc_provision_wfwx_noon_forecasts_cronjob.sh prod apply
+          PROJ_TARGET=e1e498-prod SCHEDULE="15 * * * *" bash openshift/scripts/oc_provision_wfwx_hourly_actuals_cronjob.sh prod apply
           echo Configure backups
           PROJ_TARGET=e1e498-prod CPU_REQUEST=1000m CPU_LIMIT=2000m bash openshift/scripts/oc_provision_backup_s3_postgres_cronjob.sh prod apply
           PROJ_TARGET=e1e498-prod CPU_REQUEST=50m CPU_LIMIT=500m BACKUP_VOLUME_SIZE=3Gi bash openshift/scripts/oc_provision_backup_mariadb.sh prod apply

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -18,6 +18,18 @@
             "module": "app.main",
             "console": "integratedTerminal"
         },
+        {
+            "name": "app.jobs.noon_forecasts",
+            "type": "python",
+            "request": "launch",
+            "module": "app.jobs.noon_forecasts"
+        },
+        {
+            "name": "app.jobs.hourly_actuals",
+            "type": "python",
+            "request": "launch",
+            "module": "app.jobs.hourly_actuals"
+        },
         {
             "name": "app.weather_models.env_canada RDPS",
             "type": "python",

diff --git a/api/Makefile b/api/Makefile
@@ -89,26 +89,16 @@ run-env-canada-hrdps:
 run-env-canada-rdps:
 	${call run-env-canada-model,RDPS}
 
-run-donald:
-	# Synonymn for run-env-canada-raw
-	${call run-env-canada}
-
-run-bender:
-	$(POETRY_RUN) python -m app.fireweather_bot.noon_forecasts
-
 run-noon-forecasts:
-	$(POETRY_RUN) python -m app.fireweather_bot.noon_forecasts
-
-run-marvin:
-	$(POETRY_RUN) python -m app.fireweather_bot.hourly_actuals
+	$(POETRY_RUN) python -m app.jobs.noon_forecasts
 
-run-hourly-observed:
-	$(POETRY_RUN) python -m app.fireweather_bot.hourly_actuals
+run-hourly-actuals:
+	$(POETRY_RUN) python -m app.jobs.hourly_actuals
 
-run-all-bots:
+run-all-jobs:
 	# Run hourlies and actuals 1'st - very quick.
-	$(POETRY_RUN) python -m app.fireweather_bot.hourly_actuals
-	$(POETRY_RUN) python -m app.fireweather_bot.noon_forecasts
+	$(POETRY_RUN) python -m app.jobs.hourly_actuals
+	$(POETRY_RUN) python -m app.jobs.noon_forecasts
 	${call run-env-canada}
 	$(POETRY_RUN) python -m app.c_haines.worker
 
@@ -191,21 +181,15 @@ docker-run-env-canada:
 	docker-compose exec api python -m app.weather_models.env_canada HRDPS
 	docker-compose exec api python -m app.weather_models.env_canada RDPS
 
-docker-run-donald:
-	# Synonymn for docker-run-env-canada
-	docker-compose exec api python -m app.weather_models.env_canada GDPS
-	docker-compose exec api python -m app.weather_models.env_canada HRDPS
-	docker-compose exec api python -m app.weather_models.env_canada RDPS
-
-docker-run-bender:
+docker-run-noon-forecasts:
 	# Run a python script to download forecast data from BC Wild Fire on docker
 	# Needs to run "make docker-run" first
-	docker-compose exec api python -m app.fireweather_bot.noon_forecasts
+	docker-compose exec api python -m app.jobs.noon_forecasts
 
-docker-run-marvin:
+docker-run-hourly-actuals:
 	# Run a python script to download hourly actuals from BC Wild Fire on docker.
 	# Needs to run "make docker-run" first
-	docker-compose exec api python -m app.fireweather_bot.hourly_actuals
+	docker-compose exec api python -m app.jobs.hourly_actuals
 
 database-upgrade:
 	PYTHONPATH=. $(POETRY_RUN) alembic upgrade head

diff --git a/api/alembic/versions/39806f02cdec_wfwx_update_date_part_of_unique_.py b/api/alembic/versions/39806f02cdec_wfwx_update_date_part_of_unique_.py
@@ -0,0 +1,36 @@
+"""wfwx_update_date part of unique constraint
+
+Revision ID: 39806f02cdec
+Revises: d99fcdc4800d
+Create Date: 2021-12-14 11:38:35.435805
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '39806f02cdec'
+down_revision = 'd99fcdc4800d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint('noon_forecasts_weather_date_station_code_temp_valid_tempera_key',
+                       'noon_forecasts', type_='unique')
+    op.create_unique_constraint('unique_forecast', 'noon_forecasts', [
+                                'weather_date', 'wfwx_update_date', 'station_code'])
+    op.drop_column('noon_forecasts', 'danger_rating',
+                   existing_type=sa.INTEGER(),
+                   nullable=True)
+
+
+def downgrade():
+    op.add_column('noon_forecasts',
+                  sa.Column('danger_rating',
+                            sa.INTEGER(),
+                            nullable=True))
+    op.drop_constraint('unique_forecast', 'noon_forecasts', type_='unique')
+    op.create_unique_constraint('noon_forecasts_weather_date_station_code_temp_valid_tempera_key', 'noon_forecasts', [
+                                'weather_date', 'station_code', 'temp_valid', 'temperature', 'rh_valid', 'relative_humidity', 'wdir_valid', 'wind_direction', 'wspeed_valid', 'wind_speed', 'precip_valid', 'precipitation', 'gc', 'ffmc', 'dmc', 'dc', 'isi', 'bui', 'fwi', 'danger_rating'])
diff --git a/api/alembic/versions/d99fcdc4800d_add_non_nullable_wfwx_update_date.py b/api/alembic/versions/d99fcdc4800d_add_non_nullable_wfwx_update_date.py
@@ -0,0 +1,29 @@
+"""Add non-nullable wfwx update date
+
+Revision ID: d99fcdc4800d
+Revises: 1caf3488a340
+Create Date: 2021-12-14 11:27:03.917981
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd99fcdc4800d'
+down_revision = '1caf3488a340'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('noon_forecasts', sa.Column('wfwx_update_date', sa.TIMESTAMP(timezone=True), nullable=True))
+    op.execute("UPDATE noon_forecasts SET wfwx_update_date = created_at")
+    op.alter_column('noon_forecasts', 'wfwx_update_date', nullable=False)
+    op.create_index(op.f('ix_noon_forecasts_wfwx_update_date'),
+                    'noon_forecasts', ['wfwx_update_date'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_noon_forecasts_wfwx_update_date'), table_name='noon_forecasts')
+    op.drop_column('noon_forecasts', 'wfwx_update_date')
diff --git a/api/app/c_haines/worker.py b/api/app/c_haines/worker.py
@@ -1,4 +1,6 @@
 """ Entry point for generating c-haines charts from grib files.
+TODO: Move this file to app/jobs/ to live with the rest of the jobs:
+https://app.zenhub.com/workspaces/wildfire-predictive-services-5e321393e038fba5bbe203b8/issues/bcgov/wps/1601
 """
 import logging
 import asyncio

diff --git a/api/app/db/models/forecasts.py b/api/app/db/models/forecasts.py
@@ -18,57 +18,41 @@ class NoonForecast(Base):
     __tablename__ = 'noon_forecasts'
     __table_args__ = (
         UniqueConstraint('weather_date',
-                         'station_code',
-                         'temp_valid',
-                         'temperature',
-                         'rh_valid',
-                         'relative_humidity',
-                         'wdir_valid',
-                         'wind_direction',
-                         'wspeed_valid',
-                         'wind_speed',
-                         'precip_valid',
-                         'precipitation',
-                         'gc',
-                         'ffmc',
-                         'dmc',
-                         'dc',
-                         'isi',
-                         'bui',
-                         'fwi',
-                         'danger_rating'),
+                         'wfwx_update_date',
+                         'station_code'),
         {'comment': 'The noon_forecast for a weather station and weather date.'}
     )
     id = Column(Integer, primary_key=True)
     weather_date = Column(TZTimeStamp, nullable=False, index=True)
     station_code = Column(Integer, nullable=False, index=True)
-    temp_valid = Column(Boolean, default=False, nullable=False)
-    temperature = Column(Float, nullable=False)
-    rh_valid = Column(Boolean, default=False, nullable=False)
-    relative_humidity = Column(Float, nullable=False)
-    wdir_valid = Column(Boolean, default=False, nullable=False)
+    temp_valid = Column(Boolean, nullable=False, default=False)
+    temperature = Column(Float, nullable=False, default=math.nan)
+    rh_valid = Column(Boolean, nullable=False, default=False)
+    relative_humidity = Column(Float, nullable=False, default=math.nan)
+    wdir_valid = Column(Boolean, nullable=False, default=False)
     # Set default wind_direction to NaN because some stations don't report it
     wind_direction = Column(Float, nullable=False, default=math.nan)
-    wspeed_valid = Column(Boolean, default=False, nullable=False)
-    wind_speed = Column(Float, nullable=False)
-    precip_valid = Column(Boolean, default=False, nullable=False)
-    precipitation = Column(Float, nullable=False)
+    wspeed_valid = Column(Boolean, nullable=False, default=False)
+    wind_speed = Column(Float, nullable=False, default=math.nan)
+    precip_valid = Column(Boolean, nullable=False, default=False)
+    precipitation = Column(Float, nullable=False, default=math.nan)
     gc = Column(Float, nullable=False, default=math.nan)
     ffmc = Column(Float, nullable=False, default=math.nan)
     dmc = Column(Float, nullable=False, default=math.nan)
     dc = Column(Float, nullable=False, default=math.nan)
     isi = Column(Float, nullable=False, default=math.nan)
     bui = Column(Float, nullable=False, default=math.nan)
     fwi = Column(Float, nullable=False, default=math.nan)
-    danger_rating = Column(Integer, nullable=False)
     created_at = Column(TZTimeStamp, nullable=False,
                         default=time_utils.get_utc_now(), index=True)
+    wfwx_update_date = Column(TZTimeStamp, nullable=False, index=True)
 
     def __str__(self):
         return (
             'station_code:{self.station_code}, '
             'weather_date:{self.weather_date}, '
             'created_at:{self.created_at}, '
+            'wfwx_update_date:{self.wfwx_update_date}, '
             'temp={self.temperature}, '
             'ffmc={self.ffmc}'
         ).format(self=self)
diff --git a/api/app/fireweather_bot/__init__.py b/api/app/fireweather_bot/__init__.py