Merge branch 'main' into bug/fix-precip-3017

api/app/auto_spatial_advisory/process_hfi.py

@@ -7,13 +7,10 @@
 import tempfile
 from shapely import wkb, wkt
 from shapely.validation import make_valid
-from shapely.geometry import MultiPolygon
 from osgeo import ogr, osr
-from sqlalchemy.sql import text
-from sqlalchemy.orm import Session
 from app.auto_spatial_advisory.common import get_s3_key
 from app.db.models.auto_spatial_advisory import ClassifiedHfi, HfiClassificationThreshold, RunTypeEnum
-from app.db.database import get_async_read_session_scope, get_async_write_session_scope, get_sync_tileserv_db_scope
+from app.db.database import get_async_read_session_scope, get_async_write_session_scope
 from app.db.crud.auto_spatial_advisory import (
     save_hfi, get_hfi_classification_threshold, HfiClassificationThresholdEnum, save_run_parameters,
     get_run_parameters_id)
@@ -36,41 +33,6 @@ class UnknownHFiClassification(Exception):
     """ Raised when the hfi classification is not one of the expected values. """
 
 
-def write_classified_hfi_to_tileserver(session: Session,
-                                       feature: ogr.Feature,
-                                       coordinate_transform: osr.CoordinateTransformation,
-                                       for_date: date,
-                                       run_datetime: datetime,
-                                       run_type: RunType,
-                                       advisory: HfiClassificationThreshold,
-                                       warning: HfiClassificationThreshold):
-    """
-    Given an ogr.Feature with an assigned HFI threshold value, write it to the tileserv database as a vector.
-    """
-    # https://gdal.org/api/python/osgeo.ogr.html#osgeo.ogr.Geometry
-    geometry: ogr.Geometry = feature.GetGeometryRef()
-    # Make sure the geometry is in target_srs!
-    geometry.Transform(coordinate_transform)
-    # Would be very nice to go directly from the ogr.Geometry into the database,
-    # but Sybrand can't figure out how to have the wkt output also include the fact that
-    # the SRID is target_srs. So we're doing this redundant step of creating a shapely
-    # geometry from wkt, then dumping it back into wkb, with target srid.
-    # NOTE: geometry.ExportToIsoWkb isn't consistent in it's return value between
-    # different versions of gdal (bytearray vs. bytestring) - so we're opting for
-    # wkt instead of wkb here for better compatibility.
-    polygon = wkt.loads(geometry.ExportToIsoWkt())
-    polygon = make_valid(polygon)
-    polygon = MultiPolygon([polygon])
-
-    threshold = get_threshold_from_hfi(feature, advisory, warning)
-
-    statement = text(
-        'INSERT INTO hfi (hfi, for_date, run_date, run_type, geom) VALUES (:hfi, :for_date, :run_date, :run_type, ST_GeomFromText(:geom, 3005))')
-    session.execute(statement, {'hfi': threshold.description, 'for_date': for_date,
-                    'run_date': run_datetime, 'run_type': run_type.value, 'geom': wkt.dumps(polygon)})
-    session.commit()
-
-
 def get_threshold_from_hfi(feature: ogr.Feature, advisory: HfiClassificationThreshold, warning: HfiClassificationThreshold):
     """
     Parses the HFI id value (1 or 2) attributed to an ogr.Feature, and returns the id of the
@@ -190,13 +152,6 @@ async def process_hfi(run_type: RunType, run_date: date, run_datetime: datetime,
                 # Store the unqiue combination of run type, run datetime and for date in the run_parameters table
                 await save_run_parameters(session, run_type, run_datetime, for_date)
 
-                with get_sync_tileserv_db_scope() as session:
-                    logger.info('Writing HFI vectors to tileserv...')
-                    for i in range(layer.GetFeatureCount()):
-                        feature: ogr.Feature = layer.GetFeature(i)
-                        write_classified_hfi_to_tileserver(
-                            session, feature, coordinate_transform, for_date, run_datetime, run_type, advisory, warning)
-
     perf_end = perf_counter()
     delta = perf_end - perf_start
     logger.info('%f delta count before and after processing HFI', delta)

api/app/db/database.py

@@ -31,13 +31,6 @@
 
 _write_engine = create_engine(DB_WRITE_STRING, connect_args=connect_args)
 
-tileserv_db_uri = config.get('TILESERV_POSTGRES_URI', DB_WRITE_STRING)
-
-_tileserv_db_write_engine = create_engine(tileserv_db_uri,
-                                          pool_size=5,
-                                          max_overflow=10,
-                                          pool_pre_ping=True,
-                                          connect_args=connect_args)
 # use pre-ping on read, as connections are quite often stale due to how few users we have at the moment.
 _read_engine = create_engine(
     DB_READ_STRING,
@@ -57,8 +50,6 @@
     autocommit=False, autoflush=False, bind=_write_engine)
 _read_session = sessionmaker(
     autocommit=False, autoflush=False, bind=_read_engine)
-_tileserv_write_session = sessionmaker(
-    autocommit=False, autoflush=False, bind=_tileserv_db_write_engine)
 _async_read_sessionmaker = sessionmaker(
     autocommit=False, autoflush=False, bind=_async_read_engine, class_=AsyncSession)
 _async_write_sessionmaker = sessionmaker(
@@ -75,11 +66,6 @@ def _get_read_session() -> Session:
     return _read_session()
 
 
-def _get_sync_write_tileserv_session() -> Session:
-    """ abstraction used for mocking out a read session """
-    return _tileserv_write_session()
-
-
 def _get_async_read_session() -> AsyncSession:
     """ abstraction used for mocking out a read session """
     return _async_read_sessionmaker()
@@ -114,16 +100,6 @@ async def get_async_write_session_scope() -> AsyncGenerator[AsyncSession, None]:
         await session.close()
 
 
-@contextmanager
-def get_sync_tileserv_db_scope() -> Generator[Session, None, None]:
-    session = _get_sync_write_tileserv_session()
-    try:
-        yield session
-    finally:
-        logger.info('session closed by context manager')
-        session.close()
-
-
 @contextmanager
 def get_read_session_scope() -> Generator[Session, None, None]:
     """Provide a transactional scope around a series of operations.

openshift/templates/nats.yaml

@@ -295,36 +295,6 @@ objects:
                   value: "5432"
                 - name: POSTGRES_DATABASE
                   value: ${POSTGRES_DATABASE}
-                - name: TILESERVER_POSTGRES_READ_HOST
-                  valueFrom:
-                    configMapKeyRef:
-                      name: ${GLOBAL_NAME}
-                      key: env.tileserver_postgres_read_host
-                - name: TILESERVER_POSTGRES_WRITE_HOST
-                  valueFrom:
-                    configMapKeyRef:
-                      name: ${GLOBAL_NAME}
-                      key: env.tileserver_postgres_write_host
-                - name: TILESERVER_READ_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: ${GLOBAL_NAME}
-                      key: tileserv_read_user
-                - name: TILESERVER_WRITE_USER
-                  valueFrom:
-                    secretKeyRef:
-                      name: ${GLOBAL_NAME}
-                      key: tileserv_write_user
-                - name: TILESERVER_POSTGRES_WRITE_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: ${GLOBAL_NAME}
-                      key: tileserver_postgres_write_password
-                - name: TILESERVER_POSTGRES_READ_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: ${GLOBAL_NAME}
-                      key: tileserver_postgres_read_password
                 - name: OBJECT_STORE_SERVER
                   valueFrom:
                     secretKeyRef:
@@ -345,11 +315,6 @@ objects:
                     secretKeyRef:
                       name: ${GLOBAL_NAME}
                       key: object-store-bucket
-                - name: TILESERV_POSTGRES_URI
-                  valueFrom:
-                    secretKeyRef:
-                      name: ${APP_NAME}-tileserv-${SUFFIX}-pguser-${APP_NAME}-tileserv-${SUFFIX}
-                      key: uri
                 - name: DEM_NAME
                   valueFrom:
                     configMapKeyRef: