diff --git a/api/app/auto_spatial_advisory/process_hfi.py b/api/app/auto_spatial_advisory/process_hfi.py index e4aab093a..6fc0be9c6 100644 --- a/api/app/auto_spatial_advisory/process_hfi.py +++ b/api/app/auto_spatial_advisory/process_hfi.py @@ -7,13 +7,10 @@ import tempfile from shapely import wkb, wkt from shapely.validation import make_valid -from shapely.geometry import MultiPolygon from osgeo import ogr, osr -from sqlalchemy.sql import text -from sqlalchemy.orm import Session from app.auto_spatial_advisory.common import get_s3_key from app.db.models.auto_spatial_advisory import ClassifiedHfi, HfiClassificationThreshold, RunTypeEnum -from app.db.database import get_async_read_session_scope, get_async_write_session_scope, get_sync_tileserv_db_scope +from app.db.database import get_async_read_session_scope, get_async_write_session_scope from app.db.crud.auto_spatial_advisory import ( save_hfi, get_hfi_classification_threshold, HfiClassificationThresholdEnum, save_run_parameters, get_run_parameters_id) @@ -36,41 +33,6 @@ class UnknownHFiClassification(Exception): """ Raised when the hfi classification is not one of the expected values. """ -def write_classified_hfi_to_tileserver(session: Session, - feature: ogr.Feature, - coordinate_transform: osr.CoordinateTransformation, - for_date: date, - run_datetime: datetime, - run_type: RunType, - advisory: HfiClassificationThreshold, - warning: HfiClassificationThreshold): - """ - Given an ogr.Feature with an assigned HFI threshold value, write it to the tileserv database as a vector. - """ - # https://gdal.org/api/python/osgeo.ogr.html#osgeo.ogr.Geometry - geometry: ogr.Geometry = feature.GetGeometryRef() - # Make sure the geometry is in target_srs! - geometry.Transform(coordinate_transform) - # Would be very nice to go directly from the ogr.Geometry into the database, - # but Sybrand can't figure out how to have the wkt output also include the fact that - # the SRID is target_srs. So we're doing this redundant step of creating a shapely - # geometry from wkt, then dumping it back into wkb, with target srid. - # NOTE: geometry.ExportToIsoWkb isn't consistent in it's return value between - # different versions of gdal (bytearray vs. bytestring) - so we're opting for - # wkt instead of wkb here for better compatibility. - polygon = wkt.loads(geometry.ExportToIsoWkt()) - polygon = make_valid(polygon) - polygon = MultiPolygon([polygon]) - - threshold = get_threshold_from_hfi(feature, advisory, warning) - - statement = text( - 'INSERT INTO hfi (hfi, for_date, run_date, run_type, geom) VALUES (:hfi, :for_date, :run_date, :run_type, ST_GeomFromText(:geom, 3005))') - session.execute(statement, {'hfi': threshold.description, 'for_date': for_date, - 'run_date': run_datetime, 'run_type': run_type.value, 'geom': wkt.dumps(polygon)}) - session.commit() - - def get_threshold_from_hfi(feature: ogr.Feature, advisory: HfiClassificationThreshold, warning: HfiClassificationThreshold): """ Parses the HFI id value (1 or 2) attributed to an ogr.Feature, and returns the id of the @@ -190,13 +152,6 @@ async def process_hfi(run_type: RunType, run_date: date, run_datetime: datetime, # Store the unqiue combination of run type, run datetime and for date in the run_parameters table await save_run_parameters(session, run_type, run_datetime, for_date) - with get_sync_tileserv_db_scope() as session: - logger.info('Writing HFI vectors to tileserv...') - for i in range(layer.GetFeatureCount()): - feature: ogr.Feature = layer.GetFeature(i) - write_classified_hfi_to_tileserver( - session, feature, coordinate_transform, for_date, run_datetime, run_type, advisory, warning) - perf_end = perf_counter() delta = perf_end - perf_start logger.info('%f delta count before and after processing HFI', delta) diff --git a/api/app/db/database.py b/api/app/db/database.py index 16b67dee1..aa1f8e97f 100644 --- a/api/app/db/database.py +++ b/api/app/db/database.py @@ -31,13 +31,6 @@ _write_engine = create_engine(DB_WRITE_STRING, connect_args=connect_args) -tileserv_db_uri = config.get('TILESERV_POSTGRES_URI', DB_WRITE_STRING) - -_tileserv_db_write_engine = create_engine(tileserv_db_uri, - pool_size=5, - max_overflow=10, - pool_pre_ping=True, - connect_args=connect_args) # use pre-ping on read, as connections are quite often stale due to how few users we have at the moment. _read_engine = create_engine( DB_READ_STRING, @@ -57,8 +50,6 @@ autocommit=False, autoflush=False, bind=_write_engine) _read_session = sessionmaker( autocommit=False, autoflush=False, bind=_read_engine) -_tileserv_write_session = sessionmaker( - autocommit=False, autoflush=False, bind=_tileserv_db_write_engine) _async_read_sessionmaker = sessionmaker( autocommit=False, autoflush=False, bind=_async_read_engine, class_=AsyncSession) _async_write_sessionmaker = sessionmaker( @@ -75,11 +66,6 @@ def _get_read_session() -> Session: return _read_session() -def _get_sync_write_tileserv_session() -> Session: - """ abstraction used for mocking out a read session """ - return _tileserv_write_session() - - def _get_async_read_session() -> AsyncSession: """ abstraction used for mocking out a read session """ return _async_read_sessionmaker() @@ -114,16 +100,6 @@ async def get_async_write_session_scope() -> AsyncGenerator[AsyncSession, None]: await session.close() -@contextmanager -def get_sync_tileserv_db_scope() -> Generator[Session, None, None]: - session = _get_sync_write_tileserv_session() - try: - yield session - finally: - logger.info('session closed by context manager') - session.close() - - @contextmanager def get_read_session_scope() -> Generator[Session, None, None]: """Provide a transactional scope around a series of operations. diff --git a/openshift/templates/nats.yaml b/openshift/templates/nats.yaml index a25d8ce4d..4bc4d5fa4 100644 --- a/openshift/templates/nats.yaml +++ b/openshift/templates/nats.yaml @@ -295,36 +295,6 @@ objects: value: "5432" - name: POSTGRES_DATABASE value: ${POSTGRES_DATABASE} - - name: TILESERVER_POSTGRES_READ_HOST - valueFrom: - configMapKeyRef: - name: ${GLOBAL_NAME} - key: env.tileserver_postgres_read_host - - name: TILESERVER_POSTGRES_WRITE_HOST - valueFrom: - configMapKeyRef: - name: ${GLOBAL_NAME} - key: env.tileserver_postgres_write_host - - name: TILESERVER_READ_USER - valueFrom: - secretKeyRef: - name: ${GLOBAL_NAME} - key: tileserv_read_user - - name: TILESERVER_WRITE_USER - valueFrom: - secretKeyRef: - name: ${GLOBAL_NAME} - key: tileserv_write_user - - name: TILESERVER_POSTGRES_WRITE_PASSWORD - valueFrom: - secretKeyRef: - name: ${GLOBAL_NAME} - key: tileserver_postgres_write_password - - name: TILESERVER_POSTGRES_READ_PASSWORD - valueFrom: - secretKeyRef: - name: ${GLOBAL_NAME} - key: tileserver_postgres_read_password - name: OBJECT_STORE_SERVER valueFrom: secretKeyRef: @@ -345,11 +315,6 @@ objects: secretKeyRef: name: ${GLOBAL_NAME} key: object-store-bucket - - name: TILESERV_POSTGRES_URI - valueFrom: - secretKeyRef: - name: ${APP_NAME}-tileserv-${SUFFIX}-pguser-${APP_NAME}-tileserv-${SUFFIX} - key: uri - name: DEM_NAME valueFrom: configMapKeyRef: