Merge pull request #58 from geoadmin/develop

New Release v1.8.1 - #patch

.env.default

@@ -1,8 +1,11 @@
-BOD_DB_NAME=bod_db_name
-BOD_DB_HOST=bod_db_host
+BOD_DB_NAME=bod_dev
+BOD_DB_HOST=localhost
 BOD_DB_PORT=5432
+SEARCH_WORKERS=3
 
 SEARCH_SPHINX_HOST=localhost
 SEARCH_SPHINX_PORT=9312
 
-GEODATA_STAGING=prod
+GEODATA_STAGING=dev
+
+HTTP_PORT=5000

Makefile

@@ -60,8 +60,7 @@ help:
 	@echo
 	@echo "Possible targets:"
 	@echo -e " \033[1mSetup TARGETS\033[0m "
-	@echo "- setup              Create the python virtual environment and activate it"
-	@echo "- dev                Create the python virtual environment with developper tools and activate it"
+	@echo "- setup              Create the python virtual environment with developper tools and activate it"
 	@echo "- ci                 Create the python virtual environment and install requirements based on the Pipfile.lock"
 	@echo -e " \033[1mFORMATING, LINTING AND TESTING TOOLS TARGETS\033[0m "
 	@echo "- format             Format the python source code"
@@ -85,18 +84,11 @@ help:
 
 # Build targets. Calling setup is all that is needed for the local files to be installed as needed.
 
-.PHONY: dev
-dev: $(REQUIREMENTS)
-	pipenv install --dev
-	pipenv shell
-
-
 .PHONY: setup
 setup: $(REQUIREMENTS)
-	pipenv install
-	pipenv shell
 	cp -n .env.default .env.local
-
+	pipenv install --dev
+	pipenv shell
 
 
 .PHONY: ci
@@ -181,6 +173,8 @@ dockerrun: clean_logs dockerbuild $(LOGS_DIR)
 		--env-file=${PWD}/$(ENV_FILE) \
 		--env LOGS_DIR=/logs \
 		--env SCRIPT_NAME=$(ROUTE_PREFIX) \
+		--env PGUSER=${PGUSER} \
+		--env PGPASSWORD=${PGPASSWORD} \
 		--net host \
 		--mount type=bind,source="${LOGS_DIR}",target=/logs \
 		$(DOCKER_IMG_LOCAL_TAG)

README.md

@@ -5,22 +5,20 @@
 | develop | ![Build Status](https://codebuild.eu-central-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiUDZNMlVLR3d5bUhsTUF3ZEo3RTRPdDFKdS90czR4ZE5vYmNjTXhtK2tzNGlOckNXb29yaE1DNktwVXFJSVpMdExEVWYzZHA5U1drcmdsTE5BU3lJWDBJPSIsIml2UGFyYW1ldGVyU3BlYyI6IjM2YlhQR1ltcEtlTU16WC8iLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop) |
 | master  | ![Build Status](https://codebuild.eu-central-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiUDZNMlVLR3d5bUhsTUF3ZEo3RTRPdDFKdS90czR4ZE5vYmNjTXhtK2tzNGlOckNXb29yaE1DNktwVXFJSVpMdExEVWYzZHA5U1drcmdsTE5BU3lJWDBJPSIsIml2UGFyYW1ldGVyU3BlYyI6IjM2YlhQR1ltcEtlTU16WC8iLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=master)  |
 
-
 ## Table of content
 
-- [service-search-wsgi](#service-search-wsgi)
-  - [Table of content](#table-of-content)
-  - [Description](#description)
-  - [Versioning](#versioning)
-  - [Local Development](#local-development)
-    - [Make Dependencies](#make-dependencies)
-    - [Setting up to work](#setting-up-to-work)
-      - [Database access](#database-access)
-    - [Linting and formatting your work](#linting-and-formatting-your-work)
-    - [Test your work](#test-your-work)
-  - [Docker](#docker)
-  - [Deployment](#deployment)
-    - [Deployment configuration](#deployment-configuration)
+- [Table of content](#table-of-content)
+- [Description](#description)
+- [Versioning](#versioning)
+- [Local Development](#local-development)
+  - [Make Dependencies](#make-dependencies)
+  - [Setting up to work](#setting-up-to-work)
+    - [Database access](#database-access)
+  - [Linting and formatting your work](#linting-and-formatting-your-work)
+  - [Test your work](#test-your-work)
+- [Docker](#docker)
+- [Deployment](#deployment)
+  - [Deployment configuration](#deployment-configuration)
 
 ## Description
 
@@ -47,23 +45,18 @@ First, you'll need to clone the repo
 git clone git@github.com:geoadmin/service-search-wsgi
 ```
 
-Then, you can run the setup target to ensure you have everything needed to develop, test and serve locally
-
-Virtual environment to run the service (f.ex. make serve)
-
-```bash
-make setup
-```
+Then, you can run the `setup` target to ensure you have everything needed to develop, test and serve locally
 
 Virtual environment to develop and debug the service
 
 ```bash
-make dev
+make setup
 ```
 
 To run the service you will have to adapt **.env.local**, which is a copy of **.env.default** And to set the variables.
 
-For local development you will need access to a running sphinx search server and to the database.
+For local development you will need access to a running sphinx search server and to the database. To do so you can use 
+ssh port forwarding to the DB and to the current sphinx deployment server.
 
 #### Database access
 
@@ -93,19 +86,19 @@ make test
 This command run the unit tests.
 
 ```bash
-make serve
+summon make serve
 ```
 
 This will serve the application through Flask without any wsgi in front.
 
 ```bash
-make gunicornserve
+summon make gunicornserve
 ```
 
 This will serve the application with the Gunicorn layer in front of the application
 
 ```bash
-make dockerrun
+summon make dockerrun
 ```
 
 This will serve the application with the wsgi server, inside a container.

app/helpers/helpers_search.py

@@ -1,3 +1,5 @@
+import base64
+import hashlib
 import logging
 import math
 import unicodedata
@@ -108,8 +110,20 @@ def round_geometry_coordinates(geom, precision=None):
     return geom
 
 
-@cache.memoize(timeout=60)
+def transform_geom_make_cache_key(geom, srid_from, srid_to, rounding=True):
+    cache_key = f"{str(geom)}{srid_from}{srid_to}{rounding}"
+    hasher = hashlib.md5()
+    hasher.update(cache_key.encode("utf-8"))
+    cache_key = base64.b64encode(hasher.digest())
+    cache_key = cache_key.decode("utf-8")
+    return f"trg_{cache_key}"
+
+
+@cache.cached(timeout=60, make_cache_key=transform_geom_make_cache_key)
 def transform_round_geometry(geom, srid_from, srid_to, rounding=True):
+    logger.debug(
+        "Transform geometry=%s from=%s to=%s rounding=%s", geom, srid_from, srid_to, rounding
+    )
     if srid_from == srid_to:
         if rounding:
             precision = get_precision_for_proj(srid_to)
@@ -187,18 +201,12 @@ def float_raise_nan(val):
 # used by search.py
 def parse_box2d(stringBox2D):
     extent = stringBox2D.replace('BOX(', '').replace(')', '').replace(',', ' ')
-    # Python2/3
-    box = map(float, extent.split(' '))
-    if not isinstance(box, list):
-        box = list(box)
+    box = list(map(float, extent.split(' ')))
     return box
 
 
 # used by center_from_box_2d used by search.py
 def is_box2d(box2D):
-    # Python2/3
-    if not isinstance(box2D, list):
-        box2D = list(box2D)
     # Bottom left to top right only
     if len(box2D) != 4 or box2D[0] > box2D[2] or box2D[1] > box2D[3]:
         raise ValueError(f'Invalid box2D {box2D}.')

app/helpers/validation_search.py

@@ -127,7 +127,6 @@ def searchText(self, value):
             raise BadRequest("Please provide a search text")
         searchTextList = value.split(' ')
         # Remove empty strings
-        # Python2/3
         searchTextList = list(filter(None, searchTextList))
         if ilen(searchTextList) > MAX_SEARCH_TERMS:
             msg = "The searchText parameter can not contain more than 10 words"

app/search.py

@@ -583,9 +583,8 @@ def _add_feature_queries(self, queryText, timeFilter):
                     raise BadRequest(msg)
                 self.sphinx.AddQuery(queryText, index=str(index))
 
-    def _box2d_transform(self, res_in):
+    def _box2d_transform(self, res):
         """Reproject a ST_BOX2 from EPSG:21781 to SRID"""
-        res = res_in
         try:
             box2d = res['geom_st_box2d']
             box_str = box2d[4:-1]
@@ -594,14 +593,11 @@ def _box2d_transform(self, res_in):
             bbox = transform_shape(shape, self.DEFAULT_SRID, self.srid).bounds
             res['geom_st_box2d'] = f"BOX({bbox[0]} {bbox[1]},{bbox[2]} {bbox[3]})"
         except Exception as e:
-            msg = f'Error while converting BOX2D ({res_in}) to EPSG:{self.srid}'
+            msg = f'Error while converting BOX2D ({res}) to EPSG:{self.srid}'
             logger.error(msg, e)
             raise InternalServerError(msg) from e
-        return res
-
-    def _parse_locations(self, transformer, res_in):
 
-        res = res_in
+    def _parse_locations(self, transformer, res):
         if not self.returnGeometry:
             attrs2Del = ['x', 'y', 'lon', 'lat', 'geom_st_box2d']
             list(map(lambda x: res.pop(x) if x in res else x, attrs2Del))
@@ -612,9 +608,9 @@ def _parse_locations(self, transformer, res_in):
                     res['x'] = res['lon']
                     res['y'] = res['lat']
                 except KeyError as error:
-                    logger.error("Sphinx location has no lat/long defined %s", res_in)
+                    logger.error("Sphinx location has no lat/long defined %s", res)
                     raise InternalServerError(
-                        f'Sphinx location has no lat/long defined {res_in}'
+                        f'Sphinx location has no lat/long defined {res}'
                     ) from error
             else:
                 try:
@@ -623,9 +619,9 @@ def _parse_locations(self, transformer, res_in):
                     res['x'] = x
                     res['y'] = y
                 except (pyproj.exceptions.CRSError) as error:
-                    logger.error("Error while converting point %s to %s", res_in, self.srid)
+                    logger.error("Error while converting point %s to %s", res, self.srid)
                     raise InternalServerError(
-                        f'Error while converting point({res_in}), to EPSG:{self.srid}'
+                        f'Error while converting point({res}), to EPSG:{self.srid}'
                     ) from error
         return res
 

secrets.yml

@@ -0,0 +1,2 @@
+PGUSER: !var infra-gopass-bgdi/mf-chsdi3/PG PGUSER
+PGPASSWORD: !var infra-gopass-bgdi/mf-chsdi3/PG PGPASSWORD