Merge pull request #108 from geoadmin/develop

New Release v1.3.0 - #minor

app/__init__.py

@@ -32,15 +32,7 @@
 @app.before_request
 def log_route():
     g.setdefault('request_started', time.time())
-    route_logger.info('%s %s', request.method, request.path)
-
-
-@app.after_request
-def wrap_in_callback_if_present(response):
-    if "callback" in request.args:
-        response.headers['Content-Type'] = 'application/javascript'
-        response.data = f'{request.args.get("callback")}({response.get_data(as_text=True)})'
-    return response
+    route_logger.debug('%s %s', request.method, request.path)
 
 
 # Add CORS Headers to all request

app/helpers/validation/__init__.py

@@ -2,7 +2,10 @@
 
 from shapely.geometry import Polygon
 
+from flask import abort
+
 from app.helpers.helpers import float_raise_nan
+from app.settings import VALID_SRID
 
 bboxes = {
     2056:
@@ -38,3 +41,13 @@ def srs_guesser(geom):
                 sr = epsg
                 break
     return sr
+
+
+def validate_sr(sr):
+    if sr not in VALID_SRID:
+        abort(
+            400,
+            "Please provide a valid number for the spatial reference system model: "
+            f"{', '.join(map(str, VALID_SRID))}"
+        )
+    return sr

app/helpers/validation/height.py

@@ -21,12 +21,3 @@ def validate_lon_lat(lon, lat):
         abort(400, "Please provide numerical values for the parameter 'northing'/'lat'")
 
     return lon, lat
-
-
-def validate_sr(sr):
-    if sr not in (21781, 2056):
-        abort(
-            400,
-            "Please provide a valid number for the spatial reference system model 21781 or 2056"
-        )
-    return sr

app/helpers/validation/profile.py

@@ -10,10 +10,13 @@
 from app.helpers.profile_helpers import PROFILE_DEFAULT_AMOUNT_POINTS
 from app.helpers.profile_helpers import PROFILE_MAX_AMOUNT_POINTS
 from app.helpers.validation import srs_guesser
+from app.helpers.validation import validate_sr
 
 logger = logging.getLogger(__name__)
 max_content_length = 32 * 1024 * 1024  # 32MB
 
+PROFILE_VALID_GEOMETRY_TYPES = ['LineString', 'Point']
+
 
 def read_linestring():
     # param geom, list of coordinates defining the line on which we want a profile
@@ -30,22 +33,25 @@ def read_linestring():
 
     if not linestring:
         abort(400, "No 'geom' given, cannot create a profile without coordinates")
+
     try:
         geom = geojson.loads(linestring, object_hook=geojson.GeoJSON.to_instance)
     except ValueError as e:
-        logger.exception(e)
-        abort(400, "Error loading geometry in JSON string")
+        logger.error('Invalid "geom" parameter, it is not geojson: %s', e)
+        abort(400, "Invalid geom parameter, must be a GEOJSON")
+
+    if geom.get('type') not in PROFILE_VALID_GEOMETRY_TYPES:
+        abort(400, f"geom parameter must be a {'/'.join(PROFILE_VALID_GEOMETRY_TYPES)} GEOJSON")
+
     try:
         geom_to_shape = shape(geom)
-    # pylint: disable=broad-except
-    except Exception as e:
-        logger.exception(e)
-        abort(400, "Error converting JSON to Shape")
-    try:
-        geom_to_shape.is_valid
-    # pylint: disable=broad-except
-    except Exception:
-        abort(400, "Invalid Linestring syntax")
+    except ValueError as e:
+        logger.error("Failed to transformed GEOJSON to shape: %s", e)
+        abort(400, "Error converting GEOJSON to Shape")
+
+    if not geom_to_shape.is_valid:
+        abort(400, f"Invalid {geom['type']}")
+
     if len(geom_to_shape.coords) > PROFILE_MAX_AMOUNT_POINTS:
         abort(
             413,
@@ -102,18 +108,14 @@ def read_spatial_reference(linestring):
             abort(400, "No 'sr' given and cannot be guessed from 'geom'")
         spatial_reference = sr
 
-    if spatial_reference not in (21781, 2056):
-        abort(
-            400,
-            "Please provide a valid number for the spatial reference system model 21781 or 2056"
-        )
+    validate_sr(spatial_reference)
     return spatial_reference
 
 
 def read_offset():
     # param offset, used for smoothing. define how many coordinates should be included
     # in the window used for smoothing. If value is zero smoothing is disabled.
-    offset = 3
+    offset = 0
     if 'offset' in request.args:
         offset = request.args.get('offset')
         if offset.isdigit():

app/routes.py

@@ -1,4 +1,8 @@
+import csv
+import json
 import logging
+from distutils.util import strtobool
+from io import StringIO
 
 from shapely.geometry import Point
 from werkzeug.exceptions import HTTPException
@@ -18,8 +22,8 @@
 from app.helpers.route import prefix_route
 from app.helpers.validation import bboxes
 from app.helpers.validation import srs_guesser
+from app.helpers.validation import validate_sr
 from app.helpers.validation.height import validate_lon_lat
-from app.helpers.validation.height import validate_sr
 # add route prefix
 from app.statistics.statistics import load_json
 from app.statistics.statistics import prepare_data
@@ -71,20 +75,45 @@ def height_route():
     alt = get_height(sr, lon, lat, georaster_utils)
     if alt is None:
         abort(400, f'Requested coordinate ({lon},{lat}) out of bounds in sr {sr}')
-    return {'height': str(alt)}
+    data = {'height': str(alt)}
+    if "callback" in request.args:
+        data = f'{request.args.get("callback")}({json.dumps(data, separators=(",", ":"))})'
+        response = make_response(data, 200, {'Content-Type': 'application/javascript'})
+    else:
+        response = make_response(data)
+    return response
 
 
 @app.route('/profile.json', methods=['GET', 'POST'])
 def profile_json_route():
-    return __get_profile_from_helper(True)
+    profile, status_code = _get_profile(True)
+    if "callback" in request.args:
+        data = f'{request.args.get("callback")}({json.dumps(profile, separators=(",", ":"))})'
+        response = make_response(data, {'Content-Type': 'application/javascript'})
+    else:
+        response = make_response(jsonify(profile))
+    return response, status_code
 
 
 @app.route('/profile.csv', methods=['GET', 'POST'])
 def profile_csv_route():
-    return __get_profile_from_helper(False)
+    if "callback" in request.args:
+        abort(400, 'callback parameter not supported')
+    profile, status_code = _get_profile(False)
+    csv.register_dialect(
+        'semi-colon', delimiter=';', quoting=csv.QUOTE_ALL, quotechar='"', lineterminator='\r\n'
+    )
+    buffer = StringIO()
+    writer = csv.writer(buffer, dialect='semi-colon')
+    # write header
+    writer.writerow(profile['headers'])
+    writer.writerows(profile['rows'])
+    buffer.seek(0)
+
+    return buffer.read(), status_code, {'Content-Type': 'text/csv'}
 
 
-def __get_profile_from_helper(output_to_json=True):
+def _get_profile(output_to_json):
     linestring = profile_arg_validation.read_linestring()
     nb_points = profile_arg_validation.read_number_points()
     is_custom_nb_points = profile_arg_validation.read_is_custom_nb_points()
@@ -94,20 +123,20 @@ def __get_profile_from_helper(output_to_json=True):
     # param only_requested_points, which is flag that when set to True will make
     # the profile with only the given points in geom (no filling points)
     if 'only_requested_points' in request.args:
-        only_requested_points = bool(request.args.get('only_requested_points'))
+        only_requested_points = strtobool(request.args.get('only_requested_points'))
     else:
         only_requested_points = False
 
     # flag that define if filling has to be smart, aka to take resolution into account (so that
     # there's not two points closer than what the resolution is) or if points are placed without
     # care for that.
     if 'smart_filling' in request.args:
-        smart_filling = bool(request.args.get('smart_filling'))
+        smart_filling = strtobool(request.args.get('smart_filling'))
     else:
         smart_filling = False
 
     if 'distinct_points' in request.args:
-        keep_points = bool(request.args.get('distinct_points'))
+        keep_points = strtobool(request.args.get('distinct_points'))
     else:
         keep_points = False
 
@@ -122,18 +151,17 @@ def __get_profile_from_helper(output_to_json=True):
         output_to_json=output_to_json,
         georaster_utils=georaster_utils
     )
-    if output_to_json:
-        response = jsonify(result)
-    else:
-        response = str(result)
+
     # If profile calculation resulted in a lower number of point than requested (because there's no
     # need to add points closer to each other than the min resolution of 2m), we return HTTP 203 to
-    # notify that nb_points couldn't be match.
+    # notify that nb_points couldn't be match. Smartfilling can result in more points as expected.
     status_code = 200
-    if is_custom_nb_points and len(result) < nb_points:
+    if output_to_json and is_custom_nb_points and len(result) != nb_points:
         status_code = 203
-    content_type = 'application/json' if output_to_json else 'text/csv'
-    return response, status_code, {'ContentType': content_type, 'Content-Type': content_type}
+    elif not output_to_json and is_custom_nb_points and len(result['rows']) != nb_points:
+        status_code = 203
+
+    return result, status_code
 
 
 # if in debug, we add the route to the statistics page, otherwise it is not visible

app/settings.py

@@ -14,3 +14,5 @@
 PRELOAD_RASTER_FILES = strtobool(os.getenv('PRELOAD_RASTER_FILES', 'False'))
 
 TRAP_HTTP_EXCEPTIONS = True
+
+VALID_SRID = [21781, 2056]

logging-cfg-local.yml

@@ -38,7 +38,6 @@ filters:
       - method
       - headers
       - remote_addr
-      - json
 
 formatters:
   standard:
@@ -56,7 +55,6 @@ formatters:
       - flask_request_path
       - flask_request_method
       - flask_request_headers
-      - flask_request_json
       - flask_request_remote_addr
     remove_empty: True
     fmt:
@@ -65,13 +63,11 @@ formatters:
       logger: name
       module: module
       function: funcName
-      process: process
-      thread: thread
+      worker_id: "%(process)d/%(thread)x"
       request:
         path: flask_request_path
         method: flask_request_method
         headers: flask_request_headers
-        data: flask_request_json
         remote: flask_request_remote_addr
       exc_info: exc_info
       message: message

tests/unit_tests/test_height.py

@@ -263,7 +263,7 @@ def test_height_lv03_with_callback_valid(self, mock_georaster_utils):
             }
         )
         self.assertEqual(resp.content_type, 'application/javascript')
-        self.assertTrue('cb_({' in resp.get_data(as_text=True))
+        self.assertEqual('cb_({"height":"568.2"})', resp.get_data(as_text=True))
 
     @patch('app.routes.georaster_utils')
     def test_height_lv03_miss_northing(self, mock_georaster_utils):