fixing incorrect merge choices

.github/workflows/conformance.yml

@@ -55,11 +55,10 @@ jobs:
  # deploy static files and migrate database
  docker compose exec django python manage.py collectstatic --no-input
  docker compose exec django python manage.py migrate --no-input
- docker compose exec django python manage.py populate_vl
- docker compose exec django python manage.py populate_signs_poles
+ docker compose exec django python manage.py populate_data
 
  - name: Healthcheck
- run: wget --no-check-certificate https://localhost/oapif/collections/signalo_core.pole/items
+ run: wget --no-check-certificate https://localhost/oapif/collections/tests.point_2056_10fields/items
 
  - name: Run conformance test suite
  run: docker compose run conformance_test

.github/workflows/deploy.yml

@@ -30,10 +30,7 @@ jobs:
  docker-compose exec django python manage.py migrate --no-input
  docker-compose exec django python manage.py collectstatic --no-input
  docker-compose exec django python manage.py populate_users
- docker-compose exec django python manage.py populate_vl
- docker-compose exec django python manage.py populate_signs_poles
- docker-compose exec django python manage.py populate_edge_cases
- docker-compose exec django python manage.py populate_roads
+ docker-compose exec django python manage.py populate_data
 
  - name: Failure logs
  if: failure()

docker-compose.yml

@@ -20,12 +20,12 @@ services:
  - 443:443
 
  django:
- image: opengisch/signalo-oapif:latest
+ image: opengisch/django-oapif:latest
  build:
  context: .
  dockerfile: docker/django/Dockerfile
  cache_from:
- - opengisch/signalo-oapif:latest
+ - opengisch/django-oapif:latest
  restart: unless-stopped
  depends_on:
  postgres:

docker/django/Dockerfile

@@ -1,5 +1,5 @@
 # pull official base image
-FROM python:3.9-buster
+FROM python:3.10-buster
 
 # set work directory
 WORKDIR /usr/src/app

docker/integration-tests/test_integration_qgis.py

@@ -13,7 +13,7 @@
 
 ROOT_URL = "http://django:8000/oapif/"
 COLLECTIONS_URL = "http://django:8000/oapif/collections"
-POLES_URL = "http://django:8000/oapif/collections/signalo_core.pole"
+POINTS_URL = "http://django:8000/oapif/collections/tests.point_2056_10fields"
 
 
 class TestStack(unittest.TestCase):
@@ -27,36 +27,36 @@ def setUpClass(cls):
  def test_endpoint_ok(self):
  root_response = requests.get(ROOT_URL)
  collections_response = requests.get(COLLECTIONS_URL)
- poles_response = requests.get(POLES_URL)
+ points_response = requests.get(POINTS_URL)
 
  self.assertTrue(root_response.status_code == 200)
  self.assertTrue(collections_response.status_code == 200)
- self.assertTrue(poles_response.status_code == 200)
+ self.assertTrue(points_response.status_code == 200)
 
  def test_collection_exists(self):
  res = requests.get(COLLECTIONS_URL).json()
  self.assertTrue(
- "signalo_core.pole"
+ "tests.point_2056_10fields"
  in [collection["id"] for collection in res["collections"]]
  )
 
- def test_many_poles(self):
- poles = requests.get(POLES_URL).json()
- self.assertTrue(len(poles) > 1)
+ def test_many_points(self):
+ points = requests.get(POINTS_URL).json()
+ self.assertTrue(len(points) > 1)
 
  def test_load_layer(self):
  uri = QgsDataSourceUri()
  uri.setParam("service", "wfs")
- uri.setParam("typename", "signalo_core.pole")
+ uri.setParam("typename", "tests.point_2056_10fields")
  uri.setParam("url", ROOT_URL)
- layer = QgsVectorLayer(uri.uri(), "pole", "OAPIF")
+ layer = QgsVectorLayer(uri.uri(), "point", "OAPIF")
  self.assertTrue(layer.isValid())
 
  layer = self.project.addMapLayer(layer)
  self.assertIsNotNone(layer)
 
  f = None
- for f in layer.getFeatures("name='1-1'"):
+ for f in layer.getFeatures("field_0 is not null"):
  pass
  self.assertIsInstance(f, QgsFeature)
 
@@ -70,12 +70,12 @@ def test_load_layer(self):
  def test_load_with_basic_auth(self):
  uri = QgsDataSourceUri()
  uri.setParam("service", "wfs")
- uri.setParam("typename", "signalo_core.pole")
+ uri.setParam("typename", "tests.point_2056_10fields")
  uri.setParam("url", ROOT_URL)
  uri.setPassword(self.password)
  uri.setUsername(self.user)
 
- layer = QgsVectorLayer(uri.uri(), "pole", "OAPIF")
+ layer = QgsVectorLayer(uri.uri(), "point", "OAPIF")
  self.assertTrue(layer.isValid())
  layer = self.project.addMapLayer(layer)
  self.assertIsNotNone(layer)
@@ -88,15 +88,15 @@ def test_load_with_basic_auth(self):
  )
 
  f = None
- for f in layer.getFeatures("name='1-1'"):
+ for f in layer.getFeatures():
  pass
  self.assertIsInstance(f, QgsFeature)
 
- f["name"] = "xyz"
+ f["field_1"] = "xyz"
  with edit(layer):
  layer.updateFeature(f)
 
  f = None
- for f in layer.getFeatures("name='xyz'"):
+ for f in layer.getFeatures("field_1='xyz'"):
  pass
  self.assertIsInstance(f, QgsFeature)

requirements.txt

@@ -1,9 +1,11 @@
 django>=4
 django-computedfields
-psycopg2-binary
-transifex-client
-djangorestframework
 djangorestframework-gis
+djangorestframework
+fiona
+json-stream-generator
+psycopg2-binary
+pyproj
 pyyaml
+transifex-client
 uritemplate
-pyproj

scripts/restart.sh

@@ -4,23 +4,22 @@ set -e
 
 FULL=$1
 
-SRID=4326
+#if [[ $FULL == "reset" ]];then
+ #rm signalo/signalo_app/migrations/00*.py || true
+#fi
 
-if [[ $FULL == "reset" ]];then
- rm signalo/signalo_app/migrations/00*.py || true
- ./scripts/fixture-generator.py -s $SRID -m 100
-fi
+export COMPOSE_FILE=docker-compose.dev.yml:docker-compose.yml
 
-docker-compose down --volumes || true
+docker compose down --volumes || true
 
-docker-compose up --build -d
+docker compose up --build --force-recreate -d
 sleep 5
 
 if [[ $FULL == "reset" ]];then
- docker-compose exec django python manage.py makemigrations
- docker-compose exec django python manage.py migrate
+ docker compose exec django python manage.py makemigrations
+ docker compose exec django python manage.py migrate
 fi
 
-docker-compose exec django python manage.py loaddata pole
-docker-compose exec django python manage.py loaddata sign
-docker-compose exec django python manage.py updatedata
+docker compose exec django python manage.py collectstatic --no-input
+docker compose exec django python manage.py populate_users
+docker compose exec django python manage.py populate_data

src/django_oapif/decorators.py

@@ -1,11 +1,12 @@
 from typing import Any, Callable, Dict, Optional
 
 from django.db.models import Model
-from rest_framework import viewsets
+from rest_framework import renderers, viewsets
 from rest_framework_gis.serializers import GeoFeatureModelSerializer
 
 from django_oapif.metadata import OAPIFMetadata
 from django_oapif.mixins import OAPIFDescribeModelViewSetMixin
+from django_oapif.renderers import FGBRenderer, JSONStreamingRenderer
 from django_oapif.urls import oapif_router
 
 from .filters import BboxFilterBackend
@@ -57,21 +58,39 @@ class Meta:
  else:
  """
  viewset_serializer_class = AutoSerializer
+ viewset_renderer_classes = [
+ renderers.JSONRenderer,
+ FGBRenderer,
+ JSONStreamingRenderer,
+ ]
  viewset_oapif_geom_lookup = (
  "geom" # one day this will be retrieved automatically from the serializer
  )
+ _geo_field = "geom"
+ if skip_geom:
+ _viewset_oapif_geom_lookup = None
+ _geo_field = None
+
+ class AutoSerializer(GeoFeatureModelSerializer):
+ class Meta:
+ nonlocal _geo_field
+
+ model = Model
+ fields = "__all__"
+ geo_field = _geo_field
 
  # Create the viewset
  class Viewset(OAPIFDescribeModelViewSetMixin, viewsets.ModelViewSet):
  queryset = Model.objects.all()
  serializer_class = viewset_serializer_class
+ renderer_classes = viewset_renderer_classes
 
  # TODO: these should probably be moved to the mixin
  oapif_title = Model._meta.verbose_name
  oapif_description = Model.__doc__
 
  # (one day this will be retrieved automatically from the serializer)
- oapif_geom_lookup = viewset_oapif_geom_lookup
+ oapif_geom_lookup = _viewset_oapif_geom_lookup
  filter_backends = [BboxFilterBackend]
 
  # Allowing '.' and '-' in urls
@@ -81,6 +100,7 @@ class Viewset(OAPIFDescribeModelViewSetMixin, viewsets.ModelViewSet):
  metadata_class = OAPIFMetadata
 
  def finalize_response(self, request, response, *args, **kwargs):
+ """Ensure OPTIONS requests get a correct response."""
  response = super().finalize_response(request, response, *args, **kwargs)
  if request.method == "OPTIONS":
  allowed_actions = self.metadata_class().determine_actions(
@@ -90,6 +110,19 @@ def finalize_response(self, request, response, *args, **kwargs):
  response.headers["Allow"] = allowed_actions
  return response
 
+ def list(self, request, *args, **kwargs):
+ """
+ Stream collection items as JSON chunks if 'streaming=true' is passed.
+ Stream them as FGB chunks if 'format=fgb' is passed.
+ Otherwise render them as a single JSON chunk JSON.
+ """
+ if request.query_params.get("format") == "json":
+ if request.query_params.get("streaming", "").casefold() == "true":
+ self.renderer_classes = [JSONStreamingRenderer]
+ elif request.query_params.get("format") == "fgb":
+ self.renderer_classes = [FGBRenderer]
+ return super().list(request, *args, **kwargs)
+
  # ON HOLD, WAITING ON GeoFeatureModelSerializer to admit of null geometries
  """
  # Apply custom serializer attributes

src/django_oapif/pagination.py

@@ -6,6 +6,8 @@
 class OapifPagination(pagination.LimitOffsetPagination):
  """OAPIF-compatible django rest paginator"""
 
+ default_limit = 1000
+
  def get_paginated_response(self, data):
  return Response(
  {

src/django_oapif/renderers.py

@@ -0,0 +1,54 @@
+import io
+from typing import OrderedDict
+
+import fiona
+from django.conf import settings
+from django.http import StreamingHttpResponse
+from fiona.crs import CRS
+from json_stream_generator import json_generator
+from rest_framework import renderers
+
+
+class FGBRenderer(renderers.BaseRenderer):
+ format = "fgb"
+ media_type = "application/event-stream"
+ # FIXME: This should be sent by the model.
+ schema = {"geometry": "Point", "properties": {"name": "str", "_serialized": "str"}}
+
+ def render(
+ self, data: OrderedDict, accepted_media_type=None, renderer_context=None
+ ) -> StreamingHttpResponse:
+ """Renders pre-serialized Python objects as a flatgeobuf binary stream"""
+ features_data = (
+ data["features"] if "features" in data else data["results"]["features"]
+ )
+ features = (fiona.Feature.from_dict(obj) for obj in features_data)
+ buffer_wrapper = io.BytesIO()
+
+ with fiona.open(
+ buffer_wrapper,
+ mode="w",
+ driver="FlatGeobuf",
+ schema=self.schema,
+ crs=CRS.from_epsg(settings.GEOMETRY_SRID),
+ ) as fh:
+ for feature in features:
+ fh.write(feature)
+
+ buffer_wrapper.seek(0)
+ return StreamingHttpResponse(buffer_wrapper)
+
+
+class JSONStreamingRenderer(renderers.BaseRenderer):
+ format = "json"
+ media_type = "application/x-ndjson"
+
+ def render(
+ self, data: OrderedDict, accepted_media_type=None, renderer_context=None
+ ) -> StreamingHttpResponse:
+ """Renders JSON encoded stream."""
+ features_data = (
+ data["features"] if "features" in data else data["results"]["features"]
+ )
+ generate = json_generator(feature for feature in features_data)
+ return StreamingHttpResponse(generate)

src/manage.py

@@ -6,7 +6,7 @@
 
 def main():
  """Run administrative tasks."""
- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "signalo.settings")
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings")
  try:
  from django.core.management import execute_from_command_line
  except ImportError as exc: