Flatgeobuf + JSON + streaming renderers

.github/workflows/test.yml

@@ -17,51 +17,50 @@ jobs:
  OGCAPIF_HOST: localhost
 
  steps:
- - uses: actions/checkout@v3
+  - uses: actions/checkout@v3
 
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
+  - name: Set up Docker Buildx
+  uses: docker/setup-buildx-action@v2
 
- - name: Login to DockerHub
- if: github.repository == 'opengisch/django-ogcapif'
- uses: docker/login-action@v2
- with:
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+  - name: Login to DockerHub
+  if: github.repository == 'opengisch/django-ogcapif'
+  uses: docker/login-action@v2
+  with:
+  username: ${{ secrets.DOCKERHUB_USERNAME }}
+  password: ${{ secrets.DOCKERHUB_TOKEN }}
 
- - name: Build Django image
- uses: docker/build-push-action@v4
- with:
- context: .
- provenance: false
- file: docker/django/Dockerfile
- push: ${{ github.repository == 'opengisch/django-ogcapif' }}
- pull: true
- cache-from: type=registry,ref=opengisch/django-oapif:latest
- cache-to: type=registry,ref=opengisch/django-oapif:latest,mode=max
- tags: opengisch/django-oapif:latest
+  - name: Build Django image
+  uses: docker/build-push-action@v4
+  with:
+  context: .
+  provenance: false
+  file: docker/django/Dockerfile
+  push: ${{ github.repository == 'opengisch/django-ogcapif' }}
+  pull: true
+  cache-from: type=registry,ref=opengisch/django-oapif:latest
+  cache-to: type=registry,ref=opengisch/django-oapif:latest,mode=max
+  tags: opengisch/django-oapif:latest
 
- - name: Setup Compose
- run: |
- # copy default conf
- cp .env.example .env
+  - name: Setup Compose
+  run: |
+  # copy default conf
+  cp .env.example .env
 
- # start the stack
- docker compose up --build -d
+  # start the stack
+  docker compose up --build -d
 
- # deploy static files and migrate database
- docker compose exec django python manage.py collectstatic --no-input
- docker compose exec django python manage.py migrate --no-input
- docker compose exec django python manage.py populate_users
- docker compose exec django python manage.py populate_data
+  # deploy static files and migrate database
+  docker compose exec django python manage.py collectstatic --no-input
+  docker compose exec django python manage.py migrate --no-input
+  docker compose exec django python manage.py populate_users
+  docker compose exec django python manage.py populate_data
 
-# - name: Run unit tests
-# run: |
-# docker compose exec django python manage.py test signalo.value_lists
+ - name: Run unit tests
+ run: docker compose exec django python manage.py test tests
 
- - name: Run integration tests
- run: docker compose run integration_tests
+  - name: Run integration tests
+  run: docker compose exec integration_tests
 
- - name: Failure logs
- if: failure()
- run: docker-compose logs
+  - name: Failure logs
+  if: failure()
+  run: docker-compose logs

requirements.txt

@@ -1,9 +1,14 @@
 django>=4
 django-computedfields
-psycopg2-binary
-transifex-client
+djangorestframework-gis
+git+https://github.com/3nids/django-rest-framework-gis@93d9718d123c2158f4d1c5f6cf461ba8f6cf89e7
 djangorestframework
-git+https://github.com/3nids/django-rest-framework-gis@patch-1
+fiona
+json-stream-generator
+psycopg2-binary
+pyproj
 pyyaml
+transifex-client
 uritemplate
-pyproj
+orjson
+ujson

src/django_oapif/decorators.py

@@ -2,19 +2,25 @@
 
 from django.db import models
 from django.db.models.functions import Cast
-from rest_framework import serializers, viewsets
+from django.http.response import StreamingHttpResponse
+from rest_framework import renderers, response, serializers, viewsets
 from rest_framework.serializers import ModelSerializer
 from rest_framework_gis.serializers import GeoFeatureModelSerializer
 
 from django_oapif.metadata import OAPIFMetadata
 from django_oapif.mixins import OAPIFDescribeModelViewSetMixin
+from django_oapif.renderers import (
+ FGBRenderer,
+ JSONorjson,
+ JSONStreamingRenderer,
+ JSONujson,
+)
 from django_oapif.urls import oapif_router
 
 from .filters import BboxFilterBackend
 from .functions import AsGeoJSON
 
 USE_PG_GEOJSON = False
-USE_PG_GEOJSON = True
 
 
 def register_oapif_viewset(
@@ -87,6 +93,7 @@ class Meta:
  class Viewset(OAPIFDescribeModelViewSetMixin, viewsets.ModelViewSet):
  queryset = Model.objects.all()
  serializer_class = AutoSerializer
+ renderer_classes = [renderers.JSONRenderer, FGBRenderer]
 
  # TODO: these should probably be moved to the mixin
  oapif_title = Model._meta.verbose_name
@@ -103,13 +110,61 @@ class Viewset(OAPIFDescribeModelViewSetMixin, viewsets.ModelViewSet):
  metadata_class = OAPIFMetadata
 
  def finalize_response(self, request, response, *args, **kwargs):
+ """Ensure OPTIONS requests get a correct response."""
  response = super().finalize_response(request, response, *args, **kwargs)
  if request.method == "OPTIONS":
  allowed_actions = self.metadata_class().determine_actions(request, self)
  allowed_actions = ", ".join(allowed_actions.keys())
  response.headers["Allow"] = allowed_actions
  return response
 
+ def list(self, request, *args, **kwargs) -> response.Response | StreamingHttpResponse:
+ """
+ Stream collection items as FGB chunks if 'format=fgb' is passed.
+ Stream them as JSON chunks if 'format=json' and streaming=true' are passed.
+ Otherwise render them as a single JSON chunk, using a variety of renderers depending on the request.
+ """
+ streaming = request.query_params.get("streaming", "").casefold() == "true"
+ format = request.query_params.get("format")
+ encoder = request.query_params.get("json_encoder")
+
+ if format == "fgb":
+ renderer = FGBRenderer()
+ elif format == "json":
+ if streaming:
+ renderer = JSONStreamingRenderer()
+ elif encoder == "orjson":
+ renderer = JSONorjson()
+ elif encoder == "ujson":
+ renderer = JSONujson()
+ else:
+ renderer = renderers.JSONRenderer()
+ else:
+ renderer = renderers.JSONRenderer()
+
+ if streaming:
+ renderer_context = self.get_renderer_context()
+ serializer = self.get_serializer_class()
+ qs = self.filter_queryset(self.get_queryset())
+ if paginated_qs := self.paginate_queryset(qs):
+ serialized = serializer(paginated_qs, many=True).data
+ else:
+ serialized = serializer(qs, many=True).data
+ rendered = renderer.render(serialized, renderer.media_type, renderer_context)
+ return StreamingHttpResponse(rendered)
+ else:
+ request.accepted_renderer = renderer
+ request.accepted_media_type = renderer.media_type
+ return super().list(request, *args, **kwargs)
+
+ def get_renderer_context(self):
+ context = super().get_renderer_context()
+
+ if hasattr(Model, "get_schema"):
+ context.update({"schema": Model.get_schema()})
+
+ return context
+
  def get_queryset(self):
  qs = super().get_queryset()
 

src/django_oapif/renderers.py

@@ -0,0 +1,62 @@
+import io
+from typing import Any, Generator, OrderedDict
+
+import fiona
+import orjson
+import ujson
+from django.conf import settings
+from fiona.crs import CRS
+from json_stream_generator import json_generator
+from rest_framework import renderers
+
+
+class FGBRenderer(renderers.BaseRenderer):
+ format = "fgb"
+ media_type = "application/event-stream"
+
+ def render(self, data: OrderedDict, accepted_media_type=None, renderer_context=None) -> io.BytesIO:
+ """Renders data as a flatgeobuf binary stream"""
+ assert renderer_context
+ features_data = data["features"] if "features" in data else data["results"]["features"]
+ features = (fiona.Feature.from_dict(obj) for obj in features_data)
+ buffer_wrapper = io.BytesIO()
+ with fiona.open(
+ buffer_wrapper,
+ mode="w",
+ driver="FlatGeobuf",
+ schema=renderer_context["schema"],
+ crs=CRS.from_epsg(settings.GEOMETRY_SRID),
+ ) as fh:
+ for feature in features:
+ fh.write(feature)
+
+ buffer_wrapper.seek(0)
+ return buffer_wrapper
+
+
+class JSONStreamingRenderer(renderers.BaseRenderer):
+ format = "json"
+ media_type = "application/x-ndjson"
+
+ def render(self, data: OrderedDict, accepted_media_type=None, renderer_context=None) -> Generator[Any, None, None]:
+ """Renders JSON encoded stream."""
+ features_data = data["features"] if "features" in data else data["results"]["features"]
+ return json_generator(feature for feature in features_data)
+
+
+class JSONorjson(renderers.BaseRenderer):
+ format = "json"
+ media_type = "application/json"
+
+ def render(self, data: OrderedDict, accepted_media_type=None, renderer_context=None) -> bytes:
+ features_data = data["features"] if "features" in data else data["results"]["features"]
+ return orjson.dumps(features_data)
+
+
+class JSONujson(renderers.BaseRenderer):
+ format = "json"
+ media_type = "application/json"
+
+ def render(self, data: OrderedDict, accepted_media_type=None, renderer_context=None) -> str:
+ features_data = data["features"] if "features" in data else data["results"]["features"]
+ return ujson.dumps(features_data)

src/signalo/core/models.py

@@ -28,6 +28,10 @@ class Pole(ComputedFieldsModel):
  def _serialized(self):
  return f'{{"id": "{str(self.id)}", "type": "Feature", "geometry": {self.geom.geojson}, "properties": {{"name": "{self.name}"}}}}'
 
+ @staticmethod
+ def get_schema() -> dict:
+ return {"geometry": "Point", "properties": {"name": "str", "_serialized": "str"}}
+
 
 @register_oapif_viewset()
 class Azimuth(ComputedFieldsModel):

src/signalo/roads/models.py

@@ -9,11 +9,11 @@
 from signalo.settings import GEOMETRY_SRID
 
 
-class Limit1000(OapifPagination):
- default_limit = 1000
+class Limit100000(OapifPagination):
+ default_limit = 100000
 
 
-@register_oapif_viewset(custom_viewset_attrs={"pagination_class": Limit1000})
+@register_oapif_viewset(custom_viewset_attrs={"pagination_class": Limit100000})
 class Road(models.Model):
  id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
  geom = models.MultiLineStringField(srid=GEOMETRY_SRID, verbose_name=_("Geometry"))
@@ -22,3 +22,7 @@ def get_geom(self, geometry: str = "wkb") -> Union[memoryview, str]:
  if geometry == "wkb":
  return self.geom.wkb
  return self.geom.wkt
+
+ @staticmethod
+ def get_schema() -> dict:
+ return {"geometry": "MultiLineString", "properties": {}}

src/tests/settings.py

@@ -37,6 +37,10 @@
 
 INSTALLED_APPS = [
  "tests",
+ "signalo.core",
+ "signalo.value_lists",
+ "signalo.edge_cases",
+ "signalo.roads",
  "django.contrib.admin",
  "django.contrib.auth",
  "django.contrib.contenttypes",
@@ -171,6 +175,9 @@
 OAPIF_TITLE = "DJANGO_OAPIF"
 OAPIF_DESCRIPTION = "Django OAPIF tests"
 
+# Geometry's SRID. This can only be changed prior to initializing the database.
+GEOMETRY_SRID = int(os.environ.get("GEOMETRY_SRID", "2056"))
+
 INTERNAL_IPS = [
  "127.0.0.1", # so that Django toolbar is displayed to localhost
 ]