Merged master into release

.github/workflows/build_and_push.yml

@@ -98,3 +98,16 @@ jobs:
  tags: |
  opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_tag }}
  opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_commit }}
+
+ # Nginx
+ - name: Docker Build and Push nginx
+ id: docker_build_and_push_nginx
+ uses: docker/build-push-action@v2
+ with:
+ builder: ${{ steps.buildx.outputs.name }}
+ context: ./docker-nginx
+ file: ./docker-nginx/Dockerfile
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: |
+ opengisch/qfieldcloud-nginx:${{ steps.prepare.outputs.docker_tag }}
+ opengisch/qfieldcloud-nginx:${{ steps.prepare.outputs.docker_commit }}

.gitignore

@@ -8,7 +8,7 @@ __pycache__/
 .env
 docker-compose.override.yml
 client/projects
-conf/nginx/certs/*
+docker-nginx/certs/*
 conf/certbot/*
 Pipfile*
 **/site-packages

README.md

@@ -207,11 +207,11 @@ Note if you run tests using the `docker-compose.test.yml` configuration, the `ap
 
 ## Add root certificate
 
-QFieldCloud will automatically generate a certificate and it's root certificate in `./config/nginx/certs`. However, you need to trust the root certificate first, so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance.
+QFieldCloud will automatically generate a certificate and it's root certificate in `./docker-nginx/certs`. However, you need to trust the root certificate first, so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance.
 
 On Debian/Ubuntu, copy the root certificate to the directory with trusted certificates. Note the extension has been changed to `.crt`:
 
- sudo cp ./conf/nginx/certs/rootCA.pem /usr/local/share/ca-certificates/rootCA.crt
+ sudo cp ./docker-nginx/certs/rootCA.pem /usr/local/share/ca-certificates/rootCA.crt
 
 Trust the newly added certificate:
 

docker-app/qfieldcloud/core/management/commands/status.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.core.management.base import BaseCommand
 from qfieldcloud.core import geodb_utils, utils
 
@@ -17,8 +16,7 @@ def handle(self, *args, **options):
  results["storage"] = "ok"
  # Check if bucket exists (i.e. the connection works)
  try:
- s3_client = utils.get_s3_client()
- s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME)
+ utils.get_s3_bucket()
  except Exception:
  results["storage"] = "error"
 

docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.24 on 2024-03-14 17:05
+
+import migrate_sql.operations
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("core", "0073_project_packaging_offliner"),
+ ]
+
+ operations = [
+ migrate_sql.operations.AlterSQL(
+ name="core_delta_geom_trigger_func",
+ sql="\n CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()\n RETURNS trigger\n AS\n $$\n DECLARE\n delta_srid int;\n BEGIN\n SELECT CASE\n WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\\d{1,10}$'\n THEN\n REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\\D*', '', 'g')::int\n ELSE\n NULL\n END INTO delta_srid;\n\n IF delta_srid IS NOT NULL\n AND EXISTS(\n SELECT *\n FROM spatial_ref_sys\n WHERE auth_name = 'EPSG'\n AND auth_srid = delta_srid\n )\n THEN\n NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );\n NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );\n ELSE\n NEW.old_geom := NULL;\n NEW.new_geom := NULL;\n END IF;\n\n IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.old_geom := ST_CurveToLine(NEW.old_geom);\n END IF;\n\n IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.new_geom := ST_CurveToLine(NEW.new_geom);\n END IF;\n\n RETURN NEW;\n END;\n $$\n LANGUAGE PLPGSQL\n ",
+ reverse_sql="\n CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()\n RETURNS trigger\n AS\n $$\n DECLARE\n srid int;\n BEGIN\n SELECT CASE\n WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\\d{1,10}$'\n THEN\n REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\\D*', '', 'g')::int\n ELSE\n NULL\n END INTO srid;\n NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );\n NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );\n\n IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.old_geom := ST_CurveToLine(NEW.old_geom);\n END IF;\n\n IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.new_geom := ST_CurveToLine(NEW.new_geom);\n END IF;\n\n RETURN NEW;\n END;\n $$\n LANGUAGE PLPGSQL\n ",
+ ),
+ ]

docker-app/qfieldcloud/core/sql_config.py

@@ -178,17 +178,30 @@
  AS
  $$
  DECLARE
- srid int;
+ delta_srid int;
  BEGIN
  SELECT CASE
  WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\d{1,10}$'
  THEN
  REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\D*', '', 'g')::int
  ELSE
  NULL
- END INTO srid;
- NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
- NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
+ END INTO delta_srid;
+
+ IF delta_srid IS NOT NULL
+ AND EXISTS(
+ SELECT *
+ FROM spatial_ref_sys
+ WHERE auth_name = 'EPSG'
+ AND auth_srid = delta_srid
+ )
+ THEN
+ NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );
+ NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );
+ ELSE
+ NEW.old_geom := NULL;
+ NEW.new_geom := NULL;
+ END IF;
 
  IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')
  THEN

docker-app/qfieldcloud/core/utils.py

@@ -386,21 +386,6 @@ def get_project_package_files_count(project_id: str) -> int:
  return len(files)
 
 
-def get_s3_object_url(
- key: str, bucket: mypy_boto3_s3.service_resource.Bucket = get_s3_bucket()
-) -> str:
- """Returns the block storage URL for a given key. The key may not exist in the bucket.
-
- Args:
- key (str): the object key
- bucket (boto3.Bucket, optional): Bucket for that object. Defaults to `get_s3_bucket()`.
-
- Returns:
- str: URL
- """
- return f"{settings.STORAGE_ENDPOINT_URL}/{bucket.name}/{key}"
-
-
 def list_files(
  bucket: mypy_boto3_s3.service_resource.Bucket,
  prefix: str,

docker-app/qfieldcloud/core/utils2/storage.py

@@ -196,7 +196,6 @@ def get_attachment_dir_prefix(
 def file_response(
  request: HttpRequest,
  key: str,
- presigned: bool = False,
  expires: int = 60,
  version: str | None = None,
  as_attachment: bool = False,
@@ -213,25 +212,22 @@ def file_response(
  https_port = http_host.split(":")[-1] if ":" in http_host else "443"
 
  if https_port == settings.WEB_HTTPS_PORT and not settings.IN_TEST_SUITE:
- if presigned:
- if as_attachment:
- extra_params["ResponseContentType"] = "application/force-download"
- extra_params[
- "ResponseContentDisposition"
- ] = f'attachment;filename="{filename}"'
-
- url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url(
- "get_object",
- Params={
- **extra_params,
- "Key": key,
- "Bucket": qfieldcloud.core.utils.get_s3_bucket().name,
- },
- ExpiresIn=expires,
- HttpMethod="GET",
- )
- else:
- url = qfieldcloud.core.utils.get_s3_object_url(key)
+ if as_attachment:
+ extra_params["ResponseContentType"] = "application/force-download"
+ extra_params[
+ "ResponseContentDisposition"
+ ] = f'attachment;filename="{filename}"'
+
+ url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url(
+ "get_object",
+ Params={
+ **extra_params,
+ "Key": key,
+ "Bucket": qfieldcloud.core.utils.get_s3_bucket().name,
+ },
+ ExpiresIn=expires,
+ HttpMethod="GET",
+ )
 
  # Let's NGINX handle the redirect to the storage and streaming the file contents back to the client
  response = HttpResponse()
@@ -293,7 +289,6 @@ def upload_user_avatar(
  file,
  key,
  {
- "ACL": "public-read",
  "ContentType": mimetype.value,
  },
  )
@@ -357,8 +352,6 @@ def upload_project_thumbail(
  file,
  key,
  {
- # TODO most probably this is not public-read, since the project might be private
- "ACL": "public-read",
  "ContentType": mimetype,
  },
  )

docker-app/qfieldcloud/core/views/files_views.py

@@ -207,7 +207,6 @@ def get(self, request, projectid, filename):
  return utils2.storage.file_response(
  request,
  key,
- presigned=True,
  expires=600,
  version=version,
  as_attachment=True,
@@ -374,7 +373,7 @@ class ProjectMetafilesView(views.APIView):
 
  def get(self, request, projectid, filename):
  key = utils.safe_join(f"projects/{projectid}/meta/", filename)
- return utils2.storage.file_response(request, key, presigned=True)
+ return utils2.storage.file_response(request, key)
 
 
 @extend_schema_view(

docker-app/qfieldcloud/core/views/package_views.py

@@ -184,9 +184,7 @@ def get(self, request, project_id, filename):
  key = f"projects/{project_id}/files/{filename}"
 
  # NOTE the `expires` kwarg is sending the `Expires` header to the client, keep it a low value (in seconds).
- return storage.file_response(
- request, key, presigned=True, expires=10, as_attachment=True
- )
+ return storage.file_response(request, key, expires=10, as_attachment=True)
 
 
 @extend_schema_view(

docker-app/qfieldcloud/core/views/status_views.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.core.cache import cache
 from drf_spectacular.utils import extend_schema, extend_schema_view
 from qfieldcloud.core import geodb_utils, utils
@@ -25,8 +24,7 @@ def get(self, request):
  results["storage"] = "ok"
  # Check if bucket exists (i.e. the connection works)
  try:
- s3_client = utils.get_s3_client()
- s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME)
+ utils.get_s3_bucket()
  except Exception:
  results["storage"] = "error"
 

docker-app/qfieldcloud/subscription/admin.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+from typing import Iterable
 
 from django import forms
 from django.contrib import admin
@@ -115,6 +116,21 @@ def save(self, commit=True):
 class SubscriptionAdmin(QFieldCloudModelAdmin):
  form = SubscriptionModelForm
 
+ fields = (
+ "plan",
+ "account",
+ "active_since",
+ "active_until",
+ "billing_cycle_anchor_at",
+ "current_period_since",
+ "current_period_until",
+ "notes",
+ "created_at",
+ "created_by",
+ "updated_at",
+ "requested_cancel_at",
+ )
+
  list_display = (
  "id",
  "account__link",
@@ -148,7 +164,35 @@ class SubscriptionAdmin(QFieldCloudModelAdmin):
  "account__user__username__iexact",
  )
 
- @admin.display(description="User")
+ def get_fields(
+ self, request: HttpRequest, obj: Subscription | None = None
+ ) -> Iterable[str]:
+ if obj is not None:
+ return (
+ "plan__link",
+ "account__link",
+ "promotion__link",
+ *self.fields[3:],
+ )
+
+ return self.fields
+
+ def get_readonly_fields(
+ self, request: HttpRequest, obj: Subscription | None = None
+ ) -> Iterable[str]:
+ if obj is not None:
+ return (
+ *self.readonly_fields,
+ "plan",
+ "account",
+ "plan__link",
+ "account__link",
+ "promotion__link",
+ )
+
+ return self.readonly_fields
+
+ @admin.display(description="Account")
  def account__link(self, instance):
  return model_admin_url(
  instance.account.user, instance.account.user.username_with_full_name

docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html

@@ -0,0 +1,8 @@
+{% extends 'admin/change_form.html' %}
+{% load i18n %}
+
+{% block form_top %}
+ {% if change %}
+ NOTE: For changing <b>Plan</b> create a new Subscription. First cancel the current subscription by setting "Active until".
+ {% endif %}
+{% endblock %}

docker-app/requirements.txt

@@ -10,7 +10,7 @@ cffi==1.15.1
 charset-normalizer==2.0.9
 coreapi==2.3.3
 coreschema==0.0.4
-cryptography==42.0.2
+cryptography==42.0.4
 defusedxml==0.7.1
 Deprecated==1.2.13
 Django==3.2.24

docker-app/worker_wrapper/wrapper.py

@@ -160,7 +160,7 @@ def run(self):
  self.job.output = output.decode("utf-8")
  self.job.feedback = feedback
  self.job.status = Job.Status.FAILED
- self.job.save(update_fields=["output", "feedback"])
+ self.job.save(update_fields=["output", "feedback", "status"])
  logger.info(
  "Set job status to `failed` due to being killed by the docker engine.",
  )

docker-compose.yml

@@ -88,14 +88,11 @@ services:
  ofelia.job-exec.runcrons.command: python manage.py runcrons
 
  nginx:
- image: nginx:stable
+ build:
+ context: ./docker-nginx
  restart: unless-stopped
  volumes:
- - ./conf/nginx/pages/:/var/www/html/pages/
- - ./conf/nginx/templates/:/etc/nginx/templates/
- - ./conf/nginx/certs/:/etc/nginx/certs/:ro
- - ./conf/nginx/options-ssl-nginx.conf:/etc/nginx/options-ssl-nginx.conf
- - ./conf/nginx/ssl-dhparams.pem:/etc/nginx/ssl-dhparams.pem
+ - ./docker-nginx/certs/:/etc/nginx/certs/:ro
  - certbot_www:/var/www/certbot
  ports:
  - ${WEB_HTTP_PORT}:80
@@ -119,7 +116,7 @@ services:
  environment:
  domain: ${QFIELDCLOUD_HOST}
  volumes:
- - ./conf/nginx/certs/:/root/.local/share/mkcert/
+ - ./docker-nginx/certs/:/root/.local/share/mkcert/
  command: /bin/sh -c 'mkcert -install && for i in $$(echo $$domain | sed "s/,/ /g"); do [ ! -f /root/.local/share/mkcert/$$i.pem ] && mkcert $$i; done && tail -f -n0 /etc/hosts'
 
  certbot:
@@ -134,6 +131,8 @@ services:
  build:
  context: ./docker-qgis
  network: host
+ args:
+ DEBUG_BUILD: ${DEBUG}
  tty: true
  command: bash -c "echo QGIS built"
  logging: *default-logging

docker-nginx/Dockerfile

@@ -0,0 +1,6 @@
+FROM nginx:stable
+
+COPY pages /var/www/html/pages/
+COPY templates/ /etc/nginx/templates/
+COPY options-ssl-nginx.conf /etc/nginx/options-ssl-nginx.conf
+COPY ssl-dhparams.pem /etc/nginx/ssl-dhparams.pem