diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml
index 8cb55558b..cfa9f54e9 100644
--- a/.github/workflows/build_and_push.yml
+++ b/.github/workflows/build_and_push.yml
@@ -98,3 +98,16 @@ jobs:
tags: |
opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_tag }}
opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_commit }}
+
+ # Nginx
+ - name: Docker Build and Push nginx
+ id: docker_build_and_push_nginx
+ uses: docker/build-push-action@v2
+ with:
+ builder: ${{ steps.buildx.outputs.name }}
+ context: ./docker-nginx
+ file: ./docker-nginx/Dockerfile
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: |
+ opengisch/qfieldcloud-nginx:${{ steps.prepare.outputs.docker_tag }}
+ opengisch/qfieldcloud-nginx:${{ steps.prepare.outputs.docker_commit }}
diff --git a/.gitignore b/.gitignore
index 88fcf6718..23825ee50 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,7 +8,7 @@ __pycache__/
.env
docker-compose.override.yml
client/projects
-conf/nginx/certs/*
+docker-nginx/certs/*
conf/certbot/*
Pipfile*
**/site-packages
diff --git a/README.md b/README.md
index e974cdc28..e54e506d6 100644
--- a/README.md
+++ b/README.md
@@ -207,11 +207,11 @@ Note if you run tests using the `docker-compose.test.yml` configuration, the `ap
## Add root certificate
-QFieldCloud will automatically generate a certificate and it's root certificate in `./config/nginx/certs`. However, you need to trust the root certificate first, so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance.
+QFieldCloud will automatically generate a certificate and it's root certificate in `./docker-nginx/certs`. However, you need to trust the root certificate first, so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance.
On Debian/Ubuntu, copy the root certificate to the directory with trusted certificates. Note the extension has been changed to `.crt`:
- sudo cp ./conf/nginx/certs/rootCA.pem /usr/local/share/ca-certificates/rootCA.crt
+ sudo cp ./docker-nginx/certs/rootCA.pem /usr/local/share/ca-certificates/rootCA.crt
Trust the newly added certificate:
diff --git a/docker-app/qfieldcloud/core/management/commands/status.py b/docker-app/qfieldcloud/core/management/commands/status.py
index a2979ebcb..c89f32310 100644
--- a/docker-app/qfieldcloud/core/management/commands/status.py
+++ b/docker-app/qfieldcloud/core/management/commands/status.py
@@ -1,4 +1,3 @@
-from django.conf import settings
from django.core.management.base import BaseCommand
from qfieldcloud.core import geodb_utils, utils
@@ -17,8 +16,7 @@ def handle(self, *args, **options):
results["storage"] = "ok"
# Check if bucket exists (i.e. the connection works)
try:
- s3_client = utils.get_s3_client()
- s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME)
+ utils.get_s3_bucket()
except Exception:
results["storage"] = "error"
diff --git a/docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py b/docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py
new file mode 100644
index 000000000..14ee9f840
--- /dev/null
+++ b/docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.24 on 2024-03-14 17:05
+
+import migrate_sql.operations
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("core", "0073_project_packaging_offliner"),
+ ]
+
+ operations = [
+ migrate_sql.operations.AlterSQL(
+ name="core_delta_geom_trigger_func",
+ sql="\n CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()\n RETURNS trigger\n AS\n $$\n DECLARE\n delta_srid int;\n BEGIN\n SELECT CASE\n WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\\d{1,10}$'\n THEN\n REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\\D*', '', 'g')::int\n ELSE\n NULL\n END INTO delta_srid;\n\n IF delta_srid IS NOT NULL\n AND EXISTS(\n SELECT *\n FROM spatial_ref_sys\n WHERE auth_name = 'EPSG'\n AND auth_srid = delta_srid\n )\n THEN\n NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );\n NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );\n ELSE\n NEW.old_geom := NULL;\n NEW.new_geom := NULL;\n END IF;\n\n IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.old_geom := ST_CurveToLine(NEW.old_geom);\n END IF;\n\n IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.new_geom := ST_CurveToLine(NEW.new_geom);\n END IF;\n\n RETURN NEW;\n END;\n $$\n LANGUAGE PLPGSQL\n ",
+ reverse_sql="\n CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()\n RETURNS trigger\n AS\n $$\n DECLARE\n srid int;\n BEGIN\n SELECT CASE\n WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\\d{1,10}$'\n THEN\n REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\\D*', '', 'g')::int\n ELSE\n NULL\n END INTO srid;\n NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );\n NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );\n\n IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.old_geom := ST_CurveToLine(NEW.old_geom);\n END IF;\n\n IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.new_geom := ST_CurveToLine(NEW.new_geom);\n END IF;\n\n RETURN NEW;\n END;\n $$\n LANGUAGE PLPGSQL\n ",
+ ),
+ ]
diff --git a/docker-app/qfieldcloud/core/sql_config.py b/docker-app/qfieldcloud/core/sql_config.py
index 98c19000e..2bce6bc29 100644
--- a/docker-app/qfieldcloud/core/sql_config.py
+++ b/docker-app/qfieldcloud/core/sql_config.py
@@ -178,7 +178,7 @@
AS
$$
DECLARE
- srid int;
+ delta_srid int;
BEGIN
SELECT CASE
WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\d{1,10}$'
@@ -186,9 +186,22 @@
REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\D*', '', 'g')::int
ELSE
NULL
- END INTO srid;
- NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
- NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
+ END INTO delta_srid;
+
+ IF delta_srid IS NOT NULL
+ AND EXISTS(
+ SELECT *
+ FROM spatial_ref_sys
+ WHERE auth_name = 'EPSG'
+ AND auth_srid = delta_srid
+ )
+ THEN
+ NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );
+ NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );
+ ELSE
+ NEW.old_geom := NULL;
+ NEW.new_geom := NULL;
+ END IF;
IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')
THEN
diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py
index 26e2aca93..6ae29a2af 100644
--- a/docker-app/qfieldcloud/core/utils.py
+++ b/docker-app/qfieldcloud/core/utils.py
@@ -386,21 +386,6 @@ def get_project_package_files_count(project_id: str) -> int:
return len(files)
-def get_s3_object_url(
- key: str, bucket: mypy_boto3_s3.service_resource.Bucket = get_s3_bucket()
-) -> str:
- """Returns the block storage URL for a given key. The key may not exist in the bucket.
-
- Args:
- key (str): the object key
- bucket (boto3.Bucket, optional): Bucket for that object. Defaults to `get_s3_bucket()`.
-
- Returns:
- str: URL
- """
- return f"{settings.STORAGE_ENDPOINT_URL}/{bucket.name}/{key}"
-
-
def list_files(
bucket: mypy_boto3_s3.service_resource.Bucket,
prefix: str,
diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py
index bb9928715..ca3cc0d59 100644
--- a/docker-app/qfieldcloud/core/utils2/storage.py
+++ b/docker-app/qfieldcloud/core/utils2/storage.py
@@ -196,7 +196,6 @@ def get_attachment_dir_prefix(
def file_response(
request: HttpRequest,
key: str,
- presigned: bool = False,
expires: int = 60,
version: str | None = None,
as_attachment: bool = False,
@@ -213,25 +212,22 @@ def file_response(
https_port = http_host.split(":")[-1] if ":" in http_host else "443"
if https_port == settings.WEB_HTTPS_PORT and not settings.IN_TEST_SUITE:
- if presigned:
- if as_attachment:
- extra_params["ResponseContentType"] = "application/force-download"
- extra_params[
- "ResponseContentDisposition"
- ] = f'attachment;filename="{filename}"'
-
- url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url(
- "get_object",
- Params={
- **extra_params,
- "Key": key,
- "Bucket": qfieldcloud.core.utils.get_s3_bucket().name,
- },
- ExpiresIn=expires,
- HttpMethod="GET",
- )
- else:
- url = qfieldcloud.core.utils.get_s3_object_url(key)
+ if as_attachment:
+ extra_params["ResponseContentType"] = "application/force-download"
+ extra_params[
+ "ResponseContentDisposition"
+ ] = f'attachment;filename="{filename}"'
+
+ url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url(
+ "get_object",
+ Params={
+ **extra_params,
+ "Key": key,
+ "Bucket": qfieldcloud.core.utils.get_s3_bucket().name,
+ },
+ ExpiresIn=expires,
+ HttpMethod="GET",
+ )
# Let's NGINX handle the redirect to the storage and streaming the file contents back to the client
response = HttpResponse()
@@ -293,7 +289,6 @@ def upload_user_avatar(
file,
key,
{
- "ACL": "public-read",
"ContentType": mimetype.value,
},
)
@@ -357,8 +352,6 @@ def upload_project_thumbail(
file,
key,
{
- # TODO most probably this is not public-read, since the project might be private
- "ACL": "public-read",
"ContentType": mimetype,
},
)
diff --git a/docker-app/qfieldcloud/core/views/files_views.py b/docker-app/qfieldcloud/core/views/files_views.py
index bc0e6ade1..a9fcf49c5 100644
--- a/docker-app/qfieldcloud/core/views/files_views.py
+++ b/docker-app/qfieldcloud/core/views/files_views.py
@@ -207,7 +207,6 @@ def get(self, request, projectid, filename):
return utils2.storage.file_response(
request,
key,
- presigned=True,
expires=600,
version=version,
as_attachment=True,
@@ -374,7 +373,7 @@ class ProjectMetafilesView(views.APIView):
def get(self, request, projectid, filename):
key = utils.safe_join(f"projects/{projectid}/meta/", filename)
- return utils2.storage.file_response(request, key, presigned=True)
+ return utils2.storage.file_response(request, key)
@extend_schema_view(
diff --git a/docker-app/qfieldcloud/core/views/package_views.py b/docker-app/qfieldcloud/core/views/package_views.py
index 297f4ac45..744b003f8 100644
--- a/docker-app/qfieldcloud/core/views/package_views.py
+++ b/docker-app/qfieldcloud/core/views/package_views.py
@@ -184,9 +184,7 @@ def get(self, request, project_id, filename):
key = f"projects/{project_id}/files/{filename}"
# NOTE the `expires` kwarg is sending the `Expires` header to the client, keep it a low value (in seconds).
- return storage.file_response(
- request, key, presigned=True, expires=10, as_attachment=True
- )
+ return storage.file_response(request, key, expires=10, as_attachment=True)
@extend_schema_view(
diff --git a/docker-app/qfieldcloud/core/views/status_views.py b/docker-app/qfieldcloud/core/views/status_views.py
index 6febe4a1f..d1c3ef406 100644
--- a/docker-app/qfieldcloud/core/views/status_views.py
+++ b/docker-app/qfieldcloud/core/views/status_views.py
@@ -1,4 +1,3 @@
-from django.conf import settings
from django.core.cache import cache
from drf_spectacular.utils import extend_schema, extend_schema_view
from qfieldcloud.core import geodb_utils, utils
@@ -25,8 +24,7 @@ def get(self, request):
results["storage"] = "ok"
# Check if bucket exists (i.e. the connection works)
try:
- s3_client = utils.get_s3_client()
- s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME)
+ utils.get_s3_bucket()
except Exception:
results["storage"] = "error"
diff --git a/docker-app/qfieldcloud/subscription/admin.py b/docker-app/qfieldcloud/subscription/admin.py
index 05a7ad9d6..3ffb8bb65 100644
--- a/docker-app/qfieldcloud/subscription/admin.py
+++ b/docker-app/qfieldcloud/subscription/admin.py
@@ -1,4 +1,5 @@
from datetime import timedelta
+from typing import Iterable
from django import forms
from django.contrib import admin
@@ -115,6 +116,21 @@ def save(self, commit=True):
class SubscriptionAdmin(QFieldCloudModelAdmin):
form = SubscriptionModelForm
+ fields = (
+ "plan",
+ "account",
+ "active_since",
+ "active_until",
+ "billing_cycle_anchor_at",
+ "current_period_since",
+ "current_period_until",
+ "notes",
+ "created_at",
+ "created_by",
+ "updated_at",
+ "requested_cancel_at",
+ )
+
list_display = (
"id",
"account__link",
@@ -148,7 +164,35 @@ class SubscriptionAdmin(QFieldCloudModelAdmin):
"account__user__username__iexact",
)
- @admin.display(description="User")
+ def get_fields(
+ self, request: HttpRequest, obj: Subscription | None = None
+ ) -> Iterable[str]:
+ if obj is not None:
+ return (
+ "plan__link",
+ "account__link",
+ "promotion__link",
+ *self.fields[3:],
+ )
+
+ return self.fields
+
+ def get_readonly_fields(
+ self, request: HttpRequest, obj: Subscription | None = None
+ ) -> Iterable[str]:
+ if obj is not None:
+ return (
+ *self.readonly_fields,
+ "plan",
+ "account",
+ "plan__link",
+ "account__link",
+ "promotion__link",
+ )
+
+ return self.readonly_fields
+
+ @admin.display(description="Account")
def account__link(self, instance):
return model_admin_url(
instance.account.user, instance.account.user.username_with_full_name
diff --git a/docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html b/docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html
new file mode 100644
index 000000000..b3772bcc1
--- /dev/null
+++ b/docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html
@@ -0,0 +1,8 @@
+{% extends 'admin/change_form.html' %}
+{% load i18n %}
+
+{% block form_top %}
+ {% if change %}
+ NOTE: For changing Plan create a new Subscription. First cancel the current subscription by setting "Active until".
+ {% endif %}
+{% endblock %}
diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt
index bc5de8fb7..5372a04e6 100644
--- a/docker-app/requirements.txt
+++ b/docker-app/requirements.txt
@@ -10,7 +10,7 @@ cffi==1.15.1
charset-normalizer==2.0.9
coreapi==2.3.3
coreschema==0.0.4
-cryptography==42.0.2
+cryptography==42.0.4
defusedxml==0.7.1
Deprecated==1.2.13
Django==3.2.24
diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py
index 606861ea7..d69bcff6c 100644
--- a/docker-app/worker_wrapper/wrapper.py
+++ b/docker-app/worker_wrapper/wrapper.py
@@ -160,7 +160,7 @@ def run(self):
self.job.output = output.decode("utf-8")
self.job.feedback = feedback
self.job.status = Job.Status.FAILED
- self.job.save(update_fields=["output", "feedback"])
+ self.job.save(update_fields=["output", "feedback", "status"])
logger.info(
"Set job status to `failed` due to being killed by the docker engine.",
)
diff --git a/docker-compose.yml b/docker-compose.yml
index 28df94370..479d5289e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -88,14 +88,11 @@ services:
ofelia.job-exec.runcrons.command: python manage.py runcrons
nginx:
- image: nginx:stable
+ build:
+ context: ./docker-nginx
restart: unless-stopped
volumes:
- - ./conf/nginx/pages/:/var/www/html/pages/
- - ./conf/nginx/templates/:/etc/nginx/templates/
- - ./conf/nginx/certs/:/etc/nginx/certs/:ro
- - ./conf/nginx/options-ssl-nginx.conf:/etc/nginx/options-ssl-nginx.conf
- - ./conf/nginx/ssl-dhparams.pem:/etc/nginx/ssl-dhparams.pem
+ - ./docker-nginx/certs/:/etc/nginx/certs/:ro
- certbot_www:/var/www/certbot
ports:
- ${WEB_HTTP_PORT}:80
@@ -119,7 +116,7 @@ services:
environment:
domain: ${QFIELDCLOUD_HOST}
volumes:
- - ./conf/nginx/certs/:/root/.local/share/mkcert/
+ - ./docker-nginx/certs/:/root/.local/share/mkcert/
command: /bin/sh -c 'mkcert -install && for i in $$(echo $$domain | sed "s/,/ /g"); do [ ! -f /root/.local/share/mkcert/$$i.pem ] && mkcert $$i; done && tail -f -n0 /etc/hosts'
certbot:
@@ -134,6 +131,8 @@ services:
build:
context: ./docker-qgis
network: host
+ args:
+ DEBUG_BUILD: ${DEBUG}
tty: true
command: bash -c "echo QGIS built"
logging: *default-logging
diff --git a/docker-nginx/Dockerfile b/docker-nginx/Dockerfile
new file mode 100644
index 000000000..200d8b581
--- /dev/null
+++ b/docker-nginx/Dockerfile
@@ -0,0 +1,6 @@
+FROM nginx:stable
+
+COPY pages /var/www/html/pages/
+COPY templates/ /etc/nginx/templates/
+COPY options-ssl-nginx.conf /etc/nginx/options-ssl-nginx.conf
+COPY ssl-dhparams.pem /etc/nginx/ssl-dhparams.pem
diff --git a/conf/nginx/options-ssl-nginx.conf b/docker-nginx/options-ssl-nginx.conf
similarity index 100%
rename from conf/nginx/options-ssl-nginx.conf
rename to docker-nginx/options-ssl-nginx.conf
diff --git a/conf/nginx/pages/403.html b/docker-nginx/pages/403.html
similarity index 100%
rename from conf/nginx/pages/403.html
rename to docker-nginx/pages/403.html
diff --git a/conf/nginx/pages/404.html b/docker-nginx/pages/404.html
similarity index 100%
rename from conf/nginx/pages/404.html
rename to docker-nginx/pages/404.html
diff --git a/conf/nginx/pages/500.html b/docker-nginx/pages/500.html
similarity index 100%
rename from conf/nginx/pages/500.html
rename to docker-nginx/pages/500.html
diff --git a/conf/nginx/pages/sad_nyuki.svg b/docker-nginx/pages/sad_nyuki.svg
similarity index 100%
rename from conf/nginx/pages/sad_nyuki.svg
rename to docker-nginx/pages/sad_nyuki.svg
diff --git a/conf/nginx/ssl-dhparams.pem b/docker-nginx/ssl-dhparams.pem
similarity index 100%
rename from conf/nginx/ssl-dhparams.pem
rename to docker-nginx/ssl-dhparams.pem
diff --git a/conf/nginx/templates/default.conf.template b/docker-nginx/templates/default.conf.template
similarity index 100%
rename from conf/nginx/templates/default.conf.template
rename to docker-nginx/templates/default.conf.template
diff --git a/docker-qgis/Dockerfile b/docker-qgis/Dockerfile
index a3b1c85eb..eca4f825c 100644
--- a/docker-qgis/Dockerfile
+++ b/docker-qgis/Dockerfile
@@ -1,13 +1,54 @@
-FROM qgis/qgis:final-3_34_2
+# NOTE if the ubuntu version is changed, also change it in `Suites:` in apt sources, and in `QGIS_VERSION`
+FROM ubuntu:jammy
-RUN apt-get update \
- && apt-get upgrade -y \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+# Install dependencies needed to add QGIS repository
+RUN apt update \
+ && apt upgrade -y \
+ && apt install -y gnupg software-properties-common wget
+
+# Add QGIS GPG key
+RUN wget -O /etc/apt/keyrings/qgis-archive-keyring.gpg https://download.qgis.org/downloads/qgis-archive-keyring.gpg
+
+# Add QGIS repository
+COPY < dict:
logger.info("Extracting QGIS project layer data…")
- project_filename = str(project_filename)
- project = QgsProject.instance()
-
- with set_bad_layer_handler(project):
- project.read(project_filename)
- layers_by_id: dict = get_layers_data(project)
+ project = open_qgis_project(str(project_filename))
+ layers_by_id: dict = get_layers_data(project)
logger.info(
f"QGIS project layer data\n{layers_data_to_string(layers_by_id)}",
@@ -155,6 +147,23 @@ def _extract_layer_data(project_filename: Union[str, Path]) -> dict:
return layers_by_id
+def _open_read_only_project(project_filename: str) -> QgsProject:
+ flags = (
+ # TODO we use `QgsProject` read flags, as the ones in `Qgis.ProjectReadFlags` do not work in QGIS 3.34.2
+ QgsProject.ReadFlags()
+ | QgsProject.ForceReadOnlyLayers
+ | QgsProject.FlagDontLoadLayouts
+ | QgsProject.FlagDontLoad3DViews
+ | QgsProject.DontLoadProjectStyles
+ )
+ return open_qgis_project(
+ project_filename,
+ force_reload=True,
+ disable_feature_count=True,
+ flags=flags,
+ )
+
+
def cmd_package_project(args: argparse.Namespace):
workflow = Workflow(
id="package_project",
@@ -326,9 +335,9 @@ def cmd_process_projectfile(args: argparse.Namespace):
id="opening_check",
name="Opening Check",
arguments={
- "project_filename": WorkDirPath("files", args.project_file),
+ "project_filename": WorkDirPathAsStr("files", args.project_file),
},
- method=qfc_worker.process_projectfile.load_project_file,
+ method=_open_read_only_project,
return_names=["project"],
),
Step(
@@ -345,7 +354,7 @@ def cmd_process_projectfile(args: argparse.Namespace):
id="generate_thumbnail_image",
name="Generate Thumbnail Image",
arguments={
- "project": StepOutput("opening_check", "project"),
+ "project_filename": WorkDirPathAsStr("files", args.project_file),
"thumbnail_filename": Path("/io/thumbnail.png"),
},
method=qfc_worker.process_projectfile.generate_thumbnail,
diff --git a/docker-qgis/qfc_worker/process_projectfile.py b/docker-qgis/qfc_worker/process_projectfile.py
index 261bd5c3a..8095bd372 100644
--- a/docker-qgis/qfc_worker/process_projectfile.py
+++ b/docker-qgis/qfc_worker/process_projectfile.py
@@ -1,10 +1,17 @@
import logging
from pathlib import Path
+from typing import Callable
from xml.etree import ElementTree
-from qgis.core import QgsMapRendererParallelJob, QgsMapSettings, QgsProject
-from qgis.PyQt.QtCore import QEventLoop, QSize
-from qgis.PyQt.QtGui import QColor
+from qgis.core import (
+ QgsLayerTree,
+ QgsMapRendererCustomPainterJob,
+ QgsMapSettings,
+ QgsProject,
+)
+from qgis.PyQt.QtCore import QSize
+from qgis.PyQt.QtGui import QColor, QImage, QPainter
+from qgis.PyQt.QtXml import QDomDocument
from .utils import (
FailedThumbnailGenerationException,
@@ -44,18 +51,6 @@ def check_valid_project_file(project_filename: Path) -> None:
logger.info("QGIS project file is valid!")
-def load_project_file(project_filename: Path) -> QgsProject:
- logger.info("Open QGIS project file…")
-
- project = QgsProject.instance()
- if not project.read(str(project_filename)):
- raise InvalidXmlFileException(error=project.error())
-
- logger.info("QGIS project file opened!")
-
- return project
-
-
def extract_project_details(project: QgsProject) -> dict[str, str]:
"""Extract project details"""
logger.info("Extract project details…")
@@ -65,33 +60,52 @@ def extract_project_details(project: QgsProject) -> dict[str, str]:
logger.info("Reading QGIS project file…")
map_settings = QgsMapSettings()
- def on_project_read(doc):
- r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
- g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
- b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
- background_color = QColor(r, g, b)
- map_settings.setBackgroundColor(background_color)
-
- details["background_color"] = background_color.name()
-
- nodes = doc.elementsByTagName("mapcanvas")
-
- for i in range(nodes.size()):
- node = nodes.item(i)
- element = node.toElement()
- if (
- element.hasAttribute("name")
- and element.attribute("name") == "theMapCanvas"
- ):
- map_settings.readXml(node)
-
- map_settings.setRotation(0)
- map_settings.setOutputSize(QSize(1024, 768))
-
- details["extent"] = map_settings.extent().asWktPolygon()
+ def on_project_read_wrapper(
+ tmp_project: QgsProject,
+ ) -> Callable[[QDomDocument], None]:
+ def on_project_read(doc: QDomDocument) -> None:
+ r, _success = tmp_project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
+ g, _success = tmp_project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
+ b, _success = tmp_project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
+ background_color = QColor(r, g, b)
+ map_settings.setBackgroundColor(background_color)
+
+ details["background_color"] = background_color.name()
+
+ nodes = doc.elementsByTagName("mapcanvas")
+
+ for i in range(nodes.size()):
+ node = nodes.item(i)
+ element = node.toElement()
+ if (
+ element.hasAttribute("name")
+ and element.attribute("name") == "theMapCanvas"
+ ):
+ map_settings.readXml(node)
+
+ map_settings.setRotation(0)
+ map_settings.setOutputSize(QSize(1024, 768))
+
+ details["extent"] = map_settings.extent().asWktPolygon()
+
+ return on_project_read
+
+ # NOTE use a temporary project to get the project extent and background color
+ # as we can disable resolving layers, which results in great speed gains
+ tmp_project = QgsProject()
+ tmp_project_read_flags = (
+ # TODO we use `QgsProject` read flags, as the ones in `Qgis.ProjectReadFlags` do not work in QGIS 3.34.2
+ QgsProject.ReadFlags()
+ | QgsProject.FlagDontResolveLayers
+ | QgsProject.FlagDontLoadLayouts
+ | QgsProject.FlagDontLoad3DViews
+ | QgsProject.DontLoadProjectStyles
+ )
+ tmp_project.readProject.connect(on_project_read_wrapper(tmp_project))
+ tmp_project.read(project.fileName(), tmp_project_read_flags)
- project.readProject.connect(on_project_read)
- project.read(project.fileName())
+ # NOTE force delete the `QgsProject`, otherwise the `QgsApplication` might be deleted by the time the project is garbage collected
+ del tmp_project
details["crs"] = project.crs().authid()
details["project_name"] = project.title()
@@ -111,61 +125,86 @@ def on_project_read(doc):
return details
-def generate_thumbnail(project: QgsProject, thumbnail_filename: Path) -> None:
+def generate_thumbnail(project_filename: str, thumbnail_filename: Path) -> None:
"""Create a thumbnail for the project
As from https://docs.qgis.org/3.16/en/docs/pyqgis_developer_cookbook/composer.html#simple-rendering
Args:
- project (QgsProject)
+ project_filename (str)
thumbnail_filename (Path)
"""
logger.info("Generate project thumbnail image…")
map_settings = QgsMapSettings()
- layer_tree = project.layerTreeRoot()
- def on_project_read(doc):
- r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
- g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
- b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
- map_settings.setBackgroundColor(QColor(r, g, b))
-
- nodes = doc.elementsByTagName("mapcanvas")
-
- for i in range(nodes.size()):
- node = nodes.item(i)
- element = node.toElement()
- if (
- element.hasAttribute("name")
- and element.attribute("name") == "theMapCanvas"
- ):
- map_settings.readXml(node)
-
- map_settings.setRotation(0)
- map_settings.setTransformContext(project.transformContext())
- map_settings.setPathResolver(project.pathResolver())
- map_settings.setOutputSize(QSize(100, 100))
- map_settings.setLayers(reversed(list(layer_tree.customLayerOrder())))
- # print(f'output size: {map_settings.outputSize().width()} {map_settings.outputSize().height()}')
- # print(f'layers: {[layer.name() for layer in map_settings.layers()]}')
-
- project.readProject.connect(on_project_read)
- project.read(project.fileName())
-
- renderer = QgsMapRendererParallelJob(map_settings)
-
- event_loop = QEventLoop()
- renderer.finished.connect(event_loop.quit)
- renderer.start()
-
- event_loop.exec_()
+ def on_project_read_wrapper(
+ tmp_project: QgsProject,
+ tmp_layer_tree: QgsLayerTree,
+ ) -> Callable[[QDomDocument], None]:
+ def on_project_read(doc: QDomDocument) -> None:
+ r, _success = tmp_project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
+ g, _success = tmp_project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
+ b, _success = tmp_project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
+ map_settings.setBackgroundColor(QColor(r, g, b))
+
+ nodes = doc.elementsByTagName("mapcanvas")
+
+ for i in range(nodes.size()):
+ node = nodes.item(i)
+ element = node.toElement()
+ if (
+ element.hasAttribute("name")
+ and element.attribute("name") == "theMapCanvas"
+ ):
+ map_settings.readXml(node)
+
+ map_settings.setRotation(0)
+ map_settings.setTransformContext(tmp_project.transformContext())
+ map_settings.setPathResolver(tmp_project.pathResolver())
+ map_settings.setOutputSize(QSize(100, 100))
+ map_settings.setLayers(reversed(list(tmp_layer_tree.customLayerOrder())))
+
+ return on_project_read
+
+ # NOTE use a temporary project to generate the layer rendering with improved speed
+ tmp_project = QgsProject()
+ tmp_layer_tree = tmp_project.layerTreeRoot()
+ tmp_project_read_flags = (
+ # TODO we use `QgsProject` read flags, as the ones in `Qgis.ProjectReadFlags` do not work in QGIS 3.34.2
+ QgsProject.ReadFlags()
+ | QgsProject.ForceReadOnlyLayers
+ | QgsProject.FlagDontLoadLayouts
+ | QgsProject.FlagDontLoad3DViews
+ | QgsProject.DontLoadProjectStyles
+ )
+ tmp_project.readProject.connect(
+ on_project_read_wrapper(tmp_project, tmp_layer_tree)
+ )
+ tmp_project.read(
+ project_filename,
+ tmp_project_read_flags,
+ )
- img = renderer.renderedImage()
+ img = QImage(map_settings.outputSize(), QImage.Format_ARGB32)
+ painter = QPainter(img)
+ job = QgsMapRendererCustomPainterJob(map_settings, painter)
+ # NOTE we use `renderSynchronously` as it does not crash and produces the thumbnail.
+ # `waitForFinishedWithEventLoop` hangs forever and `waitForFinished` produces blank thumbnail, so don't use them!
+ job.renderSynchronously()
if not img.save(str(thumbnail_filename)):
raise FailedThumbnailGenerationException(reason="Failed to save.")
+ painter.end()
+
+ # NOTE force delete the `QgsMapRendererCustomPainterJob`, `QPainter` and `QImage` because we are paranoid with Cpp objects around
+ del job
+ del painter
+ del img
+ # NOTE force delete the `QgsProject`, otherwise the `QgsApplication` might be deleted by the time the project is garbage collected
+ del tmp_project
+
logger.info("Project thumbnail image generated!")
diff --git a/docker-qgis/qfc_worker/utils.py b/docker-qgis/qfc_worker/utils.py
index ff9a949ef..5b7f1986f 100644
--- a/docker-qgis/qfc_worker/utils.py
+++ b/docker-qgis/qfc_worker/utils.py
@@ -1,4 +1,5 @@
import atexit
+import gc
import hashlib
import inspect
import io
@@ -12,13 +13,14 @@
import tempfile
import traceback
import uuid
+import xml.etree.ElementTree as ET
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
from typing import IO, Any, Callable, NamedTuple, Optional
from libqfieldsync.layer import LayerSource
-from libqfieldsync.utils.bad_layer_handler import bad_layer_handler
+from libqfieldsync.utils.bad_layer_handler import set_bad_layer_handler
from qfieldcloud_sdk import sdk
from qgis.core import (
Qgis,
@@ -26,7 +28,9 @@
QgsMapLayer,
QgsMapSettings,
QgsProject,
+ QgsProjectArchive,
QgsProviderRegistry,
+ QgsZipUtils,
)
from qgis.PyQt import QtCore, QtGui
from tabulate import tabulate
@@ -182,9 +186,6 @@ def exitQgis():
logging.info("QGIS app started!")
- # we set the `bad_layer_handler` and assume we always have only one single `QgsProject` instance within the job's life
- QgsProject.instance().setBadLayerHandler(bad_layer_handler)
-
return QGISAPP
@@ -198,13 +199,113 @@ def stop_app():
if "QGISAPP" not in globals():
return
- QgsProject.instance().read("")
+ QgsProject.instance().clear()
if QGISAPP is not None:
logging.info("Stopping QGIS app…")
+
+ # NOTE we force run the GB just to make sure there are no dangling QGIS objects when we delete the QGIS application
+ gc.collect()
+
QGISAPP.exitQgis()
+
del QGISAPP
+ logging.info("Deleted QGIS app!")
+
+
+def open_qgis_project(
+ project_filename: str,
+ force_reload: bool = False,
+ disable_feature_count: bool = False,
+ flags: Qgis.ProjectReadFlags = Qgis.ProjectReadFlags(),
+) -> QgsProject:
+ logging.info(f'Loading QGIS project "{project_filename}"…')
+
+ if not Path(project_filename).exists():
+ raise FileNotFoundError(project_filename)
+
+ project = QgsProject.instance()
+
+ if project.fileName() == str(project_filename) and not force_reload:
+ logging.info(
+ f'Skip loading QGIS project "{project_filename}", it is already loaded'
+ )
+ return project
+
+ if disable_feature_count:
+ strip_feature_count_from_project_xml(project_filename)
+
+ with set_bad_layer_handler(project):
+ if not project.read(str(project_filename), flags):
+ logging.error(f'Failed to load QGIS project "{project_filename}"!')
+
+ project.setFileName("")
+
+ raise Exception(f"Unable to open project with QGIS: {project_filename}")
+
+ logging.info("Project loaded.")
+
+ return project
+
+
+def strip_feature_count_from_project_xml(project_filename: str) -> None:
+ """Rewrites project XML file with feature count disabled.
+
+ Args:
+ project_filename (str): filename of the QGIS project file (.qgs or .qgz)
+ """
+ archive = None
+ xml_file = project_filename
+ if QgsZipUtils.isZipFile(project_filename):
+ logging.info("The project file is zipped as .qgz, unzipping…")
+
+ archive = QgsProjectArchive()
+
+ if archive.unzip(project_filename):
+ logging.info("The project file is unzipped successfully!")
+ xml_file = archive.projectFile()
+ else:
+ logging.error("The project file is unzipping failed!")
+
+ raise Exception(f"Failed to unzip {project_filename} file!")
+
+ logging.info("Parsing QGIS project file XML…")
+
+ tree = ET.parse(xml_file)
+ root = tree.getroot()
+
+ logging.info("QGIS project parsed!")
+
+ for node in root.findall(".//legendlayer"):
+ node.set("showFeatureCount", "0")
+
+ for node in root.findall(
+ './/layer-tree-layer/customproperties/Option[@type="Map"]/Option[@name="showFeatureCount"]'
+ ):
+ node.set("value", "0")
+
+ logging.info("Writing the QGIS project XML into a file…")
+
+ if archive:
+ tmp_filename = tempfile.NamedTemporaryFile().name
+ tree.write(tmp_filename, short_empty_elements=False)
+
+ if not archive.clearProjectFile():
+ raise Exception("Failed to clear project path from the archive!")
+
+ if not Path(tmp_filename).rename(xml_file):
+ raise Exception("Failed to move the temp xml file into archive dir!")
+
+ archive.addFile(xml_file)
+
+ if not archive.zip(project_filename):
+ raise Exception(f"Failed to zip {project_filename} file!")
+ else:
+ tree.write(xml_file, short_empty_elements=False)
+
+ logging.info("QGIS project file re-written!")
+
def download_project(
project_id: str, destination: Path = None, skip_attachments: bool = True
@@ -394,12 +495,12 @@ def __init__(self, step_id: str, return_name: str):
self.return_name = return_name
-class WorkDirPath:
+class WorkDirPathBase:
def __init__(self, *parts: str, mkdir: bool = False) -> None:
self.parts = parts
self.mkdir = mkdir
- def eval(self, root: Path) -> Path:
+ def eval(self, root: Path) -> Path | str:
path = root.joinpath(*self.parts)
if self.mkdir:
@@ -408,6 +509,16 @@ def eval(self, root: Path) -> Path:
return path
+class WorkDirPath(WorkDirPathBase):
+ def eval(self, root: Path) -> Path:
+ return Path(super().eval(root))
+
+
+class WorkDirPathAsStr(WorkDirPathBase):
+ def eval(self, root: Path) -> str:
+ return str(super().eval(root))
+
+
@contextmanager
def logger_context(step: Step):
log_uuid = uuid.uuid4()
@@ -550,7 +661,7 @@ def run_workflow(
for name, value in arguments.items():
if isinstance(value, StepOutput):
arguments[name] = step_returns[value.step_id][value.return_name]
- elif isinstance(value, WorkDirPath):
+ elif isinstance(value, WorkDirPathBase):
arguments[name] = value.eval(root_workdir)
return_values = step.method(**arguments)
diff --git a/docker-qgis/requirements_libqfieldsync.txt b/docker-qgis/requirements_libqfieldsync.txt
index 67c1d9893..e85c6479c 100644
--- a/docker-qgis/requirements_libqfieldsync.txt
+++ b/docker-qgis/requirements_libqfieldsync.txt
@@ -1 +1 @@
-libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@e20ff5be177c3a221ba4641d8b73a26a35016f66
+libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@5ed1f872f2449657dfb91a00e9da48469d62e226
diff --git a/scripts/init_letsencrypt.sh b/scripts/init_letsencrypt.sh
index 2d1f904b7..10f03705d 100755
--- a/scripts/init_letsencrypt.sh
+++ b/scripts/init_letsencrypt.sh
@@ -9,10 +9,10 @@ set +o allexport
CONFIG_PATH="${CONFIG_PATH:-'./conf'}"
-if [ ! -e "$CONFIG_PATH/nginx/options-ssl-nginx.conf" ] || [ ! -e "$CONFIG_PATH/nginx/ssl-dhparams.pem" ]; then
+if [ ! -e "docker-nginx/options-ssl-nginx.conf" ] || [ ! -e "docker-nginx/ssl-dhparams.pem" ]; then
echo "### Downloading recommended TLS parameters ..."
- curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$CONFIG_PATH/nginx/options-ssl-nginx.conf"
- curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$CONFIG_PATH/nginx/ssl-dhparams.pem"
+ curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "docker-nginx/options-ssl-nginx.conf"
+ curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "docker-nginx/ssl-dhparams.pem"
echo
fi
@@ -34,8 +34,8 @@ docker compose run --rm --entrypoint "\
echo
echo "### Copy the certificate and key to their final destination ..."
-cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/fullchain.pem ${CONFIG_PATH}/nginx/certs/${QFIELDCLOUD_HOST}.pem
-cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/privkey.pem ${CONFIG_PATH}/nginx/certs/${QFIELDCLOUD_HOST}-key.pem
+cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/fullchain.pem docker-nginx/certs/${QFIELDCLOUD_HOST}.pem
+cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/privkey.pem docker-nginx/certs/${QFIELDCLOUD_HOST}-key.pem
echo
echo "### Reloading nginx ..."