diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 8cb55558b..cfa9f54e9 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -98,3 +98,16 @@ jobs: tags: | opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_tag }} opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_commit }} + + # Nginx + - name: Docker Build and Push nginx + id: docker_build_and_push_nginx + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: ./docker-nginx + file: ./docker-nginx/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: | + opengisch/qfieldcloud-nginx:${{ steps.prepare.outputs.docker_tag }} + opengisch/qfieldcloud-nginx:${{ steps.prepare.outputs.docker_commit }} diff --git a/.gitignore b/.gitignore index 88fcf6718..23825ee50 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ __pycache__/ .env docker-compose.override.yml client/projects -conf/nginx/certs/* +docker-nginx/certs/* conf/certbot/* Pipfile* **/site-packages diff --git a/README.md b/README.md index e974cdc28..e54e506d6 100644 --- a/README.md +++ b/README.md @@ -207,11 +207,11 @@ Note if you run tests using the `docker-compose.test.yml` configuration, the `ap ## Add root certificate -QFieldCloud will automatically generate a certificate and it's root certificate in `./config/nginx/certs`. However, you need to trust the root certificate first, so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance. +QFieldCloud will automatically generate a certificate and it's root certificate in `./docker-nginx/certs`. However, you need to trust the root certificate first, so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance. On Debian/Ubuntu, copy the root certificate to the directory with trusted certificates. Note the extension has been changed to `.crt`: - sudo cp ./conf/nginx/certs/rootCA.pem /usr/local/share/ca-certificates/rootCA.crt + sudo cp ./docker-nginx/certs/rootCA.pem /usr/local/share/ca-certificates/rootCA.crt Trust the newly added certificate: diff --git a/docker-app/qfieldcloud/core/management/commands/status.py b/docker-app/qfieldcloud/core/management/commands/status.py index a2979ebcb..c89f32310 100644 --- a/docker-app/qfieldcloud/core/management/commands/status.py +++ b/docker-app/qfieldcloud/core/management/commands/status.py @@ -1,4 +1,3 @@ -from django.conf import settings from django.core.management.base import BaseCommand from qfieldcloud.core import geodb_utils, utils @@ -17,8 +16,7 @@ def handle(self, *args, **options): results["storage"] = "ok" # Check if bucket exists (i.e. the connection works) try: - s3_client = utils.get_s3_client() - s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME) + utils.get_s3_bucket() except Exception: results["storage"] = "error" diff --git a/docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py b/docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py new file mode 100644 index 000000000..14ee9f840 --- /dev/null +++ b/docker-app/qfieldcloud/core/migrations/0074_auto_20240314_1805.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.24 on 2024-03-14 17:05 + +import migrate_sql.operations +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0073_project_packaging_offliner"), + ] + + operations = [ + migrate_sql.operations.AlterSQL( + name="core_delta_geom_trigger_func", + sql="\n CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()\n RETURNS trigger\n AS\n $$\n DECLARE\n delta_srid int;\n BEGIN\n SELECT CASE\n WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\\d{1,10}$'\n THEN\n REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\\D*', '', 'g')::int\n ELSE\n NULL\n END INTO delta_srid;\n\n IF delta_srid IS NOT NULL\n AND EXISTS(\n SELECT *\n FROM spatial_ref_sys\n WHERE auth_name = 'EPSG'\n AND auth_srid = delta_srid\n )\n THEN\n NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );\n NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 );\n ELSE\n NEW.old_geom := NULL;\n NEW.new_geom := NULL;\n END IF;\n\n IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.old_geom := ST_CurveToLine(NEW.old_geom);\n END IF;\n\n IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.new_geom := ST_CurveToLine(NEW.new_geom);\n END IF;\n\n RETURN NEW;\n END;\n $$\n LANGUAGE PLPGSQL\n ", + reverse_sql="\n CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()\n RETURNS trigger\n AS\n $$\n DECLARE\n srid int;\n BEGIN\n SELECT CASE\n WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\\d{1,10}$'\n THEN\n REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\\D*', '', 'g')::int\n ELSE\n NULL\n END INTO srid;\n NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );\n NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );\n\n IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.old_geom := ST_CurveToLine(NEW.old_geom);\n END IF;\n\n IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')\n THEN\n NEW.new_geom := ST_CurveToLine(NEW.new_geom);\n END IF;\n\n RETURN NEW;\n END;\n $$\n LANGUAGE PLPGSQL\n ", + ), + ] diff --git a/docker-app/qfieldcloud/core/sql_config.py b/docker-app/qfieldcloud/core/sql_config.py index 98c19000e..2bce6bc29 100644 --- a/docker-app/qfieldcloud/core/sql_config.py +++ b/docker-app/qfieldcloud/core/sql_config.py @@ -178,7 +178,7 @@ AS $$ DECLARE - srid int; + delta_srid int; BEGIN SELECT CASE WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\d{1,10}$' @@ -186,9 +186,22 @@ REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\D*', '', 'g')::int ELSE NULL - END INTO srid; - NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 ); - NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 ); + END INTO delta_srid; + + IF delta_srid IS NOT NULL + AND EXISTS( + SELECT * + FROM spatial_ref_sys + WHERE auth_name = 'EPSG' + AND auth_srid = delta_srid + ) + THEN + NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 ); + NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), delta_srid ), 4326 ); + ELSE + NEW.old_geom := NULL; + NEW.new_geom := NULL; + END IF; IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface') THEN diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index 26e2aca93..6ae29a2af 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -386,21 +386,6 @@ def get_project_package_files_count(project_id: str) -> int: return len(files) -def get_s3_object_url( - key: str, bucket: mypy_boto3_s3.service_resource.Bucket = get_s3_bucket() -) -> str: - """Returns the block storage URL for a given key. The key may not exist in the bucket. - - Args: - key (str): the object key - bucket (boto3.Bucket, optional): Bucket for that object. Defaults to `get_s3_bucket()`. - - Returns: - str: URL - """ - return f"{settings.STORAGE_ENDPOINT_URL}/{bucket.name}/{key}" - - def list_files( bucket: mypy_boto3_s3.service_resource.Bucket, prefix: str, diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py index bb9928715..ca3cc0d59 100644 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -196,7 +196,6 @@ def get_attachment_dir_prefix( def file_response( request: HttpRequest, key: str, - presigned: bool = False, expires: int = 60, version: str | None = None, as_attachment: bool = False, @@ -213,25 +212,22 @@ def file_response( https_port = http_host.split(":")[-1] if ":" in http_host else "443" if https_port == settings.WEB_HTTPS_PORT and not settings.IN_TEST_SUITE: - if presigned: - if as_attachment: - extra_params["ResponseContentType"] = "application/force-download" - extra_params[ - "ResponseContentDisposition" - ] = f'attachment;filename="{filename}"' - - url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url( - "get_object", - Params={ - **extra_params, - "Key": key, - "Bucket": qfieldcloud.core.utils.get_s3_bucket().name, - }, - ExpiresIn=expires, - HttpMethod="GET", - ) - else: - url = qfieldcloud.core.utils.get_s3_object_url(key) + if as_attachment: + extra_params["ResponseContentType"] = "application/force-download" + extra_params[ + "ResponseContentDisposition" + ] = f'attachment;filename="{filename}"' + + url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url( + "get_object", + Params={ + **extra_params, + "Key": key, + "Bucket": qfieldcloud.core.utils.get_s3_bucket().name, + }, + ExpiresIn=expires, + HttpMethod="GET", + ) # Let's NGINX handle the redirect to the storage and streaming the file contents back to the client response = HttpResponse() @@ -293,7 +289,6 @@ def upload_user_avatar( file, key, { - "ACL": "public-read", "ContentType": mimetype.value, }, ) @@ -357,8 +352,6 @@ def upload_project_thumbail( file, key, { - # TODO most probably this is not public-read, since the project might be private - "ACL": "public-read", "ContentType": mimetype, }, ) diff --git a/docker-app/qfieldcloud/core/views/files_views.py b/docker-app/qfieldcloud/core/views/files_views.py index bc0e6ade1..a9fcf49c5 100644 --- a/docker-app/qfieldcloud/core/views/files_views.py +++ b/docker-app/qfieldcloud/core/views/files_views.py @@ -207,7 +207,6 @@ def get(self, request, projectid, filename): return utils2.storage.file_response( request, key, - presigned=True, expires=600, version=version, as_attachment=True, @@ -374,7 +373,7 @@ class ProjectMetafilesView(views.APIView): def get(self, request, projectid, filename): key = utils.safe_join(f"projects/{projectid}/meta/", filename) - return utils2.storage.file_response(request, key, presigned=True) + return utils2.storage.file_response(request, key) @extend_schema_view( diff --git a/docker-app/qfieldcloud/core/views/package_views.py b/docker-app/qfieldcloud/core/views/package_views.py index 297f4ac45..744b003f8 100644 --- a/docker-app/qfieldcloud/core/views/package_views.py +++ b/docker-app/qfieldcloud/core/views/package_views.py @@ -184,9 +184,7 @@ def get(self, request, project_id, filename): key = f"projects/{project_id}/files/{filename}" # NOTE the `expires` kwarg is sending the `Expires` header to the client, keep it a low value (in seconds). - return storage.file_response( - request, key, presigned=True, expires=10, as_attachment=True - ) + return storage.file_response(request, key, expires=10, as_attachment=True) @extend_schema_view( diff --git a/docker-app/qfieldcloud/core/views/status_views.py b/docker-app/qfieldcloud/core/views/status_views.py index 6febe4a1f..d1c3ef406 100644 --- a/docker-app/qfieldcloud/core/views/status_views.py +++ b/docker-app/qfieldcloud/core/views/status_views.py @@ -1,4 +1,3 @@ -from django.conf import settings from django.core.cache import cache from drf_spectacular.utils import extend_schema, extend_schema_view from qfieldcloud.core import geodb_utils, utils @@ -25,8 +24,7 @@ def get(self, request): results["storage"] = "ok" # Check if bucket exists (i.e. the connection works) try: - s3_client = utils.get_s3_client() - s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME) + utils.get_s3_bucket() except Exception: results["storage"] = "error" diff --git a/docker-app/qfieldcloud/subscription/admin.py b/docker-app/qfieldcloud/subscription/admin.py index 05a7ad9d6..3ffb8bb65 100644 --- a/docker-app/qfieldcloud/subscription/admin.py +++ b/docker-app/qfieldcloud/subscription/admin.py @@ -1,4 +1,5 @@ from datetime import timedelta +from typing import Iterable from django import forms from django.contrib import admin @@ -115,6 +116,21 @@ def save(self, commit=True): class SubscriptionAdmin(QFieldCloudModelAdmin): form = SubscriptionModelForm + fields = ( + "plan", + "account", + "active_since", + "active_until", + "billing_cycle_anchor_at", + "current_period_since", + "current_period_until", + "notes", + "created_at", + "created_by", + "updated_at", + "requested_cancel_at", + ) + list_display = ( "id", "account__link", @@ -148,7 +164,35 @@ class SubscriptionAdmin(QFieldCloudModelAdmin): "account__user__username__iexact", ) - @admin.display(description="User") + def get_fields( + self, request: HttpRequest, obj: Subscription | None = None + ) -> Iterable[str]: + if obj is not None: + return ( + "plan__link", + "account__link", + "promotion__link", + *self.fields[3:], + ) + + return self.fields + + def get_readonly_fields( + self, request: HttpRequest, obj: Subscription | None = None + ) -> Iterable[str]: + if obj is not None: + return ( + *self.readonly_fields, + "plan", + "account", + "plan__link", + "account__link", + "promotion__link", + ) + + return self.readonly_fields + + @admin.display(description="Account") def account__link(self, instance): return model_admin_url( instance.account.user, instance.account.user.username_with_full_name diff --git a/docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html b/docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html new file mode 100644 index 000000000..b3772bcc1 --- /dev/null +++ b/docker-app/qfieldcloud/subscription/templates/admin/subscription_change_form.html @@ -0,0 +1,8 @@ +{% extends 'admin/change_form.html' %} +{% load i18n %} + +{% block form_top %} + {% if change %} + NOTE: For changing Plan create a new Subscription. First cancel the current subscription by setting "Active until". + {% endif %} +{% endblock %} diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt index bc5de8fb7..5372a04e6 100644 --- a/docker-app/requirements.txt +++ b/docker-app/requirements.txt @@ -10,7 +10,7 @@ cffi==1.15.1 charset-normalizer==2.0.9 coreapi==2.3.3 coreschema==0.0.4 -cryptography==42.0.2 +cryptography==42.0.4 defusedxml==0.7.1 Deprecated==1.2.13 Django==3.2.24 diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 606861ea7..d69bcff6c 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -160,7 +160,7 @@ def run(self): self.job.output = output.decode("utf-8") self.job.feedback = feedback self.job.status = Job.Status.FAILED - self.job.save(update_fields=["output", "feedback"]) + self.job.save(update_fields=["output", "feedback", "status"]) logger.info( "Set job status to `failed` due to being killed by the docker engine.", ) diff --git a/docker-compose.yml b/docker-compose.yml index 28df94370..479d5289e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,14 +88,11 @@ services: ofelia.job-exec.runcrons.command: python manage.py runcrons nginx: - image: nginx:stable + build: + context: ./docker-nginx restart: unless-stopped volumes: - - ./conf/nginx/pages/:/var/www/html/pages/ - - ./conf/nginx/templates/:/etc/nginx/templates/ - - ./conf/nginx/certs/:/etc/nginx/certs/:ro - - ./conf/nginx/options-ssl-nginx.conf:/etc/nginx/options-ssl-nginx.conf - - ./conf/nginx/ssl-dhparams.pem:/etc/nginx/ssl-dhparams.pem + - ./docker-nginx/certs/:/etc/nginx/certs/:ro - certbot_www:/var/www/certbot ports: - ${WEB_HTTP_PORT}:80 @@ -119,7 +116,7 @@ services: environment: domain: ${QFIELDCLOUD_HOST} volumes: - - ./conf/nginx/certs/:/root/.local/share/mkcert/ + - ./docker-nginx/certs/:/root/.local/share/mkcert/ command: /bin/sh -c 'mkcert -install && for i in $$(echo $$domain | sed "s/,/ /g"); do [ ! -f /root/.local/share/mkcert/$$i.pem ] && mkcert $$i; done && tail -f -n0 /etc/hosts' certbot: @@ -134,6 +131,8 @@ services: build: context: ./docker-qgis network: host + args: + DEBUG_BUILD: ${DEBUG} tty: true command: bash -c "echo QGIS built" logging: *default-logging diff --git a/docker-nginx/Dockerfile b/docker-nginx/Dockerfile new file mode 100644 index 000000000..200d8b581 --- /dev/null +++ b/docker-nginx/Dockerfile @@ -0,0 +1,6 @@ +FROM nginx:stable + +COPY pages /var/www/html/pages/ +COPY templates/ /etc/nginx/templates/ +COPY options-ssl-nginx.conf /etc/nginx/options-ssl-nginx.conf +COPY ssl-dhparams.pem /etc/nginx/ssl-dhparams.pem diff --git a/conf/nginx/options-ssl-nginx.conf b/docker-nginx/options-ssl-nginx.conf similarity index 100% rename from conf/nginx/options-ssl-nginx.conf rename to docker-nginx/options-ssl-nginx.conf diff --git a/conf/nginx/pages/403.html b/docker-nginx/pages/403.html similarity index 100% rename from conf/nginx/pages/403.html rename to docker-nginx/pages/403.html diff --git a/conf/nginx/pages/404.html b/docker-nginx/pages/404.html similarity index 100% rename from conf/nginx/pages/404.html rename to docker-nginx/pages/404.html diff --git a/conf/nginx/pages/500.html b/docker-nginx/pages/500.html similarity index 100% rename from conf/nginx/pages/500.html rename to docker-nginx/pages/500.html diff --git a/conf/nginx/pages/sad_nyuki.svg b/docker-nginx/pages/sad_nyuki.svg similarity index 100% rename from conf/nginx/pages/sad_nyuki.svg rename to docker-nginx/pages/sad_nyuki.svg diff --git a/conf/nginx/ssl-dhparams.pem b/docker-nginx/ssl-dhparams.pem similarity index 100% rename from conf/nginx/ssl-dhparams.pem rename to docker-nginx/ssl-dhparams.pem diff --git a/conf/nginx/templates/default.conf.template b/docker-nginx/templates/default.conf.template similarity index 100% rename from conf/nginx/templates/default.conf.template rename to docker-nginx/templates/default.conf.template diff --git a/docker-qgis/Dockerfile b/docker-qgis/Dockerfile index a3b1c85eb..eca4f825c 100644 --- a/docker-qgis/Dockerfile +++ b/docker-qgis/Dockerfile @@ -1,13 +1,54 @@ -FROM qgis/qgis:final-3_34_2 +# NOTE if the ubuntu version is changed, also change it in `Suites:` in apt sources, and in `QGIS_VERSION` +FROM ubuntu:jammy -RUN apt-get update \ - && apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ +# Install dependencies needed to add QGIS repository +RUN apt update \ + && apt upgrade -y \ + && apt install -y gnupg software-properties-common wget + +# Add QGIS GPG key +RUN wget -O /etc/apt/keyrings/qgis-archive-keyring.gpg https://download.qgis.org/downloads/qgis-archive-keyring.gpg + +# Add QGIS repository +COPY < dict: logger.info("Extracting QGIS project layer data…") - project_filename = str(project_filename) - project = QgsProject.instance() - - with set_bad_layer_handler(project): - project.read(project_filename) - layers_by_id: dict = get_layers_data(project) + project = open_qgis_project(str(project_filename)) + layers_by_id: dict = get_layers_data(project) logger.info( f"QGIS project layer data\n{layers_data_to_string(layers_by_id)}", @@ -155,6 +147,23 @@ def _extract_layer_data(project_filename: Union[str, Path]) -> dict: return layers_by_id +def _open_read_only_project(project_filename: str) -> QgsProject: + flags = ( + # TODO we use `QgsProject` read flags, as the ones in `Qgis.ProjectReadFlags` do not work in QGIS 3.34.2 + QgsProject.ReadFlags() + | QgsProject.ForceReadOnlyLayers + | QgsProject.FlagDontLoadLayouts + | QgsProject.FlagDontLoad3DViews + | QgsProject.DontLoadProjectStyles + ) + return open_qgis_project( + project_filename, + force_reload=True, + disable_feature_count=True, + flags=flags, + ) + + def cmd_package_project(args: argparse.Namespace): workflow = Workflow( id="package_project", @@ -326,9 +335,9 @@ def cmd_process_projectfile(args: argparse.Namespace): id="opening_check", name="Opening Check", arguments={ - "project_filename": WorkDirPath("files", args.project_file), + "project_filename": WorkDirPathAsStr("files", args.project_file), }, - method=qfc_worker.process_projectfile.load_project_file, + method=_open_read_only_project, return_names=["project"], ), Step( @@ -345,7 +354,7 @@ def cmd_process_projectfile(args: argparse.Namespace): id="generate_thumbnail_image", name="Generate Thumbnail Image", arguments={ - "project": StepOutput("opening_check", "project"), + "project_filename": WorkDirPathAsStr("files", args.project_file), "thumbnail_filename": Path("/io/thumbnail.png"), }, method=qfc_worker.process_projectfile.generate_thumbnail, diff --git a/docker-qgis/qfc_worker/process_projectfile.py b/docker-qgis/qfc_worker/process_projectfile.py index 261bd5c3a..8095bd372 100644 --- a/docker-qgis/qfc_worker/process_projectfile.py +++ b/docker-qgis/qfc_worker/process_projectfile.py @@ -1,10 +1,17 @@ import logging from pathlib import Path +from typing import Callable from xml.etree import ElementTree -from qgis.core import QgsMapRendererParallelJob, QgsMapSettings, QgsProject -from qgis.PyQt.QtCore import QEventLoop, QSize -from qgis.PyQt.QtGui import QColor +from qgis.core import ( + QgsLayerTree, + QgsMapRendererCustomPainterJob, + QgsMapSettings, + QgsProject, +) +from qgis.PyQt.QtCore import QSize +from qgis.PyQt.QtGui import QColor, QImage, QPainter +from qgis.PyQt.QtXml import QDomDocument from .utils import ( FailedThumbnailGenerationException, @@ -44,18 +51,6 @@ def check_valid_project_file(project_filename: Path) -> None: logger.info("QGIS project file is valid!") -def load_project_file(project_filename: Path) -> QgsProject: - logger.info("Open QGIS project file…") - - project = QgsProject.instance() - if not project.read(str(project_filename)): - raise InvalidXmlFileException(error=project.error()) - - logger.info("QGIS project file opened!") - - return project - - def extract_project_details(project: QgsProject) -> dict[str, str]: """Extract project details""" logger.info("Extract project details…") @@ -65,33 +60,52 @@ def extract_project_details(project: QgsProject) -> dict[str, str]: logger.info("Reading QGIS project file…") map_settings = QgsMapSettings() - def on_project_read(doc): - r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255) - g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255) - b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255) - background_color = QColor(r, g, b) - map_settings.setBackgroundColor(background_color) - - details["background_color"] = background_color.name() - - nodes = doc.elementsByTagName("mapcanvas") - - for i in range(nodes.size()): - node = nodes.item(i) - element = node.toElement() - if ( - element.hasAttribute("name") - and element.attribute("name") == "theMapCanvas" - ): - map_settings.readXml(node) - - map_settings.setRotation(0) - map_settings.setOutputSize(QSize(1024, 768)) - - details["extent"] = map_settings.extent().asWktPolygon() + def on_project_read_wrapper( + tmp_project: QgsProject, + ) -> Callable[[QDomDocument], None]: + def on_project_read(doc: QDomDocument) -> None: + r, _success = tmp_project.readNumEntry("Gui", "/CanvasColorRedPart", 255) + g, _success = tmp_project.readNumEntry("Gui", "/CanvasColorGreenPart", 255) + b, _success = tmp_project.readNumEntry("Gui", "/CanvasColorBluePart", 255) + background_color = QColor(r, g, b) + map_settings.setBackgroundColor(background_color) + + details["background_color"] = background_color.name() + + nodes = doc.elementsByTagName("mapcanvas") + + for i in range(nodes.size()): + node = nodes.item(i) + element = node.toElement() + if ( + element.hasAttribute("name") + and element.attribute("name") == "theMapCanvas" + ): + map_settings.readXml(node) + + map_settings.setRotation(0) + map_settings.setOutputSize(QSize(1024, 768)) + + details["extent"] = map_settings.extent().asWktPolygon() + + return on_project_read + + # NOTE use a temporary project to get the project extent and background color + # as we can disable resolving layers, which results in great speed gains + tmp_project = QgsProject() + tmp_project_read_flags = ( + # TODO we use `QgsProject` read flags, as the ones in `Qgis.ProjectReadFlags` do not work in QGIS 3.34.2 + QgsProject.ReadFlags() + | QgsProject.FlagDontResolveLayers + | QgsProject.FlagDontLoadLayouts + | QgsProject.FlagDontLoad3DViews + | QgsProject.DontLoadProjectStyles + ) + tmp_project.readProject.connect(on_project_read_wrapper(tmp_project)) + tmp_project.read(project.fileName(), tmp_project_read_flags) - project.readProject.connect(on_project_read) - project.read(project.fileName()) + # NOTE force delete the `QgsProject`, otherwise the `QgsApplication` might be deleted by the time the project is garbage collected + del tmp_project details["crs"] = project.crs().authid() details["project_name"] = project.title() @@ -111,61 +125,86 @@ def on_project_read(doc): return details -def generate_thumbnail(project: QgsProject, thumbnail_filename: Path) -> None: +def generate_thumbnail(project_filename: str, thumbnail_filename: Path) -> None: """Create a thumbnail for the project As from https://docs.qgis.org/3.16/en/docs/pyqgis_developer_cookbook/composer.html#simple-rendering Args: - project (QgsProject) + project_filename (str) thumbnail_filename (Path) """ logger.info("Generate project thumbnail image…") map_settings = QgsMapSettings() - layer_tree = project.layerTreeRoot() - def on_project_read(doc): - r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255) - g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255) - b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255) - map_settings.setBackgroundColor(QColor(r, g, b)) - - nodes = doc.elementsByTagName("mapcanvas") - - for i in range(nodes.size()): - node = nodes.item(i) - element = node.toElement() - if ( - element.hasAttribute("name") - and element.attribute("name") == "theMapCanvas" - ): - map_settings.readXml(node) - - map_settings.setRotation(0) - map_settings.setTransformContext(project.transformContext()) - map_settings.setPathResolver(project.pathResolver()) - map_settings.setOutputSize(QSize(100, 100)) - map_settings.setLayers(reversed(list(layer_tree.customLayerOrder()))) - # print(f'output size: {map_settings.outputSize().width()} {map_settings.outputSize().height()}') - # print(f'layers: {[layer.name() for layer in map_settings.layers()]}') - - project.readProject.connect(on_project_read) - project.read(project.fileName()) - - renderer = QgsMapRendererParallelJob(map_settings) - - event_loop = QEventLoop() - renderer.finished.connect(event_loop.quit) - renderer.start() - - event_loop.exec_() + def on_project_read_wrapper( + tmp_project: QgsProject, + tmp_layer_tree: QgsLayerTree, + ) -> Callable[[QDomDocument], None]: + def on_project_read(doc: QDomDocument) -> None: + r, _success = tmp_project.readNumEntry("Gui", "/CanvasColorRedPart", 255) + g, _success = tmp_project.readNumEntry("Gui", "/CanvasColorGreenPart", 255) + b, _success = tmp_project.readNumEntry("Gui", "/CanvasColorBluePart", 255) + map_settings.setBackgroundColor(QColor(r, g, b)) + + nodes = doc.elementsByTagName("mapcanvas") + + for i in range(nodes.size()): + node = nodes.item(i) + element = node.toElement() + if ( + element.hasAttribute("name") + and element.attribute("name") == "theMapCanvas" + ): + map_settings.readXml(node) + + map_settings.setRotation(0) + map_settings.setTransformContext(tmp_project.transformContext()) + map_settings.setPathResolver(tmp_project.pathResolver()) + map_settings.setOutputSize(QSize(100, 100)) + map_settings.setLayers(reversed(list(tmp_layer_tree.customLayerOrder()))) + + return on_project_read + + # NOTE use a temporary project to generate the layer rendering with improved speed + tmp_project = QgsProject() + tmp_layer_tree = tmp_project.layerTreeRoot() + tmp_project_read_flags = ( + # TODO we use `QgsProject` read flags, as the ones in `Qgis.ProjectReadFlags` do not work in QGIS 3.34.2 + QgsProject.ReadFlags() + | QgsProject.ForceReadOnlyLayers + | QgsProject.FlagDontLoadLayouts + | QgsProject.FlagDontLoad3DViews + | QgsProject.DontLoadProjectStyles + ) + tmp_project.readProject.connect( + on_project_read_wrapper(tmp_project, tmp_layer_tree) + ) + tmp_project.read( + project_filename, + tmp_project_read_flags, + ) - img = renderer.renderedImage() + img = QImage(map_settings.outputSize(), QImage.Format_ARGB32) + painter = QPainter(img) + job = QgsMapRendererCustomPainterJob(map_settings, painter) + # NOTE we use `renderSynchronously` as it does not crash and produces the thumbnail. + # `waitForFinishedWithEventLoop` hangs forever and `waitForFinished` produces blank thumbnail, so don't use them! + job.renderSynchronously() if not img.save(str(thumbnail_filename)): raise FailedThumbnailGenerationException(reason="Failed to save.") + painter.end() + + # NOTE force delete the `QgsMapRendererCustomPainterJob`, `QPainter` and `QImage` because we are paranoid with Cpp objects around + del job + del painter + del img + # NOTE force delete the `QgsProject`, otherwise the `QgsApplication` might be deleted by the time the project is garbage collected + del tmp_project + logger.info("Project thumbnail image generated!") diff --git a/docker-qgis/qfc_worker/utils.py b/docker-qgis/qfc_worker/utils.py index ff9a949ef..5b7f1986f 100644 --- a/docker-qgis/qfc_worker/utils.py +++ b/docker-qgis/qfc_worker/utils.py @@ -1,4 +1,5 @@ import atexit +import gc import hashlib import inspect import io @@ -12,13 +13,14 @@ import tempfile import traceback import uuid +import xml.etree.ElementTree as ET from contextlib import contextmanager from datetime import datetime from pathlib import Path from typing import IO, Any, Callable, NamedTuple, Optional from libqfieldsync.layer import LayerSource -from libqfieldsync.utils.bad_layer_handler import bad_layer_handler +from libqfieldsync.utils.bad_layer_handler import set_bad_layer_handler from qfieldcloud_sdk import sdk from qgis.core import ( Qgis, @@ -26,7 +28,9 @@ QgsMapLayer, QgsMapSettings, QgsProject, + QgsProjectArchive, QgsProviderRegistry, + QgsZipUtils, ) from qgis.PyQt import QtCore, QtGui from tabulate import tabulate @@ -182,9 +186,6 @@ def exitQgis(): logging.info("QGIS app started!") - # we set the `bad_layer_handler` and assume we always have only one single `QgsProject` instance within the job's life - QgsProject.instance().setBadLayerHandler(bad_layer_handler) - return QGISAPP @@ -198,13 +199,113 @@ def stop_app(): if "QGISAPP" not in globals(): return - QgsProject.instance().read("") + QgsProject.instance().clear() if QGISAPP is not None: logging.info("Stopping QGIS app…") + + # NOTE we force run the GB just to make sure there are no dangling QGIS objects when we delete the QGIS application + gc.collect() + QGISAPP.exitQgis() + del QGISAPP + logging.info("Deleted QGIS app!") + + +def open_qgis_project( + project_filename: str, + force_reload: bool = False, + disable_feature_count: bool = False, + flags: Qgis.ProjectReadFlags = Qgis.ProjectReadFlags(), +) -> QgsProject: + logging.info(f'Loading QGIS project "{project_filename}"…') + + if not Path(project_filename).exists(): + raise FileNotFoundError(project_filename) + + project = QgsProject.instance() + + if project.fileName() == str(project_filename) and not force_reload: + logging.info( + f'Skip loading QGIS project "{project_filename}", it is already loaded' + ) + return project + + if disable_feature_count: + strip_feature_count_from_project_xml(project_filename) + + with set_bad_layer_handler(project): + if not project.read(str(project_filename), flags): + logging.error(f'Failed to load QGIS project "{project_filename}"!') + + project.setFileName("") + + raise Exception(f"Unable to open project with QGIS: {project_filename}") + + logging.info("Project loaded.") + + return project + + +def strip_feature_count_from_project_xml(project_filename: str) -> None: + """Rewrites project XML file with feature count disabled. + + Args: + project_filename (str): filename of the QGIS project file (.qgs or .qgz) + """ + archive = None + xml_file = project_filename + if QgsZipUtils.isZipFile(project_filename): + logging.info("The project file is zipped as .qgz, unzipping…") + + archive = QgsProjectArchive() + + if archive.unzip(project_filename): + logging.info("The project file is unzipped successfully!") + xml_file = archive.projectFile() + else: + logging.error("The project file is unzipping failed!") + + raise Exception(f"Failed to unzip {project_filename} file!") + + logging.info("Parsing QGIS project file XML…") + + tree = ET.parse(xml_file) + root = tree.getroot() + + logging.info("QGIS project parsed!") + + for node in root.findall(".//legendlayer"): + node.set("showFeatureCount", "0") + + for node in root.findall( + './/layer-tree-layer/customproperties/Option[@type="Map"]/Option[@name="showFeatureCount"]' + ): + node.set("value", "0") + + logging.info("Writing the QGIS project XML into a file…") + + if archive: + tmp_filename = tempfile.NamedTemporaryFile().name + tree.write(tmp_filename, short_empty_elements=False) + + if not archive.clearProjectFile(): + raise Exception("Failed to clear project path from the archive!") + + if not Path(tmp_filename).rename(xml_file): + raise Exception("Failed to move the temp xml file into archive dir!") + + archive.addFile(xml_file) + + if not archive.zip(project_filename): + raise Exception(f"Failed to zip {project_filename} file!") + else: + tree.write(xml_file, short_empty_elements=False) + + logging.info("QGIS project file re-written!") + def download_project( project_id: str, destination: Path = None, skip_attachments: bool = True @@ -394,12 +495,12 @@ def __init__(self, step_id: str, return_name: str): self.return_name = return_name -class WorkDirPath: +class WorkDirPathBase: def __init__(self, *parts: str, mkdir: bool = False) -> None: self.parts = parts self.mkdir = mkdir - def eval(self, root: Path) -> Path: + def eval(self, root: Path) -> Path | str: path = root.joinpath(*self.parts) if self.mkdir: @@ -408,6 +509,16 @@ def eval(self, root: Path) -> Path: return path +class WorkDirPath(WorkDirPathBase): + def eval(self, root: Path) -> Path: + return Path(super().eval(root)) + + +class WorkDirPathAsStr(WorkDirPathBase): + def eval(self, root: Path) -> str: + return str(super().eval(root)) + + @contextmanager def logger_context(step: Step): log_uuid = uuid.uuid4() @@ -550,7 +661,7 @@ def run_workflow( for name, value in arguments.items(): if isinstance(value, StepOutput): arguments[name] = step_returns[value.step_id][value.return_name] - elif isinstance(value, WorkDirPath): + elif isinstance(value, WorkDirPathBase): arguments[name] = value.eval(root_workdir) return_values = step.method(**arguments) diff --git a/docker-qgis/requirements_libqfieldsync.txt b/docker-qgis/requirements_libqfieldsync.txt index 67c1d9893..e85c6479c 100644 --- a/docker-qgis/requirements_libqfieldsync.txt +++ b/docker-qgis/requirements_libqfieldsync.txt @@ -1 +1 @@ -libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@e20ff5be177c3a221ba4641d8b73a26a35016f66 +libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@5ed1f872f2449657dfb91a00e9da48469d62e226 diff --git a/scripts/init_letsencrypt.sh b/scripts/init_letsencrypt.sh index 2d1f904b7..10f03705d 100755 --- a/scripts/init_letsencrypt.sh +++ b/scripts/init_letsencrypt.sh @@ -9,10 +9,10 @@ set +o allexport CONFIG_PATH="${CONFIG_PATH:-'./conf'}" -if [ ! -e "$CONFIG_PATH/nginx/options-ssl-nginx.conf" ] || [ ! -e "$CONFIG_PATH/nginx/ssl-dhparams.pem" ]; then +if [ ! -e "docker-nginx/options-ssl-nginx.conf" ] || [ ! -e "docker-nginx/ssl-dhparams.pem" ]; then echo "### Downloading recommended TLS parameters ..." - curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$CONFIG_PATH/nginx/options-ssl-nginx.conf" - curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$CONFIG_PATH/nginx/ssl-dhparams.pem" + curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "docker-nginx/options-ssl-nginx.conf" + curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "docker-nginx/ssl-dhparams.pem" echo fi @@ -34,8 +34,8 @@ docker compose run --rm --entrypoint "\ echo echo "### Copy the certificate and key to their final destination ..." -cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/fullchain.pem ${CONFIG_PATH}/nginx/certs/${QFIELDCLOUD_HOST}.pem -cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/privkey.pem ${CONFIG_PATH}/nginx/certs/${QFIELDCLOUD_HOST}-key.pem +cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/fullchain.pem docker-nginx/certs/${QFIELDCLOUD_HOST}.pem +cp ${CONFIG_PATH}/certbot/conf/live/${QFIELDCLOUD_HOST}/privkey.pem docker-nginx/certs/${QFIELDCLOUD_HOST}-key.pem echo echo "### Reloading nginx ..."