Merge branch 'master' into release

.env.example

@@ -88,7 +88,9 @@ EMAIL_HOST_USER=user
 EMAIL_HOST_PASSWORD=password
 DEFAULT_FROM_EMAIL="webmaster@localhost"
 
-QFIELDCLOUD_DEFAULT_NETWORK=qfieldcloud_default
+# Docker compose default network also used by the docker in docker workers
+# If empty value, a default name will be generated at build time, for example `qfieldcloud_default`.
+# QFIELDCLOUD_DEFAULT_NETWORK=""
 
 # Admin URI. Requires slash in the end. Please use something that is hard to guess.
 QFIELDCLOUD_ADMIN_URI=admin/
@@ -122,6 +124,11 @@ QFIELDCLOUD_DEFAULT_TIME_ZONE="Europe/Zurich"
 # DEFAULT: ""
 QFIELDCLOUD_LIBQFIELDSYNC_VOLUME_PATH=""
 
+# QFieldCloud SDK volume path to be mounted by the `worker_wrapper` into `worker` containers.
+# If empty value or invalid value, the pip installed version defined in `requirements_libqfieldsync.txt` will be used.
+# DEFAULT: ""
+QFIELDCLOUD_QFIELDCLOUD_SDK_VOLUME_PATH=""
+
 # The Django development port. Not used in production.
 # DEFAULT: 8011
 DJANGO_DEV_PORT=8011
@@ -162,7 +169,3 @@ DEBUG_DEBUGPY_APP_PORT=5678
 # Debugpy port used for the `worker_wrapper` service
 # DEFAULT: 5679
 DEBUG_DEBUGPY_WORKER_WRAPPER_PORT=5679
-
-# Path to the nginx, letsencrypt, etc configuration files, used by script in `./scripts/`.
-# DEFAULT: ./conf
-CONFIG_PATH=./conf

.gitignore

@@ -13,3 +13,4 @@ conf/certbot/*
 Pipfile*
 **/site-packages
 docker-qgis/libqfieldsync
+docker-qgis/qfieldcloud-sdk-python

docker-app/qfieldcloud/core/admin.py

@@ -19,7 +19,7 @@
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import admin_urlname
 from django.contrib.admin.views.main import ChangeList
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import PermissionDenied, ValidationError
 from django.db.models import Q, QuerySet
 from django.db.models.fields.json import JSONField
 from django.db.models.functions import Lower
@@ -31,6 +31,7 @@
 from django.urls import path, reverse
 from django.utils.decorators import method_decorator
 from django.utils.html import escape, format_html
+from django.utils.http import urlencode
 from django.utils.safestring import SafeText
 from django.utils.translation import gettext_lazy as _
 from django.views.decorators.cache import never_cache
@@ -48,14 +49,15 @@
  Person,
  Project,
  ProjectCollaborator,
+ Secret,
  Team,
  TeamMember,
  User,
  UserAccount,
 )
 from qfieldcloud.core.paginators import LargeTablePaginator
 from qfieldcloud.core.templatetags.filters import filesizeformat10
-from qfieldcloud.core.utils2 import delta_utils, jobs
+from qfieldcloud.core.utils2 import delta_utils, jobs, pg_service_file
 from rest_framework.authtoken.models import TokenProxy
 
 admin.site.unregister(LogEntry)
@@ -115,6 +117,15 @@ def has_delete_permission(self, request, obj=None):
  return super().has_delete_permission(request, obj)
 
 
+class QFieldCloudInlineAdmin(admin.TabularInline):
+ template = "admin/edit_inline/tabular_customized.html"
+
+ def get_formset(self, request, obj=None, **kwargs):
+ self.parent_obj = obj
+
+ return super().get_formset(request, obj, **kwargs)
+
+
 def admin_urlname_by_obj(value, arg):
  if isinstance(value, User):
  if value.is_person:
@@ -584,6 +595,132 @@ def queryset(self, request, queryset):
  return queryset.filter(owner__type=value)
 
 
+class ProjectSecretForm(ModelForm):
+ class Meta:
+ model = Secret
+ fields = ("project", "name", "type", "value", "created_by")
+
+ name = fields.CharField(widget=widgets.TextInput)
+ value = fields.CharField(widget=widgets.Textarea)
+
+ def get_initial_for_field(self, field, field_name):
+ if self.instance.pk and field_name == "value":
+ return ""
+ return super().get_initial_for_field(field, field_name)
+
+ def clean(self):
+ cleaned_data = super().clean()
+
+ if self.instance.pk:
+ type = self.instance.type
+ else:
+ type = cleaned_data.get("type")
+ if type == Secret.Type.PGSERVICE:
+ # validate the pg_service.conf
+ value = cleaned_data.get("value")
+ if value:
+ try:
+ pg_service_file.validate_pg_service_conf(value)
+ except ValidationError as err:
+ raise ValidationError({"value": err.message})
+
+ # ensure name with PGSERVICE_SECRET_NAME_PREFIX
+ name = cleaned_data.get("name")
+ if name and not name.startswith(
+ pg_service_file.PGSERVICE_SECRET_NAME_PREFIX
+ ):
+ cleaned_data[
+ "name"
+ ] = f"{pg_service_file.PGSERVICE_SECRET_NAME_PREFIX}{name}"
+
+ return cleaned_data
+
+
+class SecretAdmin(QFieldCloudModelAdmin):
+ model = Secret
+ form = ProjectSecretForm
+ fields = ("project", "name", "type", "value", "created_by")
+ readonly_fields = ("created_by",)
+ list_display = ("name", "type", "created_by__link", "project__name")
+ autocomplete_fields = ("project",)
+
+ search_fields = (
+ "name__icontains",
+ "project__name__icontains",
+ )
+
+ @admin.display(ordering="created_by", description=_("Created by"))
+ def created_by__link(self, instance):
+ return model_admin_url(instance.created_by)
+
+ @admin.display(ordering="project__name")
+ def project__name(self, instance):
+ return model_admin_url(instance.project, instance.project.name)
+
+ def get_readonly_fields(self, request, obj=None):
+ readonly_fields = super().get_readonly_fields(request, obj)
+
+ if obj:
+ return (*readonly_fields, "name", "type", "project")
+
+ return readonly_fields
+
+ def save_model(self, request, obj, form, change):
+ # only set created_by during the first save
+ if not change:
+ obj.created_by = request.user
+ super().save_model(request, obj, form, change)
+
+ def get_changeform_initial_data(self, request):
+ project_id = request.GET.get("project_id")
+
+ if project_id:
+ project = Project.objects.get(id=project_id)
+ else:
+ project = None
+
+ return {"project": project}
+
+
+class ProjectSecretInline(QFieldCloudInlineAdmin):
+ model = Secret
+ fields = ("link_to_secret", "type", "created_by")
+ readonly_fields = ("link_to_secret",)
+ max_num = 0
+ extra = 0
+
+ @admin.display(description=_("Name"))
+ def link_to_secret(self, obj):
+ url = reverse("admin:core_secret_change", args=[obj.pk])
+ return format_html('<a href="{}">{}</a>', url, obj.name)
+
+ def has_add_permission(self, request, obj=None):
+ return False
+
+ def has_change_permission(self, request, obj=None):
+ return False
+
+ def has_delete_permission(self, request, obj=None):
+ return False
+
+ @property
+ def bottom_html(self):
+ if self.parent_obj:
+ return format_html(
+ """
+ <a href="{url}?{query_params}" class="btn btn-default form-control">
+ <i class="fa fa-plus-circle"></i>
+ {text}
+ </a>
+ """,
+ url=reverse("admin:core_secret_add"),
+ query_params=urlencode({"project_id": self.parent_obj.pk}),
+ text="Add Secret",
+ )
+ else:
+ return ""
+
+
 class ProjectForm(ModelForm):
  project_files = fields.CharField(
  disabled=True, required=False, widget=ProjectFilesWidget
@@ -642,7 +779,7 @@ class ProjectAdmin(QFieldCloudModelAdmin):
  "data_last_packaged_at",
  "project_details__pre",
  )
- inlines = (ProjectCollaboratorInline,)
+ inlines = (ProjectCollaboratorInline, ProjectSecretInline)
  search_fields = (
  "id",
  "name__icontains",
@@ -652,6 +789,8 @@ class ProjectAdmin(QFieldCloudModelAdmin):
 
  ordering = ("-updated_at",)
 
+ change_form_template = "admin/project_change_form.html"
+
  def get_form(self, *args, **kwargs):
  help_texts = {
  "file_storage_bytes": _(
@@ -778,6 +917,7 @@ class JobAdmin(QFieldCloudModelAdmin):
  "project__name__iexact",
  "project__owner__username__iexact",
  "id",
+ "project__id__iexact",
  )
  readonly_fields = (
  "project",
@@ -1365,6 +1505,7 @@ class LogEntryAdmin(
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Team, TeamAdmin)
 admin.site.register(Project, ProjectAdmin)
+admin.site.register(Secret, SecretAdmin)
 admin.site.register(Delta, DeltaAdmin)
 admin.site.register(Job, JobAdmin)
 admin.site.register(Geodb, GeodbAdmin)

docker-app/qfieldcloud/core/cron.py

@@ -7,7 +7,7 @@
 from invitations.utils import get_invitation_model
 from sentry_sdk import capture_message
 
-from ..core.models import Job, Project
+from ..core.models import ApplyJob, ApplyJobDelta, Delta, Job, Project
 from ..core.utils2 import storage
 from .invitations_utils import send_invitation
 
@@ -60,6 +60,22 @@ def do(self):
  capture_message(
  f'Job "{job.id}" was with status "{job.status}", but worker container no longer exists. Job unexpectedly terminated.'
  )
+ if job.type == Job.Type.DELTA_APPLY:
+ ApplyJob.objects.get(id=job.id).deltas_to_apply.update(
+ last_status=Delta.Status.ERROR,
+ last_feedback=None,
+ last_modified_pk=None,
+ last_apply_attempt_at=job.started_at,
+ last_apply_attempt_by=job.created_by,
+ )
+
+ ApplyJobDelta.objects.filter(
+ apply_job_id=job.id,
+ ).update(
+ status=Delta.Status.ERROR,
+ feedback=None,
+ modified_pk=None,
+ )
 
  jobs.update(
  status=Job.Status.FAILED,

docker-app/qfieldcloud/core/templates/admin/edit_inline/tabular_customized.html

@@ -0,0 +1,5 @@
+{% if qfc_admin_inline_included != 1 %}
+ {% include "admin/edit_inline/tabular_extended.html" with qfc_admin_inline_included=1 %}
+{% endif %}
+
+{{ fieldset.opts.bottom_html }}

docker-app/qfieldcloud/core/templates/admin/edit_inline/tabular_extended.html

@@ -0,0 +1 @@
+{% extends "admin/edit_inline/tabular.html" %}

docker-app/qfieldcloud/core/templates/admin/project_change_form.html

@@ -0,0 +1,10 @@
+{% extends 'admin/change_form.html' %}
+{% load i18n %}
+
+{% block submit_buttons_bottom %}
+ {{ block.super }}
+
+ <div class="submit-row">
+ <a href="{% url 'admin:core_job_changelist' %}?q={{original.id}}">{% trans 'Project jobs' %}</a>
+ </div>
+{% endblock %}

docker-app/qfieldcloud/core/templates/admin/project_files_widget.html

@@ -4,10 +4,10 @@
  <dialog class="qfc-admin-project-files-dialog">
  <pre></pre>
 
- <button type="button">OK</button>
+ <button class="btn btn-primary" type="button">OK</button>
  </dialog>
 
- <button class="qfc-admin-projects-files-actions-reload-btn" type="button">
+ <button class="qfc-admin-projects-files-actions-reload-btn btn btn-sm btn-info" type="button">
  {% trans 'Refresh Files List' %}
  </button>
  <span class="qfc-admin-project-files-count"></span>
@@ -44,16 +44,16 @@
  <td></td>
  <td class="qfc-admin-project-files-text-right"></td>
  <td>
- <button type="button" class="qfc-admin-projects-files-actions-info-btn">{% trans 'Details' %}</button>
+ <button type="button" class="qfc-admin-projects-files-actions-info-btn btn btn-sm btn-outline-info">{% trans 'Details' %}</button>
  </td>
  <td>
- <select>
+ <select class="custom-select-sm">
  <option value="">{% trans 'Select a version...' %}</option>
  </select>
  </td>
  <td>
- <button type="button" class="qfc-admin-projects-files-actions-download-btn" title="{% trans 'Download the selected version or, by default the latest' %}">{% trans 'Download' %}</button>
- <button type="button" class="qfc-admin-projects-files-actions-delete-btn" data-csrf-token="">{% trans 'Delete' %}</button>
+ <button type="button" class="qfc-admin-projects-files-actions-download-btn btn btn-sm btn-outline-info" title="{% trans 'Download the selected version or, by default the latest' %}">{% trans 'Download' %}</button>
+ <button type="button" class="qfc-admin-projects-files-actions-delete-btn btn btn-sm btn-outline-danger" data-csrf-token="">{% trans 'Delete' %}</button>
  </td>
  </tr>
  </template>
@@ -68,27 +68,6 @@
  overflow-y: auto;
  }
 
- .qfc-admin-projects-files-actions-reload-btn,
- .qfc-admin-projects-files-actions-info-btn,
- .qfc-admin-projects-files-actions-delete-btn,
- .qfc-admin-projects-files-actions-download-btn {
- padding: 8px 15px;
- border-radius: 4px;
- cursor: pointer;
- transition: background 0.15s;
- vertical-align: middle;
- font-family: "Roboto", "Lucida Grande", Verdana, Arial, sans-serif;
- font-weight: normal;
- font-size: 13px;
- border: none;
- background: var(--button-bg);
- color: var(--button-fg);
- }
-
- .qfc-admin-projects-files-actions-delete-btn {
- background: var(--delete-button-bg);
- }
-
  .qfc-admin-project-files-modal {
  display: none; /* Hidden by default */
  position: fixed; /* Stay in place */

docker-app/qfieldcloud/core/tests/test_delta.py

@@ -791,7 +791,7 @@ def test_non_spatial_geom_empty_str_delta(self):
 
  self.assertEqual(
  self.get_file_contents(project, "nonspatial.csv"),
- b'fid,col1\n"1",foo\n"2",newfeature\n',
+ b'fid,col1\n"1",new_value\n',
  )
 
  def test_special_data_types(self):

docker-app/qfieldcloud/core/tests/testdata/delta/deltas/nonspatial_geom_empty_str.json

@@ -12,13 +12,13 @@
  "new": {
  "geometry": "",
  "attributes": {
- "col1": "foo"
+ "col1": "new_value"
  }
  },
  "old": {
  "geometry": null,
  "attributes": {
- "col1": "bar"
+ "col1": "foo"
  }
  }
  }