Merge commit 'f7ccf4d6c3f289fea5ec90e87f56731741aea6bf' into release

.env.example

@@ -52,8 +52,14 @@ GEODB_USER=postgres
 GEODB_PASSWORD=KUAa7h!G&wQEmkS3
 GEODB_DB=postgres
 
+# Sentry DSN. Missing value disables Sentry logging. Can be found on https://opengisch.sentry.io/settings/projects/qfieldcloud/keys/ .
+# DEFAULT: <NO VALUE>
 SENTRY_DSN=
 
+# Sentry sample rate between 0 and 1. Read more on https://docs.sentry.io/platforms/python/configuration/sampling/ .
+# DEFAULT: 1
+SENTRY_SAMPLE_RATE=1
+
 REDIS_PASSWORD=change_me_with_a_very_loooooooooooong_password
 REDIS_PORT=6379
 

conf/nginx/templates/default.conf.template

@@ -21,6 +21,7 @@ log_format json-logger escape=json
  '"upstream_connect_time":"$upstream_connect_time",'
  '"upstream_header_time":"$upstream_header_time",'
  '"upstream_response_time":"$upstream_response_time",'
+ '"request_id":"$request_id",'
  '"source":"nginx"'
 '}';
 

docker-app/Dockerfile

@@ -74,6 +74,7 @@ ENTRYPOINT ["/entrypoint.sh"]
 
 # a separate stage for webserver runtime environment
 FROM base as webserver_runtime
+ENV LOGGER_SOURCE=app
 COPY ./requirements_runtime.txt .
 RUN pip3 install -r requirements_runtime.txt
 EXPOSE 8000
@@ -88,6 +89,7 @@ USER app
 
 # a separate stage for webserver test environment
 FROM base as webserver_test
+ENV LOGGER_SOURCE=app
 COPY ./requirements_test.txt .
 RUN pip3 install -r requirements_test.txt
 EXPOSE 8000
@@ -102,6 +104,7 @@ USER app
 
 # a separate stage for worker wrapper runtime environment
 FROM base as worker_wrapper_runtime
+ENV LOGGER_SOURCE=worker_wrapper
 COPY ./requirements_worker_wrapper.txt .
 RUN pip3 install -r requirements_worker_wrapper.txt
 

docker-app/qfieldcloud/core/admin.py

@@ -11,7 +11,9 @@
 from django.contrib.auth.models import Group
 from django.core.exceptions import PermissionDenied
 from django.db.models.fields.json import JSONField
+from django.db.models.functions import Lower
 from django.forms import ModelForm, fields, widgets
+from django.http import HttpRequest
 from django.http.response import Http404, HttpResponseRedirect
 from django.shortcuts import resolve_url
 from django.template.response import TemplateResponse
@@ -245,7 +247,7 @@ class PersonAdmin(admin.ModelAdmin):
  "is_staff",
  )
 
- search_fields = ("username__icontains",)
+ search_fields = ("username__icontains", "email__iexact")
 
  fields = (
  "username",
@@ -330,8 +332,18 @@ def password_reset_url(self, request, user_id, form_url=""):
 
 class ProjectCollaboratorInline(admin.TabularInline):
  model = ProjectCollaborator
+
  extra = 0
 
+ readonly_fields = (
+ "created_by",
+ "updated_by",
+ "created_at",
+ "updated_at",
+ )
+
+ autocomplete_fields = ("collaborator",)
+
 
 class ProjectFilesWidget(widgets.Input):
  template_name = "admin/project_files_widget.html"
@@ -371,7 +383,7 @@ class ProjectAdmin(admin.ModelAdmin):
  "is_public",
  "owner",
  "project_filename",
- "storage_size_mb",
+ "file_storage_bytes",
  "created_at",
  "updated_at",
  "data_last_updated_at",
@@ -381,7 +393,7 @@ class ProjectAdmin(admin.ModelAdmin):
  )
  readonly_fields = (
  "id",
- "storage_size_mb",
+ "file_storage_bytes",
  "created_at",
  "updated_at",
  "data_last_updated_at",
@@ -394,6 +406,7 @@ class ProjectAdmin(admin.ModelAdmin):
  "name__icontains",
  "owner__username__iexact",
  )
+ autocomplete_fields = ("owner",)
 
  ordering = ("-updated_at",)
 
@@ -433,6 +446,17 @@ def project_details__pre(self, instance):
 
  return format_pre_json(instance.project_details)
 
+ def save_formset(self, request, form, formset, change):
+ for form_obj in formset:
+ if isinstance(form_obj.instance, ProjectCollaborator):
+ # add created_by only if it's a newly created collaborator
+ if form_obj.instance.id is None:
+ form_obj.instance.created_by = request.user
+
+ form_obj.instance.updated_by = request.user
+
+ super().save_formset(request, form, formset, change)
+
 
 class DeltaInline(admin.TabularInline):
  model = ApplyJob.deltas_to_apply.through
@@ -726,6 +750,8 @@ class OrganizationMemberInline(admin.TabularInline):
  fk_name = "organization"
  extra = 0
 
+ autocomplete_fields = ("member",)
+
 
 class TeamInline(admin.TabularInline):
  model = Team
@@ -767,12 +793,16 @@ class OrganizationAdmin(admin.ModelAdmin):
  search_fields = (
  "username__icontains",
  "organization_owner__username__icontains",
+ "email__iexact",
+ "organization_owner__email__iexact",
  )
 
  list_select_related = ("organization_owner",)
 
  list_filter = ("date_joined",)
 
+ autocomplete_fields = ("organization_owner",)
+
  def organization_owner__link(self, instance):
  return model_admin_url(
  instance.organization_owner, instance.organization_owner.username
@@ -808,6 +838,8 @@ class TeamMemberInline(admin.TabularInline):
  fk_name = "team"
  extra = 0
 
+ autocomplete_fields = ("member",)
+
 
 class TeamAdmin(admin.ModelAdmin):
  inlines = (TeamMemberInline,)
@@ -826,6 +858,8 @@ class TeamAdmin(admin.ModelAdmin):
 
  list_filter = ("date_joined",)
 
+ autocomplete_fields = ("team_organization",)
+
  def save_model(self, request, obj, form, change):
  if not obj.username.startswith("@"):
  obj.username = f"@{obj.team_organization.username}/{obj.username}"
@@ -843,6 +877,36 @@ class InvitationAdmin(InvitationAdminBase):
  search_fields = ("email__icontains", "inviter__username__iexact")
 
 
+class UserAccountAdmin(admin.ModelAdmin):
+ """The sole purpose of this admin module is only to support autocomplete fields in Django admin."""
+
+ ordering = (Lower("user__username"),)
+ search_fields = ("user__username__icontains",)
+ list_select_related = ("user",)
+
+ def has_module_permission(self, request: HttpRequest) -> bool:
+ # hide this module from Django admin, it is accessible via "Person" and "Organization" as inline edit
+ return False
+
+
+class UserAdmin(admin.ModelAdmin):
+ """The sole purpose of this admin module is only to support autocomplete fields in Django admin."""
+
+ ordering = (Lower("username"),)
+ search_fields = ("username__icontains",)
+
+ def get_queryset(self, request: HttpRequest):
+ return (
+ super()
+ .get_queryset(request)
+ .filter(type__in=(User.Type.PERSON, User.Type.ORGANIZATION))
+ )
+
+ def has_module_permission(self, request: HttpRequest) -> bool:
+ # hide this module from Django admin, it is accessible via "Person" and "Organization" as inline edit
+ return False
+
+
 admin.site.register(Invitation, InvitationAdmin)
 admin.site.register(Person, PersonAdmin)
 admin.site.register(Organization, OrganizationAdmin)
@@ -851,3 +915,7 @@ class InvitationAdmin(InvitationAdminBase):
 admin.site.register(Delta, DeltaAdmin)
 admin.site.register(Job, JobAdmin)
 admin.site.register(Geodb, GeodbAdmin)
+
+# The sole purpose of the `User` and `UserAccount` admin modules is only to support autocomplete fields in Django admin
+admin.site.register(User, UserAdmin)
+admin.site.register(UserAccount, UserAccountAdmin)

docker-app/qfieldcloud/core/logging/formatters.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 import json_log_formatter
+from django.conf import settings
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.serializers.json import DjangoJSONEncoder
 
@@ -23,7 +24,7 @@ def json_record(self, message, extra, record):
  :return: Dictionary which will be passed to JSON lib.
 
  """
- if "ts" in extra:
+ if "ts" not in extra:
  extra["ts"] = datetime.utcnow()
 
  # Include builtins
@@ -34,6 +35,7 @@ def json_record(self, message, extra, record):
  extra["filename"] = record.filename
  extra["lineno"] = record.lineno
  extra["thread"] = record.thread
+ extra["source"] = settings.LOGGER_SOURCE
 
  if record.exc_info:
  extra["exc_info"] = self.formatException(record.exc_info)
@@ -49,11 +51,11 @@ def to_json(self, record):
  """
  try:
  return self.json_lib.dumps(
- record, default=json_default, sort_keys=True, cls=JsonEncoder
+ record, default=json_default, cls=JsonEncoder, separators=(",", ":")
  )
  # ujson doesn't support default argument and raises TypeError.
  except TypeError:
- return self.json_lib.dumps(record, cls=JsonEncoder)
+ return self.json_lib.dumps(record, cls=JsonEncoder, separators=(",", ":"))
 
 
 def json_default(obj):

docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py

@@ -22,7 +22,7 @@ def handle(self, *args, **options):
  extra_filters["id"] = project_id
 
  if not project_id and not force_recalculate:
- extra_filters["storage_size_mb"] = 0
+ extra_filters["file_storage_bytes"] = 0
 
  projects_qs = Project.objects.filter(
  project_filename__isnull=False,
@@ -36,5 +36,5 @@ def handle(self, *args, **options):
  )
  project.save(recompute_storage=True)
  print(
- f'Project files storage size for "{project.id}" is {project.storage_size_mb}MB'
+ f'Project files storage size for "{project.id}" is {project.file_storage_bytes} bytes.'
  )

docker-app/qfieldcloud/core/migrations/0060_alter_project_storage_size_mb.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.17 on 2023-02-10 00:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("core", "0059_auto_20221028_1806"),
+ ]
+
+ operations = [
+ migrations.RenameField(
+ model_name="project",
+ old_name="storage_size_mb",
+ new_name="file_storage_bytes",
+ ),
+ migrations.RunSQL(
+ """
+ UPDATE core_project
+ SET
+ file_storage_bytes = file_storage_bytes * 1000 * 1000
+ """,
+ """
+ UPDATE core_project
+ SET
+ file_storage_bytes = file_storage_bytes / 1000 / 1000
+ """,
+ ),
+ migrations.AlterField(
+ model_name="project",
+ name="file_storage_bytes",
+ field=models.PositiveBigIntegerField(default=0),
+ ),
+ ]