Merge remote-tracking branch 'origin/master' into release

.github/workflows/release_drafter.yml

@@ -4,7 +4,7 @@ on:
  push:
  # branches to consider in the event; optional, defaults to all
  branches:
- - master
+ - release
 
 jobs:
  update_release_draft:

.github/workflows/test.yml

@@ -1,8 +1,8 @@
 name: Test
 
 on:
- - push
- - pull_request
+ push:
+ pull_request:
 
 jobs:
  check_format:

.gitignore

@@ -1,5 +1,6 @@
 __pycache__/
 *.log
+*.orig
 .htmlcov/
 .coverage
 /conf/supervisord.conf
@@ -8,4 +9,5 @@ __pycache__/
 docker-compose.override.yml
 client/projects
 conf/nginx/certs/*
+conf/certbot/*
 Pipfile*

README.md

@@ -209,12 +209,14 @@ Run the django database migrations
  docker compose exec app python manage.py migrate
 
 
-## Create a certificate using Let's Encrypt
+## Create or renew a certificate using Let's Encrypt
 
 If you are running the server on a server with a public domain, you can install Let's Encrypt certificate by running the following command:
 
  ./scripts/init_letsencrypt.sh
 
+The same command can also be used to update an expired certificate.
+
 Note you may want to change the `LETSENCRYPT_EMAIL`, `LETSENCRYPT_RSA_KEY_SIZE` and `LETSENCRYPT_STAGING` variables.
 
 ### Infrastructure

docker-app/qfieldcloud/core/admin.py

@@ -16,6 +16,7 @@
 from django.contrib.admin.templatetags.admin_urls import admin_urlname
 from django.contrib.auth.models import Group
 from django.core.exceptions import PermissionDenied
+from django.db.models import Q
 from django.db.models.fields.json import JSONField
 from django.db.models.functions import Lower
 from django.forms import ModelForm, fields, widgets
@@ -48,6 +49,7 @@
  User,
  UserAccount,
 )
+from qfieldcloud.core.templatetags.filters import filesizeformat10
 from qfieldcloud.core.utils2 import jobs
 from rest_framework.authtoken.models import TokenProxy
 
@@ -368,6 +370,7 @@ class PersonAdmin(admin.ModelAdmin):
  "is_active",
  "date_joined",
  "last_login",
+ "storage_usage__field",
  )
  list_filter = (
  "type",
@@ -397,6 +400,7 @@ class PersonAdmin(admin.ModelAdmin):
  readonly_fields = (
  "date_joined",
  "last_login",
+ "storage_usage__field",
  )
 
  inlines = (
@@ -407,6 +411,13 @@ class PersonAdmin(admin.ModelAdmin):
  add_form_template = "admin/change_form.html"
  change_form_template = "admin/person_change_form.html"
 
+ @admin.display(description=_("Storage"))
+ def storage_usage__field(self, instance) -> str:
+ used_storage = filesizeformat10(instance.useraccount.storage_used_bytes)
+ free_storage = filesizeformat10(instance.useraccount.storage_free_bytes)
+ used_storage_perc = instance.useraccount.storage_used_ratio * 100
+ return f"{used_storage} {free_storage} ({used_storage_perc:.2f}%)"
+
  def save_model(self, request, obj, form, change):
  # Set the password to the value in the field if it's changed.
  if obj.pk:
@@ -612,6 +623,24 @@ def has_delete_permission(self, request, obj):
  # return format_pre_json(instance.feedback)
 
 
+class IsFinalizedJobFilter(admin.SimpleListFilter):
+ title = _("finalized job")
+ parameter_name = "finalized"
+
+ def lookups(self, request, model_admin):
+ return (
+ ("finalized", _("finalized")),
+ ("not finalized", _("not finalized")),
+ )
+
+ def queryset(self, request, queryset):
+ q = Q(status="pending") | Q(status="started") | Q(status="queued")
+ if self.value() == "not finalized":
+ return queryset.filter(q)
+ else:
+ return queryset.filter(~q)
+
+
 class JobAdmin(admin.ModelAdmin):
  list_display = (
  "id",
@@ -623,7 +652,7 @@ class JobAdmin(admin.ModelAdmin):
  "created_at",
  "updated_at",
  )
- list_filter = ("type", "status", "updated_at")
+ list_filter = ("type", "status", "updated_at", IsFinalizedJobFilter)
  list_select_related = ("project", "project__owner", "created_by")
  exclude = ("feedback", "output")
  ordering = ("-updated_at",)
@@ -727,6 +756,24 @@ def has_delete_permission(self, request, obj):
  return False
 
 
+class IsFinalizedDeltaJobFilter(admin.SimpleListFilter):
+ title = _("finalized delta job")
+ parameter_name = "finalized"
+
+ def lookups(self, request, model_admin):
+ return (
+ ("finalized", _("finalized")),
+ ("not finalized", _("not finalized")),
+ )
+
+ def queryset(self, request, queryset):
+ q = Q(last_status="pending") | Q(last_status="started")
+ if self.value() == "not finalized":
+ return queryset.filter(q)
+ else:
+ return queryset.filter(~q)
+
+
 class DeltaAdmin(admin.ModelAdmin):
  list_display = (
  "id",
@@ -738,7 +785,7 @@ class DeltaAdmin(admin.ModelAdmin):
  "created_at",
  "updated_at",
  )
- list_filter = ("last_status", "updated_at")
+ list_filter = ("last_status", "updated_at", IsFinalizedDeltaJobFilter)
 
  actions = (
  "set_status_pending",
@@ -923,6 +970,7 @@ class OrganizationAdmin(admin.ModelAdmin):
  "email",
  "organization_owner__link",
  "date_joined",
+ "storage_usage__field",
  )
 
  search_fields = (
@@ -932,7 +980,10 @@ class OrganizationAdmin(admin.ModelAdmin):
  "organization_owner__email__iexact",
  )
 
- readonly_fields = ("date_joined",)
+ readonly_fields = (
+ "date_joined",
+ "storage_usage__field",
+ )
 
  list_select_related = ("organization_owner",)
 
@@ -946,6 +997,13 @@ def organization_owner__link(self, instance):
  instance.organization_owner, instance.organization_owner.username
  )
 
+ @admin.display(description=_("Storage"))
+ def storage_usage__field(self, instance) -> str:
+ used_storage = filesizeformat10(instance.useraccount.storage_used_bytes)
+ free_storage = filesizeformat10(instance.useraccount.storage_free_bytes)
+ used_storage_perc = instance.useraccount.storage_used_ratio * 100
+ return f"{used_storage} {free_storage} ({used_storage_perc:.2f}%)"
+
  def get_search_results(self, request, queryset, search_term):
  filters = search_parser(
  request,

docker-app/qfieldcloud/core/templates/admin/project_files_widget.html

@@ -169,6 +169,16 @@
  .then((files) => refreshTableContents({ files }))
  .catch((error) => refreshTableContents({ error }));
  };
+ const filesize10 = n => {
+ switch(true) {
+ case n < 10 ** 6:
+ return (n / 10 ** 3).toFixed(3) + " KB";
+ case n < 10 ** 9:
+ return (n / 10 ** 6).toFixed(3) + " MB";
+ default:
+ return (n / 10 ** 9).toFixed(3) + " GB";
+ }
+ };
  const refreshTableContents = ({ error, isLoading = false, files = [] }) => {
  $tbody.innerHTML = '';
  $count.innerHTML = '';
@@ -196,7 +206,7 @@
 
  $trow.querySelector('td:nth-child(1)').innerHTML = file.name;
  $trow.querySelector('td:nth-child(2)').innerHTML = file.last_modified;
- $trow.querySelector('td:nth-child(3)').innerHTML = file.size;
+ $trow.querySelector('td:nth-child(3)').innerHTML = `<span title="${file.size} bytes">${filesize10(file.size)} KB</span>`;
 
  for (const version of file.versions) {
  const $option = document.createElement('option');
@@ -220,9 +230,11 @@
  });
 
  $downloadBtn.addEventListener('click', () => {
- window.open(buildApiUrl(`files/${projectId}/${file.name}/`, {
+ const pathToFile = `files/${projectId}/${file.name}/`
+ const buildApiUrlWithPath = () => buildApiUrl(pathToFile, {
  version: $versionsSelect.value,
- }));
+ });
+ window.open($versionsSelect.value ? buildApiUrlWithPath() : pathToFile);
  });
 
  $deleteBtn.addEventListener('click', () => {

docker-app/qfieldcloud/core/templatetags/filters.py

@@ -0,0 +1,52 @@
+from django import template
+from django.utils import formats
+from django.utils.html import avoid_wrapping
+from django.utils.translation import gettext as _
+from django.utils.translation import ngettext
+
+register = template.Library()
+
+
+@register.filter(is_safe=True)
+def filesizeformat10(bytes_) -> str:
+ """
+ Format the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
+ 102 bytes, etc.).
+
+ Unlike Django's `filesizeformat` which uses powers of 2 (e.g. 1024KB==1MB), `filesizeformat10` uses powers of 10 (e.g. 1000KB==1MB)
+ """
+ try:
+ bytes_ = int(bytes_)
+ except (TypeError, ValueError, UnicodeDecodeError):
+ value = ngettext("%(size)d byte", "%(size)d bytes", 0) % {"size": 0}
+ return avoid_wrapping(value)
+
+ def filesize_number_format(value):
+ return formats.number_format(round(value, 1), 1)
+
+ KB = 10**3
+ MB = 10**6
+ GB = 10**9
+ TB = 10**12
+ PB = 10**15
+
+ negative = bytes_ < 0
+ if negative:
+ bytes_ = -bytes_ # Allow formatting of negative numbers.
+
+ if bytes_ < KB:
+ value = ngettext("%(size)d byte", "%(size)d bytes", bytes_) % {"size": bytes_}
+ elif bytes_ < MB:
+ value = _("%s KB") % filesize_number_format(bytes_ / KB)
+ elif bytes_ < GB:
+ value = _("%s MB") % filesize_number_format(bytes_ / MB)
+ elif bytes_ < TB:
+ value = _("%s GB") % filesize_number_format(bytes_ / GB)
+ elif bytes_ < PB:
+ value = _("%s TB") % filesize_number_format(bytes_ / TB)
+ else:
+ value = _("%s PB") % filesize_number_format(bytes_ / PB)
+
+ if negative:
+ value = "-%s" % value
+ return avoid_wrapping(value)

docker-app/qfieldcloud/core/utils.py

@@ -121,11 +121,9 @@ def get_s3_bucket() -> mypy_boto3_s3.service_resource.Bucket:
 def get_s3_client() -> mypy_boto3_s3.Client:
  """Get a new S3 client instance using Django settings"""
 
- s3_client = boto3.client(
+ s3_session = get_s3_session()
+ s3_client = s3_session.client(
  "s3",
- region_name=settings.STORAGE_REGION_NAME,
- aws_access_key_id=settings.STORAGE_ACCESS_KEY_ID,
- aws_secret_access_key=settings.STORAGE_SECRET_ACCESS_KEY,
  endpoint_url=settings.STORAGE_ENDPOINT_URL,
  )
  return s3_client

docker-app/qfieldcloud/core/views/files_views.py

@@ -141,6 +141,12 @@ def post(self, request, projectid, filename, format=None):
  logger.info(
  'The key "file" was not found in `request.data`.',
  extra={
+ "data_for_key_text_content": str(request.data.get("text", ""))[
+ :1000
+ ],
+ "data_for_key_text_len": len(request.data.get("text", "")),
+ "request_content_length": request.META.get("CONTENT_LENGTH"),
+ "request_content_type": request.META.get("CONTENT_TYPE"),
  "request_data": list(request.data.keys()),
  "request_files": list(request.FILES.keys()),
  },

docker-app/qfieldcloud/settings.py

@@ -144,7 +144,9 @@
  ],
  "APP_DIRS": True,
  "OPTIONS": {
- "builtins": [],
+ "builtins": [
+ "qfieldcloud.core.templatetags.filters",
+ ],
  "context_processors": [
  "django.template.context_processors.debug",
  "django.template.context_processors.request",
@@ -263,12 +265,13 @@
  SENTRY_SAMPLE_RATE = float(os.environ.get("SENTRY_SAMPLE_RATE", 1))
 
  def before_send(event, hint):
- from qfieldcloud.core.exceptions import ProjectAlreadyExistsError
+ from qfieldcloud.core.exceptions import ProjectAlreadyExistsError, QuotaError
  from rest_framework.exceptions import ValidationError
 
  ignored_exceptions = (
  ValidationError,
  ProjectAlreadyExistsError,
+ QuotaError,
  )
 
  if "exc_info" in hint:

docker-app/qfieldcloud/subscription/models.py

@@ -19,6 +19,8 @@
 
 from .exceptions import NotPremiumPlanException
 
+logger = logging.getLogger(__name__)
+
 
 def get_subscription_model() -> "Subscription":
  return apps.get_model(settings.QFIELDCLOUD_SUBSCRIPTION_MODEL)
@@ -655,7 +657,7 @@ def update_subscription(
  update_fields.append(attr_name)
  setattr(subscription, attr_name, attr_value)
 
- logging.info(f"Updated subscription's fields: {', '.join(update_fields)}")
+ logger.info(f"Updated subscription's fields: {', '.join(update_fields)}")
 
  subscription.save(update_fields=update_fields)
 
@@ -726,6 +728,9 @@ def create_subscription(
  ), "Creating a trial plan requires `active_since` to be a valid datetime object"
 
  active_until = active_since + timedelta(days=config.TRIAL_PERIOD_DAYS)
+ logger.info(
+ f"Creating trial subscription from {active_since=} to {active_until=}"
+ )
  trial_subscription = cls.objects.create(
  plan=plan,
  account=account,
@@ -748,6 +753,7 @@ def create_subscription(
  regular_plan = plan
  regular_active_since = active_since
 
+ logger.info(f"Creating regular subscription from {regular_active_since}")
  regular_subscription = cls.objects.create(
  plan=regular_plan,
  account=account,

docker-app/requirements.txt

@@ -13,7 +13,7 @@ coreschema==0.0.4
 cryptography==36.0.1
 defusedxml==0.7.1
 Deprecated==1.2.13
-Django==3.2.17
+Django==3.2.18
 django-allauth==0.44.0
 django-appconf==1.0.5
 django-auditlog==2.2.2

docker-compose.override.dev.yml

@@ -3,8 +3,6 @@ version: '3.9'
 services:
 
  app:
- depends_on:
- - geodb
  build:
  args:
  DEBUG_BUILD: ${DEBUG}