Merge branch 'master' into release

.env.example

@@ -57,6 +57,10 @@ SENTRY_DSN=
 REDIS_PASSWORD=change_me_with_a_very_loooooooooooong_password
 REDIS_PORT=6379
 
+# Memcached port. Exposed only in docker-compose.local.yml
+# DEFAULT: 11211
+MEMCACHED_PORT=11211
+
 LOG_DIRECTORY=/tmp
 TMP_DIRECTORY=/tmp
 

README.md

@@ -224,19 +224,20 @@ Based on this example
 
 ### Ports
 
-| service | port | configuration | local | development | production |
-|---------------|------|----------------------|--------------------|--------------------|--------------------|
-| nginx http | 80 | WEB_HTTP_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| nginx https | 443 | WEB_HTTPS_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| django http | 8011 | DJANGO_DEV_PORT | :white_check_mark: | :x: | :x: |
-| postgres | 5433 | HOST_POSTGRES_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| redis | 6379 | REDIS_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| geodb | 5432 | HOST_POSTGRES_PORT | :white_check_mark: | :white_check_mark: | :x: |
-| minio API | 8009 | MINIO_API_PORT | :white_check_mark: | :x: | :x: |
-| minio browser | 8010 | MINIO_BROWSER_PORT | :white_check_mark: | :x: | :x: |
-| smtp web | 8012 | SMTP4DEV_WEB_PORT | :white_check_mark: | :x: | :x: |
-| smtp | 25 | SMTP4DEV_SMTP_PORT | :white_check_mark: | :x: | :x: |
-| imap | 143 | SMTP4DEV_IMAP_PORT | :white_check_mark: | :x: | :x: |
+| service | port | configuration | local | development | production |
+|---------------|-------|----------------------|--------------------|--------------------|--------------------|
+| nginx http | 80 | WEB_HTTP_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| nginx https | 443 | WEB_HTTPS_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| django http | 8011 | DJANGO_DEV_PORT | :white_check_mark: | :x: | :x: |
+| postgres | 5433 | HOST_POSTGRES_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| redis | 6379 | REDIS_PORT | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| memcached | 11211 | MEMCACHED_PORT | :white_check_mark: | :x: | :x: |
+| geodb | 5432 | HOST_POSTGRES_PORT | :white_check_mark: | :white_check_mark: | :x: |
+| minio API | 8009 | MINIO_API_PORT | :white_check_mark: | :x: | :x: |
+| minio browser | 8010 | MINIO_BROWSER_PORT | :white_check_mark: | :x: | :x: |
+| smtp web | 8012 | SMTP4DEV_WEB_PORT | :white_check_mark: | :x: | :x: |
+| smtp | 25 | SMTP4DEV_SMTP_PORT | :white_check_mark: | :x: | :x: |
+| imap | 143 | SMTP4DEV_IMAP_PORT | :white_check_mark: | :x: | :x: |
 
 ### Logs
 

conf/nginx/templates/default.conf.template

@@ -97,6 +97,7 @@ server {
  location @proxy_to_app {
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header X-Request-Id $request_id;
  proxy_set_header Host $http_host;
 
  proxy_read_timeout 300;

docker-app/qfieldcloud/core/logging/formatters.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 import json_log_formatter
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.serializers.json import DjangoJSONEncoder
@@ -9,6 +11,35 @@ def default(self, obj):
 
 
 class CustomisedJSONFormatter(json_log_formatter.JSONFormatter):
+ def json_record(self, message, extra, record):
+ """Prepares a JSON payload which will be logged.
+
+ Override this method to change JSON log format.
+
+ :param message: Log message, e.g., `logger.info(msg='Sign up')`.
+ :param extra: Dictionary that was passed as `extra` param
+ `logger.info('Sign up', extra={'referral_code': '52d6ce'})`.
+ :param record: `LogRecord` we got from `JSONFormatter.format()`.
+ :return: Dictionary which will be passed to JSON lib.
+
+ """
+ if "ts" in extra:
+ extra["ts"] = datetime.utcnow()
+
+ # Include builtins
+ extra["level"] = record.levelname
+ extra["name"] = record.name
+ extra["message"] = message
+ extra["request_id"] = getattr(record, "request_id", None)
+ extra["filename"] = record.filename
+ extra["lineno"] = record.lineno
+ extra["thread"] = record.thread
+
+ if record.exc_info:
+ extra["exc_info"] = self.formatException(record.exc_info)
+
+ return extra
+
  def to_json(self, record):
  """Converts record dict to a JSON string.
  It makes best effort to serialize a record (represents an object as a string)

docker-app/qfieldcloud/core/models.py

@@ -311,7 +311,7 @@ def save(self, *args, **kwargs):
 
  def delete(self, *args, **kwargs):
  if self.type != User.Type.TEAM:
- qfieldcloud.core.utils2.storage.remove_user_avatar(self)
+ qfieldcloud.core.utils2.storage.delete_user_avatar(self)
 
  with no_audits([User, UserAccount, Project]):
  super().delete(*args, **kwargs)
@@ -1195,7 +1195,7 @@ def direct_collaborators(self):
 
  def delete(self, *args, **kwargs):
  if self.thumbnail_uri:
- qfieldcloud.core.utils2.storage.remove_project_thumbail(self)
+ qfieldcloud.core.utils2.storage.delete_project_thumbnail(self)
  super().delete(*args, **kwargs)
 
  def save(self, recompute_storage=False, *args, **kwargs):

docker-app/qfieldcloud/core/tests/test_packages.py

@@ -15,13 +15,15 @@
  Job,
  Organization,
  OrganizationMember,
+ PackageJob,
  Person,
  Project,
  ProjectCollaborator,
  Secret,
  Team,
  TeamMember,
 )
+from qfieldcloud.core.utils2.storage import get_stored_package_ids
 from rest_framework import status
 from rest_framework.test import APITransactionTestCase
 
@@ -655,3 +657,54 @@ def test_collaborator_via_team_can_package(self):
  "project_qfield_attachments.zip",
  ],
  )
+
+ def test_outdated_packaged_files_are_deleted(self):
+ cur = self.conn.cursor()
+ cur.execute("CREATE TABLE point (id integer, geometry geometry(point, 2056))")
+ self.conn.commit()
+ cur.execute(
+ "INSERT INTO point(id, geometry) VALUES(1, ST_GeomFromText('POINT(2725505 1121435)', 2056))"
+ )
+ self.conn.commit()
+
+ self.upload_files_and_check_package(
+ token=self.token1.key,
+ project=self.project1,
+ files=[
+ ("delta/project2.qgs", "project.qgs"),
+ ("delta/points.geojson", "points.geojson"),
+ ],
+ expected_files=[
+ "data.gpkg",
+ "project_qfield.qgs",
+ "project_qfield_attachments.zip",
+ ],
+ )
+
+ old_package = PackageJob.objects.filter(project=self.project1).latest(
+ "created_at"
+ )
+ stored_package_ids = get_stored_package_ids(self.project1.id)
+ self.assertIn(str(old_package.id), stored_package_ids)
+ self.assertEqual(len(stored_package_ids), 1)
+
+ self.check_package(
+ self.token1.key,
+ self.project1,
+ [
+ "data.gpkg",
+ "project_qfield.qgs",
+ "project_qfield_attachments.zip",
+ ],
+ )
+
+ new_package = PackageJob.objects.filter(project=self.project1).latest(
+ "created_at"
+ )
+
+ stored_package_ids = get_stored_package_ids(self.project1.id)
+
+ self.assertNotEqual(old_package.id, new_package.id)
+ self.assertNotIn(str(old_package.id), stored_package_ids)
+ self.assertIn(str(new_package.id), stored_package_ids)
+ self.assertEqual(len(stored_package_ids), 1)

docker-app/qfieldcloud/core/utils.py

@@ -82,7 +82,7 @@ class S3ObjectWithVersions(NamedTuple):
  def total_size(self) -> int:
  """Total size of all versions"""
  # latest is also in versions
- return sum(v.size for v in self.versions)
+ return sum(v.size for v in self.versions if v.size is not None)
 
 
 def redis_is_running() -> bool:

docker-app/qfieldcloud/core/utils2/storage.py

@@ -84,11 +84,13 @@ def _delete_by_key_versioned(key: str):
  Raises:
  RuntimeError: When the given key is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling!
  """
- logging.info(f"S3 object deletion (versioned) with {key=}")
+ logging.info(f"Delete (versioned) S3 object with {key=}")
 
  # prevent disastrous results when prefix is either empty string ("") or slash ("/").
  if not isinstance(key, str) or key == "" or key == "/":
- raise RuntimeError(f"Attempt to delete S3 object with illegal {key=}")
+ raise RuntimeError(
+ f"Attempt to delete (versioned) S3 object with illegal {key=}"
+ )
 
  bucket = qfieldcloud.core.utils.get_s3_bucket()
 
@@ -116,11 +118,13 @@ def _delete_by_key_permanently(key: str):
  Raises:
  RuntimeError: When the given key is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling!
  """
- logging.info(f"S3 object deletion (versioned) with {key=}")
+ logging.info(f"Delete (permanently) S3 object with {key=}")
 
  # prevent disastrous results when prefix is either empty string ("") or slash ("/").
  if not isinstance(key, str) or key == "" or key == "/":
- raise RuntimeError(f"Attempt to delete S3 object with illegal {key=}")
+ raise RuntimeError(
+ f"Attempt to delete (permanently) S3 object with illegal {key=}"
+ )
 
  bucket = qfieldcloud.core.utils.get_s3_bucket()
 
@@ -141,7 +145,21 @@ def _delete_by_key_permanently(key: str):
  }
  )
 
- assert len(object_to_delete) > 0
+ if len(object_to_delete) == 0:
+ logging.warning(
+ f"Attempt to delete (permanently) S3 objects did not match any existing objects for {key=}",
+ extra={
+ "all_objects": [
+ (o.key, o.version_id, o.e_tag, o.last_modified, o.is_latest)
+ for o in temp_objects
+ ]
+ },
+ )
+ return None
+
+ logging.info(
+ f"Delete (permanently) S3 object with {key=} will delete delete {len(object_to_delete)} version(s)"
+ )
 
  return bucket.delete_objects(
  Delete={
@@ -277,8 +295,8 @@ def upload_user_avatar(user: "User", file: IO, mimetype: str) -> str: # noqa: F
  return key
 
 
-def remove_user_avatar(user: "User") -> None: # noqa: F821
- """Removes the user's avatar file.
+def delete_user_avatar(user: "User") -> None: # noqa: F821
+ """Deletes the user's avatar file.
 
  NOTE this function does NOT modify the `UserAccount.avatar_uri` field
 
@@ -291,7 +309,8 @@ def remove_user_avatar(user: "User") -> None: # noqa: F821
  if not key:
  return
 
- if not key or not re.match(r"^users/\w+/avatar.(png|jpg|svg)$", key):
+ # e.g. "users/suricactus/avatar.svg"
+ if not key or not re.match(r"^users/\w+/avatar\.(png|jpg|svg)$", key):
  raise RuntimeError(f"Suspicious S3 deletion of user avatar {key=}")
 
  _delete_by_key_permanently(key)
@@ -338,8 +357,8 @@ def upload_project_thumbail(
  return key
 
 
-def remove_project_thumbail(project: "Project") -> None: # noqa: F821
- """Uploads a picture as a project thumbnail.
+def delete_project_thumbnail(project: "Project") -> None: # noqa: F821
+ """Delete a picture as a project thumbnail.
 
  NOTE this function does NOT modify the `Project.thumbnail_uri` field
 
@@ -351,7 +370,9 @@ def remove_project_thumbail(project: "Project") -> None: # noqa: F821
  return
 
  if not key or not re.match(
- r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/meta/\w+.(png|jpg|svg)$", key
+ # e.g. "projects/9bf34e75-0a5d-47c3-a2f0-ebb7126eeccc/meta/thumbnail.png"
+ r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/meta/thumbnail\.(png|jpg|svg)$",
+ key,
  ):
  raise RuntimeError(f"Suspicious S3 deletion of project thumbnail image {key=}")
 
@@ -568,7 +589,11 @@ def get_stored_package_ids(project_id: str) -> Set[str]:
 def delete_stored_package(project_id: str, package_id: str) -> None:
  prefix = f"projects/{project_id}/packages/{package_id}/"
 
- if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/packages/\w+/$", prefix):
+ if not re.match(
+ # e.g. "projects/878039c4-b945-4356-a44e-a908fd3f2263/packages/633cd4f7-db14-4e6e-9b2b-c0ce98f9d338/"
+ r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/packages/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$",
+ prefix,
+ ):
  raise RuntimeError(
  f"Suspicious S3 deletion on stored project package {project_id=} {package_id=}"
  )

docker-app/qfieldcloud/settings.py

@@ -54,6 +54,13 @@
 ]
 
 
+CACHES = {
+ "default": {
+ "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
+ "LOCATION": "memcached:11211",
+ }
+}
+
 # Application definition
 INSTALLED_APPS = [
  # django contrib
@@ -98,6 +105,7 @@
 ]
 
 MIDDLEWARE = [
+ "log_request_id.middleware.RequestIDMiddleware",
  "debug_toolbar.middleware.DebugToolbarMiddleware",
  "django.middleware.security.SecurityMiddleware",
  "django.contrib.sessions.middleware.SessionMiddleware",
@@ -306,9 +314,12 @@
 TEST_RUNNER = "qfieldcloud.testing.QfcTestSuiteRunner"
 
 LOGLEVEL = os.environ.get("LOGLEVEL", "DEBUG").upper()
+LOG_REQUEST_ID_HEADER = "HTTP_X_REQUEST_ID"
+GENERATE_REQUEST_ID_IF_NOT_IN_HEADER = False
 LOGGING = {
  "version": 1,
  "disable_existing_loggers": False,
+ "filters": {"request_id": {"()": "log_request_id.filters.RequestIDFilter"}},
  "formatters": {
  "json": {
  "()": "qfieldcloud.core.logging.formatters.CustomisedJSONFormatter",
@@ -317,6 +328,7 @@
  "handlers": {
  "console.json": {
  "class": "logging.StreamHandler",
+ "filters": ["request_id"],
  "formatter": "json",
  },
  },
@@ -348,6 +360,8 @@
 QFIELDCLOUD_ADMIN_URI = os.environ.get("QFIELDCLOUD_ADMIN_URI", "admin/")
 
 CONSTANCE_BACKEND = "qfieldcloud.core.constance_backends.DatabaseBackend"
+CONSTANCE_DATABASE_CACHE_BACKEND = "default"
+CONSTANCE_DATABASE_CACHE_AUTOFILL_TIMEOUT = 60 * 60 * 24
 CONSTANCE_CONFIG = {
  "WORKER_TIMEOUT_S": (
  600,

docker-app/requirements.txt

@@ -37,6 +37,7 @@ django-notifications-hq==1.6.0
 django-phonenumber-field==7.0.0
 django-picklefield==3.1
 django-storages==1.11.1
+django-log-request-id==2.1.0
 django-tables2==2.4.1
 django-timezone-field==4.2.1
 djangorestframework==3.12.4
@@ -50,6 +51,7 @@ JSON-log-formatter==0.5.0
 jsonfield==3.1.0
 jsonschema==3.2.0
 MarkupSafe==2.0.1
+python-memcached==1.59
 mypy-boto3-s3==1.20.17
 oauthlib==3.2.1
 packaging==21.3

docker-compose.override.local.yml

@@ -59,6 +59,14 @@ services:
  - ${HOST_POSTGRES_PORT}:5432
  command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr"]
 
+ redis:
+ ports:
+ - "${REDIS_PORT}:6379"
+
+ memcached:
+ ports:
+ - "${MEMCACHED_PORT}:11211"
+
  geodb:
  image: postgis/postgis:12-3.0
  restart: unless-stopped