diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml index fccb2bc28..e03fabb1a 100644 --- a/.github/workflows/release_drafter.yml +++ b/.github/workflows/release_drafter.yml @@ -4,7 +4,7 @@ on: push: # branches to consider in the event; optional, defaults to all branches: - - master + - release jobs: update_release_draft: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bfa972ecb..eb2e7db3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,8 @@ name: Test on: - - push - - pull_request + push: + pull_request: jobs: check_format: diff --git a/.gitignore b/.gitignore index 6d39e10fe..7a873f72b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ *.log +*.orig .htmlcov/ .coverage /conf/supervisord.conf @@ -8,4 +9,5 @@ __pycache__/ docker-compose.override.yml client/projects conf/nginx/certs/* +conf/certbot/* Pipfile* diff --git a/README.md b/README.md index a987a03a6..4aa4bb000 100644 --- a/README.md +++ b/README.md @@ -209,12 +209,14 @@ Run the django database migrations docker compose exec app python manage.py migrate -## Create a certificate using Let's Encrypt +## Create or renew a certificate using Let's Encrypt If you are running the server on a server with a public domain, you can install Let's Encrypt certificate by running the following command: ./scripts/init_letsencrypt.sh +The same command can also be used to update an expired certificate. + Note you may want to change the `LETSENCRYPT_EMAIL`, `LETSENCRYPT_RSA_KEY_SIZE` and `LETSENCRYPT_STAGING` variables. ### Infrastructure diff --git a/docker-app/qfieldcloud/core/admin.py b/docker-app/qfieldcloud/core/admin.py index b18d0cc58..840aa5630 100644 --- a/docker-app/qfieldcloud/core/admin.py +++ b/docker-app/qfieldcloud/core/admin.py @@ -16,6 +16,7 @@ from django.contrib.admin.templatetags.admin_urls import admin_urlname from django.contrib.auth.models import Group from django.core.exceptions import PermissionDenied +from django.db.models import Q from django.db.models.fields.json import JSONField from django.db.models.functions import Lower from django.forms import ModelForm, fields, widgets @@ -48,6 +49,7 @@ User, UserAccount, ) +from qfieldcloud.core.templatetags.filters import filesizeformat10 from qfieldcloud.core.utils2 import jobs from rest_framework.authtoken.models import TokenProxy @@ -368,6 +370,7 @@ class PersonAdmin(admin.ModelAdmin): "is_active", "date_joined", "last_login", + "storage_usage__field", ) list_filter = ( "type", @@ -397,6 +400,7 @@ class PersonAdmin(admin.ModelAdmin): readonly_fields = ( "date_joined", "last_login", + "storage_usage__field", ) inlines = ( @@ -407,6 +411,13 @@ class PersonAdmin(admin.ModelAdmin): add_form_template = "admin/change_form.html" change_form_template = "admin/person_change_form.html" + @admin.display(description=_("Storage")) + def storage_usage__field(self, instance) -> str: + used_storage = filesizeformat10(instance.useraccount.storage_used_bytes) + free_storage = filesizeformat10(instance.useraccount.storage_free_bytes) + used_storage_perc = instance.useraccount.storage_used_ratio * 100 + return f"{used_storage} {free_storage} ({used_storage_perc:.2f}%)" + def save_model(self, request, obj, form, change): # Set the password to the value in the field if it's changed. if obj.pk: @@ -612,6 +623,24 @@ def has_delete_permission(self, request, obj): # return format_pre_json(instance.feedback) +class IsFinalizedJobFilter(admin.SimpleListFilter): + title = _("finalized job") + parameter_name = "finalized" + + def lookups(self, request, model_admin): + return ( + ("finalized", _("finalized")), + ("not finalized", _("not finalized")), + ) + + def queryset(self, request, queryset): + q = Q(status="pending") | Q(status="started") | Q(status="queued") + if self.value() == "not finalized": + return queryset.filter(q) + else: + return queryset.filter(~q) + + class JobAdmin(admin.ModelAdmin): list_display = ( "id", @@ -623,7 +652,7 @@ class JobAdmin(admin.ModelAdmin): "created_at", "updated_at", ) - list_filter = ("type", "status", "updated_at") + list_filter = ("type", "status", "updated_at", IsFinalizedJobFilter) list_select_related = ("project", "project__owner", "created_by") exclude = ("feedback", "output") ordering = ("-updated_at",) @@ -727,6 +756,24 @@ def has_delete_permission(self, request, obj): return False +class IsFinalizedDeltaJobFilter(admin.SimpleListFilter): + title = _("finalized delta job") + parameter_name = "finalized" + + def lookups(self, request, model_admin): + return ( + ("finalized", _("finalized")), + ("not finalized", _("not finalized")), + ) + + def queryset(self, request, queryset): + q = Q(last_status="pending") | Q(last_status="started") + if self.value() == "not finalized": + return queryset.filter(q) + else: + return queryset.filter(~q) + + class DeltaAdmin(admin.ModelAdmin): list_display = ( "id", @@ -738,7 +785,7 @@ class DeltaAdmin(admin.ModelAdmin): "created_at", "updated_at", ) - list_filter = ("last_status", "updated_at") + list_filter = ("last_status", "updated_at", IsFinalizedDeltaJobFilter) actions = ( "set_status_pending", @@ -923,6 +970,7 @@ class OrganizationAdmin(admin.ModelAdmin): "email", "organization_owner__link", "date_joined", + "storage_usage__field", ) search_fields = ( @@ -932,7 +980,10 @@ class OrganizationAdmin(admin.ModelAdmin): "organization_owner__email__iexact", ) - readonly_fields = ("date_joined",) + readonly_fields = ( + "date_joined", + "storage_usage__field", + ) list_select_related = ("organization_owner",) @@ -946,6 +997,13 @@ def organization_owner__link(self, instance): instance.organization_owner, instance.organization_owner.username ) + @admin.display(description=_("Storage")) + def storage_usage__field(self, instance) -> str: + used_storage = filesizeformat10(instance.useraccount.storage_used_bytes) + free_storage = filesizeformat10(instance.useraccount.storage_free_bytes) + used_storage_perc = instance.useraccount.storage_used_ratio * 100 + return f"{used_storage} {free_storage} ({used_storage_perc:.2f}%)" + def get_search_results(self, request, queryset, search_term): filters = search_parser( request, diff --git a/docker-app/qfieldcloud/core/templates/admin/project_files_widget.html b/docker-app/qfieldcloud/core/templates/admin/project_files_widget.html index 1000a255e..b0675d27e 100644 --- a/docker-app/qfieldcloud/core/templates/admin/project_files_widget.html +++ b/docker-app/qfieldcloud/core/templates/admin/project_files_widget.html @@ -169,6 +169,16 @@ .then((files) => refreshTableContents({ files })) .catch((error) => refreshTableContents({ error })); }; + const filesize10 = n => { + switch(true) { + case n < 10 ** 6: + return (n / 10 ** 3).toFixed(3) + " KB"; + case n < 10 ** 9: + return (n / 10 ** 6).toFixed(3) + " MB"; + default: + return (n / 10 ** 9).toFixed(3) + " GB"; + } + }; const refreshTableContents = ({ error, isLoading = false, files = [] }) => { $tbody.innerHTML = ''; $count.innerHTML = ''; @@ -196,7 +206,7 @@ $trow.querySelector('td:nth-child(1)').innerHTML = file.name; $trow.querySelector('td:nth-child(2)').innerHTML = file.last_modified; - $trow.querySelector('td:nth-child(3)').innerHTML = file.size; + $trow.querySelector('td:nth-child(3)').innerHTML = `${filesize10(file.size)} KB`; for (const version of file.versions) { const $option = document.createElement('option'); @@ -220,9 +230,11 @@ }); $downloadBtn.addEventListener('click', () => { - window.open(buildApiUrl(`files/${projectId}/${file.name}/`, { + const pathToFile = `files/${projectId}/${file.name}/` + const buildApiUrlWithPath = () => buildApiUrl(pathToFile, { version: $versionsSelect.value, - })); + }); + window.open($versionsSelect.value ? buildApiUrlWithPath() : pathToFile); }); $deleteBtn.addEventListener('click', () => { diff --git a/docker-app/qfieldcloud/core/templatetags/filters.py b/docker-app/qfieldcloud/core/templatetags/filters.py new file mode 100644 index 000000000..c42e5405f --- /dev/null +++ b/docker-app/qfieldcloud/core/templatetags/filters.py @@ -0,0 +1,52 @@ +from django import template +from django.utils import formats +from django.utils.html import avoid_wrapping +from django.utils.translation import gettext as _ +from django.utils.translation import ngettext + +register = template.Library() + + +@register.filter(is_safe=True) +def filesizeformat10(bytes_) -> str: + """ + Format the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB, + 102 bytes, etc.). + + Unlike Django's `filesizeformat` which uses powers of 2 (e.g. 1024KB==1MB), `filesizeformat10` uses powers of 10 (e.g. 1000KB==1MB) + """ + try: + bytes_ = int(bytes_) + except (TypeError, ValueError, UnicodeDecodeError): + value = ngettext("%(size)d byte", "%(size)d bytes", 0) % {"size": 0} + return avoid_wrapping(value) + + def filesize_number_format(value): + return formats.number_format(round(value, 1), 1) + + KB = 10**3 + MB = 10**6 + GB = 10**9 + TB = 10**12 + PB = 10**15 + + negative = bytes_ < 0 + if negative: + bytes_ = -bytes_ # Allow formatting of negative numbers. + + if bytes_ < KB: + value = ngettext("%(size)d byte", "%(size)d bytes", bytes_) % {"size": bytes_} + elif bytes_ < MB: + value = _("%s KB") % filesize_number_format(bytes_ / KB) + elif bytes_ < GB: + value = _("%s MB") % filesize_number_format(bytes_ / MB) + elif bytes_ < TB: + value = _("%s GB") % filesize_number_format(bytes_ / GB) + elif bytes_ < PB: + value = _("%s TB") % filesize_number_format(bytes_ / TB) + else: + value = _("%s PB") % filesize_number_format(bytes_ / PB) + + if negative: + value = "-%s" % value + return avoid_wrapping(value) diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index 76b34e386..aeb50aa1a 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -121,11 +121,9 @@ def get_s3_bucket() -> mypy_boto3_s3.service_resource.Bucket: def get_s3_client() -> mypy_boto3_s3.Client: """Get a new S3 client instance using Django settings""" - s3_client = boto3.client( + s3_session = get_s3_session() + s3_client = s3_session.client( "s3", - region_name=settings.STORAGE_REGION_NAME, - aws_access_key_id=settings.STORAGE_ACCESS_KEY_ID, - aws_secret_access_key=settings.STORAGE_SECRET_ACCESS_KEY, endpoint_url=settings.STORAGE_ENDPOINT_URL, ) return s3_client diff --git a/docker-app/qfieldcloud/core/views/files_views.py b/docker-app/qfieldcloud/core/views/files_views.py index 84455ef49..f5f001bc2 100644 --- a/docker-app/qfieldcloud/core/views/files_views.py +++ b/docker-app/qfieldcloud/core/views/files_views.py @@ -141,6 +141,12 @@ def post(self, request, projectid, filename, format=None): logger.info( 'The key "file" was not found in `request.data`.', extra={ + "data_for_key_text_content": str(request.data.get("text", ""))[ + :1000 + ], + "data_for_key_text_len": len(request.data.get("text", "")), + "request_content_length": request.META.get("CONTENT_LENGTH"), + "request_content_type": request.META.get("CONTENT_TYPE"), "request_data": list(request.data.keys()), "request_files": list(request.FILES.keys()), }, diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index 31905d478..9d2bc4cf2 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -144,7 +144,9 @@ ], "APP_DIRS": True, "OPTIONS": { - "builtins": [], + "builtins": [ + "qfieldcloud.core.templatetags.filters", + ], "context_processors": [ "django.template.context_processors.debug", "django.template.context_processors.request", @@ -263,12 +265,13 @@ SENTRY_SAMPLE_RATE = float(os.environ.get("SENTRY_SAMPLE_RATE", 1)) def before_send(event, hint): - from qfieldcloud.core.exceptions import ProjectAlreadyExistsError + from qfieldcloud.core.exceptions import ProjectAlreadyExistsError, QuotaError from rest_framework.exceptions import ValidationError ignored_exceptions = ( ValidationError, ProjectAlreadyExistsError, + QuotaError, ) if "exc_info" in hint: diff --git a/docker-app/qfieldcloud/subscription/models.py b/docker-app/qfieldcloud/subscription/models.py index 06c2b11d6..b067ef18a 100644 --- a/docker-app/qfieldcloud/subscription/models.py +++ b/docker-app/qfieldcloud/subscription/models.py @@ -19,6 +19,8 @@ from .exceptions import NotPremiumPlanException +logger = logging.getLogger(__name__) + def get_subscription_model() -> "Subscription": return apps.get_model(settings.QFIELDCLOUD_SUBSCRIPTION_MODEL) @@ -655,7 +657,7 @@ def update_subscription( update_fields.append(attr_name) setattr(subscription, attr_name, attr_value) - logging.info(f"Updated subscription's fields: {', '.join(update_fields)}") + logger.info(f"Updated subscription's fields: {', '.join(update_fields)}") subscription.save(update_fields=update_fields) @@ -726,6 +728,9 @@ def create_subscription( ), "Creating a trial plan requires `active_since` to be a valid datetime object" active_until = active_since + timedelta(days=config.TRIAL_PERIOD_DAYS) + logger.info( + f"Creating trial subscription from {active_since=} to {active_until=}" + ) trial_subscription = cls.objects.create( plan=plan, account=account, @@ -748,6 +753,7 @@ def create_subscription( regular_plan = plan regular_active_since = active_since + logger.info(f"Creating regular subscription from {regular_active_since}") regular_subscription = cls.objects.create( plan=regular_plan, account=account, diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt index 140265b3f..4855e09b0 100644 --- a/docker-app/requirements.txt +++ b/docker-app/requirements.txt @@ -13,7 +13,7 @@ coreschema==0.0.4 cryptography==36.0.1 defusedxml==0.7.1 Deprecated==1.2.13 -Django==3.2.17 +Django==3.2.18 django-allauth==0.44.0 django-appconf==1.0.5 django-auditlog==2.2.2 diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 7c6cf1632..c032adf69 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -3,8 +3,6 @@ version: '3.9' services: app: - depends_on: - - geodb build: args: DEBUG_BUILD: ${DEBUG} diff --git a/docker-compose.override.local.yml b/docker-compose.override.local.yml index e51125b10..f7e5c4de9 100644 --- a/docker-compose.override.local.yml +++ b/docker-compose.override.local.yml @@ -25,10 +25,6 @@ services: # mount the source for live reload - ./docker-app/qfieldcloud:/usr/src/app/qfieldcloud - ./docker-app/worker_wrapper:/usr/src/app/worker_wrapper - depends_on: - - db - - redis - - app smtp4dev: image: rnwood/smtp4dev:v3 diff --git a/docker-compose.yml b/docker-compose.yml index 54e833c5a..bba2e9a7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,8 +73,6 @@ services: WEB_HTTP_PORT: ${WEB_HTTP_PORT} WEB_HTTPS_PORT: ${WEB_HTTPS_PORT} TRANSFORMATION_GRIDS_VOLUME_NAME: ${COMPOSE_PROJECT_NAME}_transformation_grids - depends_on: - - redis logging: driver: "json-file" options: @@ -86,6 +84,9 @@ services: ofelia.job-exec.runcrons.no-overlap: "true" ofelia.job-exec.runcrons.schedule: "@every 1m" ofelia.job-exec.runcrons.command: python manage.py runcrons + depends_on: + - db + - redis nginx: image: nginx:stable @@ -166,9 +167,6 @@ services: - ${LOG_DIRECTORY}:/log - ${TMP_DIRECTORY}:/tmp logging: *default-logging - depends_on: - - redis - - app scale: ${QFIELDCLOUD_WORKER_REPLICAS} ofelia: