From f86b437ee3fb61943808191b8e7b70f937c937d5 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Sun, 6 Sep 2020 20:54:49 +0200 Subject: [PATCH 01/13] Scaffold new compliance app --- src/compliance/__init__.py | 0 src/compliance/admin.py | 3 +++ src/compliance/apps.py | 5 +++++ src/compliance/migrations/__init__.py | 0 src/compliance/models.py | 3 +++ src/compliance/tests.py | 3 +++ src/compliance/views.py | 3 +++ src/mailguardian/settings/core_settings.py | 1 + 8 files changed, 18 insertions(+) create mode 100644 src/compliance/__init__.py create mode 100644 src/compliance/admin.py create mode 100644 src/compliance/apps.py create mode 100644 src/compliance/migrations/__init__.py create mode 100644 src/compliance/models.py create mode 100644 src/compliance/tests.py create mode 100644 src/compliance/views.py diff --git a/src/compliance/__init__.py b/src/compliance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/compliance/admin.py b/src/compliance/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/src/compliance/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/src/compliance/apps.py b/src/compliance/apps.py new file mode 100644 index 00000000..82773aa5 --- /dev/null +++ b/src/compliance/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ComplianceConfig(AppConfig): + name = 'compliance' diff --git a/src/compliance/migrations/__init__.py b/src/compliance/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/compliance/models.py b/src/compliance/models.py new file mode 100644 index 00000000..71a83623 --- /dev/null +++ b/src/compliance/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/src/compliance/tests.py b/src/compliance/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/compliance/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/compliance/views.py b/src/compliance/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/src/compliance/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/src/mailguardian/settings/core_settings.py b/src/mailguardian/settings/core_settings.py index c8308155..20e2d370 100644 --- a/src/mailguardian/settings/core_settings.py +++ b/src/mailguardian/settings/core_settings.py @@ -49,6 +49,7 @@ 'guardian', 'django_premailer', 'core', + 'compliance', 'frontend', 'setup_wizard', 'domains', From 7eef774a6a2ff70f08df4635ea8caa1898ae4352 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Mon, 7 Sep 2020 21:36:04 +0200 Subject: [PATCH 02/13] Initial implementation of `compliance` app --- src/compliance/middleware.py | 74 ++++++ src/compliance/migrations/0001_initial.py | 40 ++++ src/compliance/models.py | 266 +++++++++++++++++++++ src/compliance/receivers.py | 57 +++++ src/compliance/registry.py | 126 ++++++++++ src/compliance/tools.py | 128 ++++++++++ src/mailguardian/settings/core_settings.py | 1 + 7 files changed, 692 insertions(+) create mode 100644 src/compliance/middleware.py create mode 100644 src/compliance/migrations/0001_initial.py create mode 100644 src/compliance/receivers.py create mode 100644 src/compliance/registry.py create mode 100644 src/compliance/tools.py diff --git a/src/compliance/middleware.py b/src/compliance/middleware.py new file mode 100644 index 00000000..a228ce76 --- /dev/null +++ b/src/compliance/middleware.py @@ -0,0 +1,74 @@ +import threading +import time +from functools import partial +from django.apps import apps +from django.conf import settings +from django.db.models.signals import pre_save +from django.utils.deprecation import MiddlewareMixin +from .models import DataLogEntry + +threadlocal = threading.local() + + +class DataLogMiddleware(MiddlewareMixin): + """ + Middleware to couple the request's user to log items. This is accomplished by currying the signal receiver with the + user from the request (or None if the user is not authenticated). + """ + + def process_request(self, request): + """ + Gets the current user from the request and prepares and connects a signal receiver with the user already + attached to it. + """ + # Initialize thread local storage + threadlocal.datalog = { + 'signal_duid': (self.__class__, time.time()), + 'remote_addr': request.META.get('REMOTE_ADDR'), + } + + # In case of proxy, set 'original' address + if request.META.get('HTTP_X_FORWARDED_FOR'): + threadlocal.datalog['remote_addr'] = request.META.get('HTTP_X_FORWARDED_FOR').split(',')[0] + + # Connect signal for automatic logging + if hasattr(request, 'user') and getattr(request.user, 'is_authenticated', False): + set_actor = partial(self.set_actor, user=request.user, signal_duid=threadlocal.datalog['signal_duid']) + pre_save.connect(set_actor, sender=DataLogEntry, dispatch_uid=threadlocal.datalog['signal_duid'], weak=False) + + def process_response(self, request, response): + """ + Disconnects the signal receiver to prevent it from staying active. + """ + if hasattr(threadlocal, 'datalog'): + pre_save.disconnect(sender=DataLogEntry, dispatch_uid=threadlocal.datalog['signal_duid']) + + return response + + def process_exception(self, request, exception): + """ + Disconnects the signal receiver to prevent it from staying active in case of an exception. + """ + if hasattr(threadlocal, 'datalog'): + pre_save.disconnect(sender=DataLogEntry, dispatch_uid=threadlocal.datalog['signal_duid']) + + return None + + @staticmethod + def set_actor(user, sender, instance, signal_duid, **kwargs): + """ + Signal receiver with an extra, required 'user' kwarg. This method becomes a real (valid) signal receiver when + it is curried with the actor. + """ + if hasattr(threadlocal, 'datalog'): + if signal_duid != threadlocal.datalog['signal_duid']: + return + try: + app_label, model_name = settings.AUTH_USER_MODEL.split('.') + auth_user_model = apps.get_model(app_label, model_name) + except ValueError: + auth_user_model = apps.get_model('auth', 'user') + if sender == DataLogEntry and isinstance(user, auth_user_model) and instance.actor is None: + instance.actor = user + + instance.remote_addr = threadlocal.datalog['remote_addr'] \ No newline at end of file diff --git a/src/compliance/migrations/0001_initial.py b/src/compliance/migrations/0001_initial.py new file mode 100644 index 00000000..deedf8c9 --- /dev/null +++ b/src/compliance/migrations/0001_initial.py @@ -0,0 +1,40 @@ +# Generated by Django 3.1.1 on 2020-09-07 19:16 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='DataLogEntry', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_pk', models.CharField(db_index=True, max_length=255, verbose_name='object pk')), + ('object_id', models.BigIntegerField(blank=True, db_index=True, null=True, verbose_name='object id')), + ('object_repr', models.TextField(verbose_name='object representation')), + ('action', models.CharField(blank=True, choices=[('created', 'create'), ('updated', 'update'), ('deleted', 'delete')], max_length=32, null=True, verbose_name='action')), + ('changes', models.TextField(blank=True, verbose_name='change message')), + ('remote_addr', models.GenericIPAddressField(blank=True, null=True, verbose_name='remote address')), + ('timestamp', models.DateTimeField(auto_now_add=True, verbose_name='timestamp')), + ('additional_data', models.JSONField(blank=True, null=True, verbose_name='additional data')), + ('actor', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL, verbose_name='actor')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype', verbose_name='content type')), + ], + options={ + 'verbose_name': 'datalog entry', + 'verbose_name_plural': 'datalog entries', + 'ordering': ['-timestamp'], + 'get_latest_by': 'timestamp', + }, + ), + ] diff --git a/src/compliance/models.py b/src/compliance/models.py index 71a83623..4b6449fc 100644 --- a/src/compliance/models.py +++ b/src/compliance/models.py @@ -1,3 +1,269 @@ from django.db import models +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ +from django.core.exceptions import FieldDoesNotExist +from django.contrib.contenttypes.models import ContentType +from django.utils import formats, timezone +from django.utils.encoding import smart_str +from django.db.models import QuerySet, Q, Field +from dateutil.tz import gettz +from dateutil import parser +import json +import ast + +class LogEntryManager(models.Manager): + """ + Custom manager for the :py:class:`LogEntry` model. + """ + + def log_create(self, instance, **kwargs): + """ + Helper method to create a new log entry. This method automatically populates some fields when no explicit value + is given. + :param instance: The model instance to log a change for. + :type instance: Model + :param kwargs: Field overrides for the :py:class:`DataLogEntry` object. + :return: The new log entry or `None` if there were no changes. + :rtype: LogEntry + """ + changes = kwargs.get('changes', None) + pk = self._get_pk_value(instance) + + if changes is not None: + kwargs.setdefault('content_type', ContentType.objects.get_for_model(instance)) + kwargs.setdefault('object_pk', pk) + kwargs.setdefault('object_repr', smart_str(instance)) + + if isinstance(pk, int): + kwargs.setdefault('object_id', pk) + + get_additional_data = getattr(instance, 'get_additional_data', None) + if callable(get_additional_data): + kwargs.setdefault('additional_data', get_additional_data()) + + # Delete log entries with the same pk as a newly created model. This should only be necessary when an pk is + # used twice. + if kwargs.get('action', None) is DataLogEntry.Action.CREATE: + if kwargs.get('object_id', None) is not None and self.filter(content_type=kwargs.get('content_type'), + object_id=kwargs.get( + 'object_id')).exists(): + self.filter(content_type=kwargs.get('content_type'), object_id=kwargs.get('object_id')).delete() + else: + self.filter(content_type=kwargs.get('content_type'), object_pk=kwargs.get('object_pk', '')).delete() + # save LogEntry to same database instance is using + db = instance._state.db + return self.create(**kwargs) if db is None or db == '' else self.using(db).create(**kwargs) + return None + + def get_for_object(self, instance): + """ + Get log entries for the specified model instance. + :param instance: The model instance to get log entries for. + :type instance: Model + :return: QuerySet of log entries for the given model instance. + :rtype: QuerySet + """ + # Return empty queryset if the given model instance is not a model instance. + if not isinstance(instance, models.Model): + return self.none() + + content_type = ContentType.objects.get_for_model(instance.__class__) + pk = self._get_pk_value(instance) + + if isinstance(pk, int): + return self.filter(content_type=content_type, object_id=pk) + else: + return self.filter(content_type=content_type, object_pk=smart_str(pk)) + + def get_for_objects(self, queryset): + """ + Get log entries for the objects in the specified queryset. + :param queryset: The queryset to get the log entries for. + :type queryset: QuerySet + :return: The LogEntry objects for the objects in the given queryset. + :rtype: QuerySet + """ + if not isinstance(queryset, QuerySet) or queryset.count() == 0: + return self.none() + + content_type = ContentType.objects.get_for_model(queryset.model) + primary_keys = list(queryset.values_list(queryset.model._meta.pk.name, flat=True)) + + if isinstance(primary_keys[0], int): + return self.filter(content_type=content_type).filter(Q(object_id__in=primary_keys)).distinct() + elif isinstance(queryset.model._meta.pk, models.UUIDField): + primary_keys = [smart_str(pk) for pk in primary_keys] + return self.filter(content_type=content_type).filter(Q(object_pk__in=primary_keys)).distinct() + else: + return self.filter(content_type=content_type).filter(Q(object_pk__in=primary_keys)).distinct() + + def get_for_model(self, model): + """ + Get log entries for all objects of a specified type. + :param model: The model to get log entries for. + :type model: class + :return: QuerySet of log entries for the given model. + :rtype: QuerySet + """ + # Return empty queryset if the given object is not valid. + if not issubclass(model, models.Model): + return self.none() + + content_type = ContentType.objects.get_for_model(model) + + return self.filter(content_type=content_type) + + def _get_pk_value(self, instance): + """ + Get the primary key field value for a model instance. + :param instance: The model instance to get the primary key for. + :type instance: Model + :return: The primary key value of the given model instance. + """ + pk_field = instance._meta.pk.name + pk = getattr(instance, pk_field, None) + + # Check to make sure that we got an pk not a model object. + if isinstance(pk, models.Model): + pk = self._get_pk_value(pk) + return pk # Create your models here. +class DataLogEntry(models.Model): + class Meta: + get_latest_by = 'timestamp' + ordering = ['-timestamp'] + verbose_name = _("datalog entry") + verbose_name_plural = _("datalog entries") + + content_type = models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE, related_name='+', + verbose_name=_("content type")) + object_pk = models.CharField(db_index=True, max_length=255, verbose_name=_("object pk")) + object_id = models.BigIntegerField(blank=True, db_index=True, null=True, verbose_name=_("object id")) + object_repr = models.TextField(verbose_name=_("object representation")) + action = models.CharField(choices=( + ('created', _("create")), + ('updated', _("update")), + ('deleted', _("delete")), + ), verbose_name=_("action"), max_length=32, null=True, blank=True) + changes = models.TextField(blank=True, verbose_name=_("change message")) + actor = models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, blank=True, null=True, + related_name='+', verbose_name=_("actor")) + remote_addr = models.GenericIPAddressField(blank=True, null=True, verbose_name=_("remote address")) + timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_("timestamp")) + additional_data = models.JSONField(blank=True, null=True, verbose_name=_("additional data")) + + def __str__(self): + if self.action == self.Action.CREATE: + fstring = _("Created {repr:s}") + elif self.action == self.Action.UPDATE: + fstring = _("Updated {repr:s}") + elif self.action == self.Action.DELETE: + fstring = _("Deleted {repr:s}") + else: + fstring = _("Logged {repr:s}") + + return fstring.format(repr=self.object_repr) + + @property + def changes_dict(self): + """ + :return: The changes recorded in this log entry as a dictionary object. + """ + try: + return json.loads(self.changes) + except ValueError: + return {} + + @property + def changes_str(self, colon=': ', arrow=' \u2192 ', separator='; '): + """ + Return the changes recorded in this log entry as a string. The formatting of the string can be customized by + setting alternate values for colon, arrow and separator. If the formatting is still not satisfying, please use + :py:func:`LogEntry.changes_dict` and format the string yourself. + :param colon: The string to place between the field name and the values. + :param arrow: The string to place between each old and new value. + :param separator: The string to place between each field. + :return: A readable string of the changes in this log entry. + """ + substrings = [] + + for field, values in self.changes_dict.items(): + substring = '{field_name:s}{colon:s}{old:s}{arrow:s}{new:s}'.format( + field_name=field, + colon=colon, + old=values[0], + arrow=arrow, + new=values[1], + ) + substrings.append(substring) + + return separator.join(substrings) + + @property + def changes_display_dict(self): + """ + :return: The changes recorded in this log entry intended for display to users as a dictionary object. + """ + # Get the model and model_fields + from .registry import auditlog + model = self.content_type.model_class() + model_fields = auditlog.get_model_fields(model._meta.model) + changes_display_dict = {} + # grab the changes_dict and iterate through + for field_name, values in self.changes_dict.items(): + # try to get the field attribute on the model + try: + field = model._meta.get_field(field_name) + except FieldDoesNotExist: + changes_display_dict[field_name] = values + continue + values_display = [] + # handle choices fields and Postgres ArrayField to get human readable version + choices_dict = None + if getattr(field, 'choices') and len(field.choices) > 0: + choices_dict = dict(field.choices) + if hasattr(field, 'base_field') and isinstance(field.base_field, Field) and getattr(field.base_field, 'choices') and len(field.base_field.choices) > 0: + choices_dict = dict(field.base_field.choices) + + if choices_dict: + for value in values: + try: + value = ast.literal_eval(value) + if type(value) is [].__class__: + values_display.append(', '.join([choices_dict.get(val, 'None') for val in value])) + else: + values_display.append(choices_dict.get(value, 'None')) + except ValueError: + values_display.append(choices_dict.get(value, 'None')) + except: + values_display.append(choices_dict.get(value, 'None')) + else: + try: + field_type = field.get_internal_type() + except AttributeError: + # if the field is a relationship it has no internal type and exclude it + continue + for value in values: + # handle case where field is a datetime, date, or time type + if field_type in ["DateTimeField", "DateField", "TimeField"]: + try: + value = parser.parse(value) + if field_type == "DateField": + value = value.date() + elif field_type == "TimeField": + value = value.time() + elif field_type == "DateTimeField": + value = value.replace(tzinfo=timezone.utc) + value = value.astimezone(gettz(settings.TIME_ZONE)) + value = formats.localize(value) + except ValueError: + pass + # check if length is longer than 140 and truncate with ellipsis + if len(value) > 140: + value = "{}...".format(value[:140]) + + values_display.append(value) + verbose_name = model_fields['mapping_fields'].get(field.name, getattr(field, 'verbose_name', field.name)) + changes_display_dict[verbose_name] = values_display + return changes_display_dict \ No newline at end of file diff --git a/src/compliance/receivers.py b/src/compliance/receivers.py new file mode 100644 index 00000000..de63360d --- /dev/null +++ b/src/compliance/receivers.py @@ -0,0 +1,57 @@ +import json +from .tools import model_instance_diff +from .models import DataLogEntry + + +def log_create(sender, instance, created, **kwargs): + """ + Signal receiver that creates a log entry when a model instance is first saved to the database. + Direct use is discouraged, connect your model through :py:func:`auditlog.registry.register` instead. + """ + if created: + changes = model_instance_diff(None, instance) + + log_entry = DataLogEntry.objects.log_create( + instance, + action=DataLogEntry.Action.CREATE, + changes=json.dumps(changes), + ) + + +def log_update(sender, instance, **kwargs): + """ + Signal receiver that creates a log entry when a model instance is changed and saved to the database. + Direct use is discouraged, connect your model through :py:func:`auditlog.registry.register` instead. + """ + if instance.pk is not None: + try: + old = sender.objects.get(pk=instance.pk) + except sender.DoesNotExist: + pass + else: + new = instance + + changes = model_instance_diff(old, new) + + # Log an entry only if there are changes + if changes: + log_entry = DataLogEntry.objects.log_create( + instance, + action=DataLogEntry.Action.UPDATE, + changes=json.dumps(changes), + ) + + +def log_delete(sender, instance, **kwargs): + """ + Signal receiver that creates a log entry when a model instance is deleted from the database. + Direct use is discouraged, connect your model through :py:func:`auditlog.registry.register` instead. + """ + if instance.pk is not None: + changes = model_instance_diff(instance, None) + + log_entry = DataLogEntry.objects.log_create( + instance, + action=DataLogEntry.Action.DELETE, + changes=json.dumps(changes), + ) \ No newline at end of file diff --git a/src/compliance/registry.py b/src/compliance/registry.py new file mode 100644 index 00000000..f93d0708 --- /dev/null +++ b/src/compliance/registry.py @@ -0,0 +1,126 @@ +from typing import Dict, Callable, Optional, List, Tuple +from django.db.models import Model +from django.db.models.base import ModelBase +from django.db.models.signals import pre_save, post_save, post_delete, ModelSignal + +DispatchUID = Tuple[int, str, int] + + +class DataLogModelRegistry(object): + """ + A registry that keeps track of the models that use datalog to track changes. + """ + + def __init__(self, create: bool = True, update: bool = True, delete: bool = True, + custom: Optional[Dict[ModelSignal, Callable]] = None): + from .receivers import log_create, log_update, log_delete + + self._registry = {} + self._signals = {} + + if create: + self._signals[post_save] = log_create + if update: + self._signals[pre_save] = log_update + if delete: + self._signals[post_delete] = log_delete + + if custom is not None: + self._signals.update(custom) + + def register(self, model: ModelBase = None, include_fields: Optional[List[str]] = None, + exclude_fields: Optional[List[str]] = None, mapping_fields: Optional[Dict[str, str]] = None): + """ + Register a model with datalog. DataLog will then track mutations on this model's instances. + :param model: The model to register. + :param include_fields: The fields to include. Implicitly excludes all other fields. + :param exclude_fields: The fields to exclude. Overrides the fields to include. + :param mapping_fields: Mapping from field names to strings in diff. + """ + + if include_fields is None: + include_fields = [] + if exclude_fields is None: + exclude_fields = [] + if mapping_fields is None: + mapping_fields = {} + + def registrar(cls): + """Register models for a given class.""" + if not issubclass(cls, Model): + raise TypeError("Supplied model is not a valid model.") + + self._registry[cls] = { + 'include_fields': include_fields, + 'exclude_fields': exclude_fields, + 'mapping_fields': mapping_fields, + } + self._connect_signals(cls) + + # We need to return the class, as the decorator is basically + # syntactic sugar for: + # MyClass = datalog.register(MyClass) + return cls + + if model is None: + # If we're being used as a decorator, return a callable with the + # wrapper. + return lambda cls: registrar(cls) + else: + # Otherwise, just register the model. + registrar(model) + + def contains(self, model: ModelBase) -> bool: + """ + Check if a model is registered with datalog. + :param model: The model to check. + :return: Whether the model has been registered. + :rtype: bool + """ + return model in self._registry + + def unregister(self, model: ModelBase) -> None: + """ + Unregister a model with datalog. This will not affect the database. + :param model: The model to unregister. + """ + try: + del self._registry[model] + except KeyError: + pass + else: + self._disconnect_signals(model) + + def get_models(self) -> List[ModelBase]: + return list(self._registry.keys()) + + def get_model_fields(self, model: ModelBase): + return { + 'include_fields': list(self._registry[model]['include_fields']), + 'exclude_fields': list(self._registry[model]['exclude_fields']), + 'mapping_fields': dict(self._registry[model]['mapping_fields']), + } + + def _connect_signals(self, model): + """ + Connect signals for the model. + """ + for signal in self._signals: + receiver = self._signals[signal] + signal.connect(receiver, sender=model, dispatch_uid=self._dispatch_uid(signal, model)) + + def _disconnect_signals(self, model): + """ + Disconnect signals for the model. + """ + for signal, receiver in self._signals.items(): + signal.disconnect(sender=model, dispatch_uid=self._dispatch_uid(signal, model)) + + def _dispatch_uid(self, signal, model) -> DispatchUID: + """ + Generate a dispatch_uid. + """ + return self.__hash__(), model.__qualname__, signal.__hash__() + + +datalog = DataLogModelRegistry() \ No newline at end of file diff --git a/src/compliance/tools.py b/src/compliance/tools.py new file mode 100644 index 00000000..eb2d75b9 --- /dev/null +++ b/src/compliance/tools.py @@ -0,0 +1,128 @@ +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Model, NOT_PROVIDED, DateTimeField +from django.utils import timezone +from django.utils.encoding import smart_text + + +def track_field(field): + """ + Returns whether the given field should be tracked by Auditlog. + Untracked fields are many-to-many relations and relations to the Auditlog LogEntry model. + :param field: The field to check. + :type field: Field + :return: Whether the given field should be tracked. + :rtype: bool + """ + from auditlog.models import LogEntry + # Do not track many to many relations + if field.many_to_many: + return False + + # Do not track relations to LogEntry + if getattr(field, 'remote_field', None) is not None and field.remote_field.model == LogEntry: + return False + + return True + + +def get_fields_in_model(instance): + """ + Returns the list of fields in the given model instance. Checks whether to use the official _meta API or use the raw + data. This method excludes many to many fields. + :param instance: The model instance to get the fields for + :type instance: Model + :return: The list of fields for the given model (instance) + :rtype: list + """ + assert isinstance(instance, Model) + + return [f for f in instance._meta.get_fields() if track_field(f)] + + +def get_field_value(obj, field): + """ + Gets the value of a given model instance field. + :param obj: The model instance. + :type obj: Model + :param field: The field you want to find the value of. + :type field: Any + :return: The value of the field as a string. + :rtype: str + """ + if isinstance(field, DateTimeField): + # DateTimeFields are timezone-aware, so we need to convert the field + # to its naive form before we can accurately compare them for changes. + try: + value = field.to_python(getattr(obj, field.name, None)) + if value is not None and settings.USE_TZ and not timezone.is_naive(value): + value = timezone.make_naive(value, timezone=timezone.utc) + except ObjectDoesNotExist: + value = field.default if field.default is not NOT_PROVIDED else None + else: + try: + value = smart_text(getattr(obj, field.name, None)) + except ObjectDoesNotExist: + value = field.default if field.default is not NOT_PROVIDED else None + + return value + + +def model_instance_diff(old, new): + """ + Calculates the differences between two model instances. One of the instances may be ``None`` (i.e., a newly + created model or deleted model). This will cause all fields with a value to have changed (from ``None``). + :param old: The old state of the model instance. + :type old: Model + :param new: The new state of the model instance. + :type new: Model + :return: A dictionary with the names of the changed fields as keys and a two tuple of the old and new field values + as value. + :rtype: dict + """ + from auditlog.registry import auditlog + + if not (old is None or isinstance(old, Model)): + raise TypeError("The supplied old instance is not a valid model instance.") + if not (new is None or isinstance(new, Model)): + raise TypeError("The supplied new instance is not a valid model instance.") + + diff = {} + + if old is not None and new is not None: + fields = set(old._meta.fields + new._meta.fields) + model_fields = auditlog.get_model_fields(new._meta.model) + elif old is not None: + fields = set(get_fields_in_model(old)) + model_fields = auditlog.get_model_fields(old._meta.model) + elif new is not None: + fields = set(get_fields_in_model(new)) + model_fields = auditlog.get_model_fields(new._meta.model) + else: + fields = set() + model_fields = None + + # Check if fields must be filtered + if model_fields and (model_fields['include_fields'] or model_fields['exclude_fields']) and fields: + filtered_fields = [] + if model_fields['include_fields']: + filtered_fields = [field for field in fields + if field.name in model_fields['include_fields']] + else: + filtered_fields = fields + if model_fields['exclude_fields']: + filtered_fields = [field for field in filtered_fields + if field.name not in model_fields['exclude_fields']] + fields = filtered_fields + + for field in fields: + old_value = get_field_value(old, field) + new_value = get_field_value(new, field) + + if old_value != new_value: + diff[field.name] = (smart_text(old_value), smart_text(new_value)) + + if len(diff) == 0: + diff = None + + return diff \ No newline at end of file diff --git a/src/mailguardian/settings/core_settings.py b/src/mailguardian/settings/core_settings.py index 20e2d370..74fcc01e 100644 --- a/src/mailguardian/settings/core_settings.py +++ b/src/mailguardian/settings/core_settings.py @@ -67,6 +67,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'compliance.middleware.DataLogMiddleware', ] ROOT_URLCONF = 'mailguardian.urls' From a81f1a0e11fd8c03a62723704bb67e770494a39b Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Mon, 7 Sep 2020 21:36:38 +0200 Subject: [PATCH 03/13] Implement initial data logging on models --- src/core/models.py | 10 +++++++++- src/domains/models.py | 5 ++++- src/lists/models.py | 3 +++ src/mail/models.py | 10 ++++++++++ src/spamassassin/models.py | 6 +++++- 5 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/core/models.py b/src/core/models.py index 1f23141e..16720130 100644 --- a/src/core/models.py +++ b/src/core/models.py @@ -18,6 +18,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.utils.crypto import get_random_string from django_cryptography.fields import encrypt +from compliance.registry import datalog # Create your models here. class UserManager(BaseUserManager): @@ -269,4 +270,11 @@ def generate_codes(self, user): @receiver(post_save, sender=settings.AUTH_USER_MODEL) def create_auth_token(sender, instance=None, created=False, **kwargs): if created: - Token.objects.create(user=instance) \ No newline at end of file + Token.objects.create(user=instance) + +datalog.register(model=User) +datalog.register(model=Setting) +datalog.register(model=MailScannerHost) +datalog.register(model=ApplicationNotification) +datalog.register(model=ApplicationTask) +datalog.register(model=TwoFactorConfiguration) \ No newline at end of file diff --git a/src/domains/models.py b/src/domains/models.py index 68a36cac..2c246300 100644 --- a/src/domains/models.py +++ b/src/domains/models.py @@ -2,6 +2,7 @@ from django.db import models from django.conf import settings from django.utils.translation import gettext_lazy as _ +from compliance.registry import datalog # https://gist.github.com/solusipse/7ed8e1da104baaee3f05 @@ -29,4 +30,6 @@ class Meta: ), default="failover") def __str__(self): - return self.name \ No newline at end of file + return self.name + +datalog.register(model=Domain) \ No newline at end of file diff --git a/src/lists/models.py b/src/lists/models.py index cc7953ea..369d641e 100644 --- a/src/lists/models.py +++ b/src/lists/models.py @@ -2,6 +2,7 @@ from django.db import models from django.conf import settings from django.utils.translation import gettext_lazy as _ +from compliance.registry import datalog class ListEntry(models.Model): class Meta: @@ -32,3 +33,5 @@ def save(self, *args, **kwargs): if '@' in self.from_address: self.from_domain = self.from_address.split('@')[-1] super(ListEntry, self).save(*args, **kwargs) + +datalog.register(model=ListEntry) \ No newline at end of file diff --git a/src/mail/models.py b/src/mail/models.py index d34d27ec..1f00a581 100644 --- a/src/mail/models.py +++ b/src/mail/models.py @@ -3,6 +3,7 @@ from django.conf import settings import os, datetime, subprocess from django.utils.translation import gettext_lazy as _ +from compliance.registry import datalog # Create your models here. class Message(models.Model): @@ -162,3 +163,12 @@ class Meta: hostname = models.CharField(_('Hostname'), max_length=255, db_index=True, default='') active = models.BooleanField(_('Active'), default=0) comment = models.TextField(_('Comment')) + +datalog.register(model=Message) +datalog.register(model=Headers) +datalog.register(model=RblReport) +datalog.register(model=SpamReport) +datalog.register(model=McpReport) +datalog.register(model=MailscannerReport) +datalog.register(model=TransportLog) +datalog.register(model=SmtpRelay) \ No newline at end of file diff --git a/src/spamassassin/models.py b/src/spamassassin/models.py index 996435a2..69bfb181 100644 --- a/src/spamassassin/models.py +++ b/src/spamassassin/models.py @@ -2,6 +2,7 @@ from django.conf import settings import re, uuid, subprocess from django.utils.translation import gettext_lazy as _ +from compliance.registry import datalog # Create your models here. class Rule(models.Model): @@ -33,4 +34,7 @@ def sync_files(self): rule.value = match[1] rule.save() else: - RuleDescription.objects.get_or_create(key=match[0], value=match[1]) \ No newline at end of file + RuleDescription.objects.get_or_create(key=match[0], value=match[1]) + +datalog.register(model=Rule) +datalog.register(model=RuleDescription) \ No newline at end of file From fe21800fcdbfa8c5de297a40ed115c1392976d0a Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Mon, 7 Sep 2020 21:36:58 +0200 Subject: [PATCH 04/13] Log actions on 2FA --- src/core/viewsets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/viewsets.py b/src/core/viewsets.py index 3053991c..e1fc21e4 100644 --- a/src/core/viewsets.py +++ b/src/core/viewsets.py @@ -32,6 +32,7 @@ from django.conf import settings import datetime, pyotp from django.utils.translation import gettext_lazy as _ +from compliance.models import DataLogEntry # ViewSets define the view behavior. class UserViewSet(viewsets.ModelViewSet): @@ -160,6 +161,7 @@ def put_enable(self, request): user = get_object_or_404(User, pk=request.user.id) TwoFactorConfiguration.objects.create(user=user, totp_key=request.data['totp_key']) TwoFactorBackupCode().generate_codes(user=request.user) + DataLogEntry.objects.log_create(user, action='created', changes='The user {} enabled 2FA on their account'.format(user.__str__())) return Response({}, status=status.HTTP_201_CREATED) @action(methods=['get'], detail=False, permission_classes=[IsAuthenticated], url_path='qr', url_name='two-factor-qr-code') @@ -176,6 +178,7 @@ def delete_disable(self, request): user = get_object_or_404(User, pk=request.user.id) TwoFactorConfiguration.objects.get(user=user).delete() TwoFactorBackupCode.objects.filter(user=user).delete() + DataLogEntry.objects.log_create(user, action='deleted', changes='The user {} disabled 2FA on their account'.format(user.__str__())) return Response({}, status=status.HTTP_204_NO_CONTENT) class TwoFactorBackupCodeViewSet(viewsets.ModelViewSet): @@ -201,4 +204,5 @@ def get_my_backup_codes(self, request): if request.user.is_staff: qs = qs.filter(user=request.user) serializer = TwoFactorBackupCodeSerializer(qs, many=True, context={'request': request}) + DataLogEntry.objects.log_create(request.user, changes='The user {} requested their 2FA backup codes'.format(request.user.__str__())) return Response(serializer.data) \ No newline at end of file From d2b70af01d14159be0673eba0ec22b5618fd8558 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Mon, 7 Sep 2020 22:01:01 +0200 Subject: [PATCH 05/13] Bugfixes and API implementation --- src/compliance/models.py | 10 ++++++---- src/compliance/receivers.py | 12 ++++++------ src/compliance/serializers.py | 22 ++++++++++++++++++++++ src/compliance/tools.py | 16 ++++++++-------- src/compliance/viewsets.py | 14 ++++++++++++++ src/mailguardian/urls.py | 4 ++++ 6 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 src/compliance/serializers.py create mode 100644 src/compliance/viewsets.py diff --git a/src/compliance/models.py b/src/compliance/models.py index 4b6449fc..b835e97c 100644 --- a/src/compliance/models.py +++ b/src/compliance/models.py @@ -43,7 +43,7 @@ def log_create(self, instance, **kwargs): # Delete log entries with the same pk as a newly created model. This should only be necessary when an pk is # used twice. - if kwargs.get('action', None) is DataLogEntry.Action.CREATE: + if kwargs.get('action', None) is 'created': if kwargs.get('object_id', None) is not None and self.filter(content_type=kwargs.get('content_type'), object_id=kwargs.get( 'object_id')).exists(): @@ -153,12 +153,14 @@ class Meta: timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_("timestamp")) additional_data = models.JSONField(blank=True, null=True, verbose_name=_("additional data")) + objects = LogEntryManager() + def __str__(self): - if self.action == self.Action.CREATE: + if self.action == 'created': fstring = _("Created {repr:s}") - elif self.action == self.Action.UPDATE: + elif self.action == 'updated': fstring = _("Updated {repr:s}") - elif self.action == self.Action.DELETE: + elif self.action == 'deleted': fstring = _("Deleted {repr:s}") else: fstring = _("Logged {repr:s}") diff --git a/src/compliance/receivers.py b/src/compliance/receivers.py index de63360d..312ffa43 100644 --- a/src/compliance/receivers.py +++ b/src/compliance/receivers.py @@ -6,14 +6,14 @@ def log_create(sender, instance, created, **kwargs): """ Signal receiver that creates a log entry when a model instance is first saved to the database. - Direct use is discouraged, connect your model through :py:func:`auditlog.registry.register` instead. + Direct use is discouraged, connect your model through :py:func:`compliance.registry.register` instead. """ if created: changes = model_instance_diff(None, instance) log_entry = DataLogEntry.objects.log_create( instance, - action=DataLogEntry.Action.CREATE, + action='created', changes=json.dumps(changes), ) @@ -21,7 +21,7 @@ def log_create(sender, instance, created, **kwargs): def log_update(sender, instance, **kwargs): """ Signal receiver that creates a log entry when a model instance is changed and saved to the database. - Direct use is discouraged, connect your model through :py:func:`auditlog.registry.register` instead. + Direct use is discouraged, connect your model through :py:func:`compliance.registry.register` instead. """ if instance.pk is not None: try: @@ -37,7 +37,7 @@ def log_update(sender, instance, **kwargs): if changes: log_entry = DataLogEntry.objects.log_create( instance, - action=DataLogEntry.Action.UPDATE, + action='updated', changes=json.dumps(changes), ) @@ -45,13 +45,13 @@ def log_update(sender, instance, **kwargs): def log_delete(sender, instance, **kwargs): """ Signal receiver that creates a log entry when a model instance is deleted from the database. - Direct use is discouraged, connect your model through :py:func:`auditlog.registry.register` instead. + Direct use is discouraged, connect your model through :py:func:`compliance.registry.register` instead. """ if instance.pk is not None: changes = model_instance_diff(instance, None) log_entry = DataLogEntry.objects.log_create( instance, - action=DataLogEntry.Action.DELETE, + action='deleted', changes=json.dumps(changes), ) \ No newline at end of file diff --git a/src/compliance/serializers.py b/src/compliance/serializers.py new file mode 100644 index 00000000..58361192 --- /dev/null +++ b/src/compliance/serializers.py @@ -0,0 +1,22 @@ +from rest_framework import serializers +from .models import DataLogEntry + +class DataLogEntrySerializer(serializers.HyperlinkedModelSerializer): + content_type_name = serializers.SerializerMethodField() + class Meta: + model = DataLogEntry + fields = ( + 'content_type_name', + 'object_pk', + 'object_id', + 'object_repr', + 'action', + 'changes', + 'actor', + 'remote_addr', + 'timestamp', + 'additional_data', + ) + + def get_content_type_name(self, obj): + return obj.content_type.__str__() \ No newline at end of file diff --git a/src/compliance/tools.py b/src/compliance/tools.py index eb2d75b9..0adfa586 100644 --- a/src/compliance/tools.py +++ b/src/compliance/tools.py @@ -7,20 +7,20 @@ def track_field(field): """ - Returns whether the given field should be tracked by Auditlog. - Untracked fields are many-to-many relations and relations to the Auditlog LogEntry model. + Returns whether the given field should be tracked by datalog. + Untracked fields are many-to-many relations and relations to the datalog LogEntry model. :param field: The field to check. :type field: Field :return: Whether the given field should be tracked. :rtype: bool """ - from auditlog.models import LogEntry + from .models import DataLogEntry # Do not track many to many relations if field.many_to_many: return False # Do not track relations to LogEntry - if getattr(field, 'remote_field', None) is not None and field.remote_field.model == LogEntry: + if getattr(field, 'remote_field', None) is not None and field.remote_field.model == DataLogEntry: return False return True @@ -80,7 +80,7 @@ def model_instance_diff(old, new): as value. :rtype: dict """ - from auditlog.registry import auditlog + from .registry import datalog if not (old is None or isinstance(old, Model)): raise TypeError("The supplied old instance is not a valid model instance.") @@ -91,13 +91,13 @@ def model_instance_diff(old, new): if old is not None and new is not None: fields = set(old._meta.fields + new._meta.fields) - model_fields = auditlog.get_model_fields(new._meta.model) + model_fields = datalog.get_model_fields(new._meta.model) elif old is not None: fields = set(get_fields_in_model(old)) - model_fields = auditlog.get_model_fields(old._meta.model) + model_fields = datalog.get_model_fields(old._meta.model) elif new is not None: fields = set(get_fields_in_model(new)) - model_fields = auditlog.get_model_fields(new._meta.model) + model_fields = datalog.get_model_fields(new._meta.model) else: fields = set() model_fields = None diff --git a/src/compliance/viewsets.py b/src/compliance/viewsets.py new file mode 100644 index 00000000..a5ece0a9 --- /dev/null +++ b/src/compliance/viewsets.py @@ -0,0 +1,14 @@ +from rest_framework import viewsets +from .models import DataLogEntry +from .serializers import DataLogEntrySerializer +from rest_framework.permissions import IsAdminUser + +class DataLogEntryViewSet(viewsets.ModelViewSet): + queryset = DataLogEntry.objects.all() + serializer_class = DataLogEntrySerializer + permission_classes = (IsAdminUser,) + model = DataLogEntry + + def get_queryset(self): + qs = super(DataLogEntryViewSet, self).get_queryset() + return qs \ No newline at end of file diff --git a/src/mailguardian/urls.py b/src/mailguardian/urls.py index b40ff56f..7d489939 100644 --- a/src/mailguardian/urls.py +++ b/src/mailguardian/urls.py @@ -29,6 +29,9 @@ TwoFactorConfigurationViewSet, TwoFactorBackupCodeViewSet ) +from compliance.viewsets import ( + DataLogEntryViewSet +) from mail.viewsets import ( MessageViewSet, SpamReportViewSet, @@ -88,6 +91,7 @@ router.register(r'notifications', ApplicationNotificationViewSet) router.register(r'two-factor', TwoFactorConfigurationViewSet) router.register(r'two-factor-codes', TwoFactorBackupCodeViewSet) +router.register(r'datalog', DataLogEntryViewSet) urlpatterns = [ path('', IndexTemplateView.as_view()), From b8c2dfc69c70228276c69f651a2cd31ad39ec26f Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:28:33 +0200 Subject: [PATCH 06/13] Reimplement logging UI --- assets/src/js/pages/Admin/AuditLog/Detail.vue | 4 ++-- assets/src/js/pages/Admin/AuditLog/Index.vue | 4 ++-- assets/src/js/pages/Reports/Index.vue | 4 ---- assets/src/js/pages/Tools/Index.vue | 4 ++++ 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/assets/src/js/pages/Admin/AuditLog/Detail.vue b/assets/src/js/pages/Admin/AuditLog/Detail.vue index fcf362e4..7154c1ff 100644 --- a/assets/src/js/pages/Admin/AuditLog/Detail.vue +++ b/assets/src/js/pages/Admin/AuditLog/Detail.vue @@ -20,7 +20,7 @@
- {{ entry.module }} + {{ entry.content_type_name }}
@@ -103,7 +103,7 @@ export default { methods: { get() { this.setLoading(true); - axios.get('/api/audit-log/'+this.id+'/').then(response => { + axios.get('/api/datalog/'+this.id+'/').then(response => { this.entry = response.data; this.setLoading(false); }).catch(error => { diff --git a/assets/src/js/pages/Admin/AuditLog/Index.vue b/assets/src/js/pages/Admin/AuditLog/Index.vue index b0a24793..b43989d5 100644 --- a/assets/src/js/pages/Admin/AuditLog/Index.vue +++ b/assets/src/js/pages/Admin/AuditLog/Index.vue @@ -16,7 +16,7 @@ {{ entry.timestamp | ago }} - {{ entry.module }} + {{ entry.content_type_name }} {{ entry.object_pk }} {{ entry.action_name }} {{ entry.actor_email }} @@ -56,7 +56,7 @@ export default { if (query && page) { qs = '?search='+query+'&page='+page; } - axios.get('/api/audit-log/'+qs).then(response => { + axios.get('/api/datalog/'+qs).then(response => { this.log = response.data.results; this.setLoading(false); }).catch(error => { diff --git a/assets/src/js/pages/Reports/Index.vue b/assets/src/js/pages/Reports/Index.vue index 3116f9cc..8b8b3ed4 100644 --- a/assets/src/js/pages/Reports/Index.vue +++ b/assets/src/js/pages/Reports/Index.vue @@ -139,10 +139,6 @@
Spam rule hits
--> - - Audit Log
- Show the audit log -
diff --git a/assets/src/js/pages/Tools/Index.vue b/assets/src/js/pages/Tools/Index.vue index 9e4f884a..64da60f2 100644 --- a/assets/src/js/pages/Tools/Index.vue +++ b/assets/src/js/pages/Tools/Index.vue @@ -32,6 +32,10 @@ Application update status
Check for updates for the application itself + + Audit Log
+ Show the audit log + Data import
Perform import of data into various parts of the application From 21a0f69e1ba6a8044985ff32454c73fcacad6912 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:29:41 +0200 Subject: [PATCH 07/13] Remove unused libraries --- requirements.txt | 1 - src/mailguardian/settings/core_settings.py | 2 -- upgrade-reqs.txt | 1 - 3 files changed, 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index d0d32529..94536ae8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,6 @@ django-cryptography==1.0 django-encrypted-model-fields==0.5.8 django-extensions==3.0.8 django-filter==2.3.0 -django-guardian==2.3.0 django-jsonfield==1.4.0 django-premailer==0.2.0 django-rest-auth==0.9.5 diff --git a/src/mailguardian/settings/core_settings.py b/src/mailguardian/settings/core_settings.py index 74fcc01e..1cf0f300 100644 --- a/src/mailguardian/settings/core_settings.py +++ b/src/mailguardian/settings/core_settings.py @@ -46,7 +46,6 @@ 'rest_framework', 'rest_framework.authtoken', 'rest_auth', - 'guardian', 'django_premailer', 'core', 'compliance', @@ -185,7 +184,6 @@ # https://django-guardian.readthedocs.io/en/stable/configuration.html#configuration AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', # this is default - 'guardian.backends.ObjectPermissionBackend', ) # Django-Premailer diff --git a/upgrade-reqs.txt b/upgrade-reqs.txt index d8395238..2656dac9 100644 --- a/upgrade-reqs.txt +++ b/upgrade-reqs.txt @@ -22,7 +22,6 @@ django-cryptography>=1.0 django-encrypted-model-fields>=0.5.8 django-extensions>=3.0.5 django-filter>=2.3.0 -django-guardian>=2.3.0 django-jsonfield>=1.4.0 django-premailer>=0.2.0 django-rest-auth>=0.9.5 From 95e2046560ffe8233334d737410faddc8d1e156f Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:30:33 +0200 Subject: [PATCH 08/13] Add missing serializer fields --- src/compliance/serializers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/compliance/serializers.py b/src/compliance/serializers.py index 58361192..5c709e37 100644 --- a/src/compliance/serializers.py +++ b/src/compliance/serializers.py @@ -3,9 +3,12 @@ class DataLogEntrySerializer(serializers.HyperlinkedModelSerializer): content_type_name = serializers.SerializerMethodField() + actor_email = serializers.SerializerMethodField() class Meta: model = DataLogEntry fields = ( + 'id', + 'url', 'content_type_name', 'object_pk', 'object_id', @@ -13,10 +16,14 @@ class Meta: 'action', 'changes', 'actor', + 'actor_email', 'remote_addr', 'timestamp', 'additional_data', ) def get_content_type_name(self, obj): - return obj.content_type.__str__() \ No newline at end of file + return obj.content_type.__str__() + + def get_actor_email(self, obj): + return obj.actor.email if obj.actor else None \ No newline at end of file From 76375bd538634c328b3fab510716dbce37d7eb32 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:31:00 +0200 Subject: [PATCH 09/13] Log when user logs in, logs out or login fails --- src/core/models.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/core/models.py b/src/core/models.py index 16720130..2c9461b5 100644 --- a/src/core/models.py +++ b/src/core/models.py @@ -19,6 +19,9 @@ from django.utils.crypto import get_random_string from django_cryptography.fields import encrypt from compliance.registry import datalog +from compliance.models import DataLogEntry +from django.dispatch import receiver +from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed # Create your models here. class UserManager(BaseUserManager): @@ -277,4 +280,19 @@ def create_auth_token(sender, instance=None, created=False, **kwargs): datalog.register(model=MailScannerHost) datalog.register(model=ApplicationNotification) datalog.register(model=ApplicationTask) -datalog.register(model=TwoFactorConfiguration) \ No newline at end of file +datalog.register(model=TwoFactorConfiguration) + +@receiver(user_logged_in) +def user_logged_in_audit(sender, request, user, **kwargs): + print('User has logged in') + DataLogEntry.objects.log_create(user, action='updated', changes='User has logged in') + +@receiver(user_logged_out) +def user_logged_out_audit(sender, request, user, **kwargs): + print('User has logged out') + DataLogEntry.objects.log_create(user, action='deleted', changes='User has logged out') + +@receiver(user_login_failed) +def user_login_failed_audit(sender, credentials, **kwargs): + user = User.objects.get(email=kwargs['request'].user) + DataLogEntry.objects.log_create(user, changes='Login failed for username {}'.format(credentials['username'])) \ No newline at end of file From 244cdb89ad1920ab7a54ed0b850b8fa61bf9e879 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:32:22 +0200 Subject: [PATCH 10/13] Log GeoLite2 updates --- src/core/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/views.py b/src/core/views.py index 66160f63..341c7d31 100644 --- a/src/core/views.py +++ b/src/core/views.py @@ -2,6 +2,7 @@ from rest_framework.response import Response from .serializers import UserSerializer, LoginSerializer from .models import MailScannerConfiguration, User, TwoFactorConfiguration, TwoFactorBackupCode +from compliance.models import DataLogEntry from rest_framework.permissions import IsAuthenticated, IsAdminUser from rest_auth.views import LoginView as RestAuthBaseLoginView from rest_framework.parsers import FileUploadParser @@ -191,4 +192,5 @@ def get(self, request): if os.path.exists(os.path.join(settings.MAXMIND_DB_PATH, path, 'GeoLite2-Country.mmdb')): os.rename(os.path.join(settings.MAXMIND_DB_PATH, path, 'GeoLite2-Country.mmdb'), settings.MAXMIND_DB_FILE) shutil.rmtree(os.path.join(settings.MAXMIND_DB_PATH, path)) + DataLogEntry.objects.log_create(request.user, changes='User {} has performed an update of the MaxMind GeoLite2 database'.format(request.user.email)) return Response({}, status=status.HTTP_200_OK) From 30bdd176d1077b69dc80452d2ea30895b53359f3 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:32:57 +0200 Subject: [PATCH 11/13] Log interactions with single mails and the queue --- src/mail/viewsets.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mail/viewsets.py b/src/mail/viewsets.py index a0dd8144..63fc1d87 100644 --- a/src/mail/viewsets.py +++ b/src/mail/viewsets.py @@ -11,6 +11,7 @@ from pymailq.store import PostqueueStore from django.conf import settings from core.models import Setting, MailScannerHost +from compliance.models import DataLogEntry import datetime from django.db.models import Q import subprocess @@ -163,6 +164,7 @@ def get_message_mailscanner_report(self, request, pk=None): @action(methods=['get'], detail=False, permission_classes=[IsAdminUser], url_path='queue', url_name='message-queue') def get_queue(self, request): + DataLogEntry.objects.log_create(None, changes='User {} requested to view the mail queue'.format(request.user.email)) host_count = MailScannerHost.objects.count() store = PostqueueStore() store.load() @@ -218,7 +220,7 @@ def post_resend(self, request): if message['hostname'] == settings.APP_HOSTNAME: command = "{} -i {}".format(settings.POSTQUEUE_BIN, message['qid']) output = subprocess.check_output(command, shell=True) - message.released = True + DataLogEntry.objects.log_create(message, changes='Message {} was resent from the queue'.format(message.id)) message.save() response.append({ 'qid':message['qid'], 'command': command, 'output': output }) elif host_count > 0 and not settings.API_ONLY: @@ -306,6 +308,7 @@ def post_action_spam(self, request): if settings.APP_HOSTNAME == message.mailscanner_hostname: command = "{0} -p {1} -r {2}".format(settings.SA_BIN, settings.MAILSCANNER_CONFIG_DIR + '/spamassassin.conf', message.file_path()) output = subprocess.check_output(command, shell=True) + DataLogEntry.objects.log_create(message, action='updated', changes='Message was marked as spam') response.append({ 'id':message_id, 'command': command, 'output': output }) elif not settings.API_ONLY: token = Token.objects.get(user=request.user) @@ -344,6 +347,7 @@ def post_action_nonspam(self, request): if settings.APP_HOSTNAME == message.mailscanner_hostname: command = "{0} -p {1} -k {2}".format(settings.SA_BIN, settings.MAILSCANNER_CONFIG_DIR + '/spamassassin.conf', message.file_path()) output = subprocess.check_output(command, shell=True) + DataLogEntry.objects.log_create(message, action='updated', changes='Message was marked as not being spam') response.append({ 'id':message_id, 'command': command, 'output': output }) elif not settings.API_ONLY: token = Token.objects.get(user=request.user) From 230afbd80b13517ecf22a32ebbc5ee26716a6bd9 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:33:13 +0200 Subject: [PATCH 12/13] Opt in to TailwindCSS V2 features --- tailwind.config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tailwind.config.js b/tailwind.config.js index d8c203d9..c703dda4 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -7,7 +7,8 @@ module.exports = { './src/**/*.html' ], future: { - removeDeprecatedGapUtilities: true + removeDeprecatedGapUtilities: true, + purgeLayersByDefault: true }, theme: { container: { From ed799b0da724757e70b4cdeaa8f119ee66b205e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hansen Date: Wed, 9 Sep 2020 21:36:42 +0200 Subject: [PATCH 13/13] Log SpamAssassin rule updates --- src/spamassassin/viewsets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/spamassassin/viewsets.py b/src/spamassassin/viewsets.py index db6711fb..125c90e7 100644 --- a/src/spamassassin/viewsets.py +++ b/src/spamassassin/viewsets.py @@ -7,6 +7,7 @@ from rest_framework.response import Response from django.db.models import Q import datetime +from compliance.models import DataLogEntry class RuleViewSet(viewsets.ModelViewSet): queryset = Rule.objects.all() @@ -26,6 +27,7 @@ def post_sync_rule_descriptions(self, request): sa = RuleDescription() sa.sync_files() Setting.objects.update_or_create(key='sa.last_updated', defaults={'value':str(datetime.datetime.now())}) + DataLogEntry.objects.log_create(None, actor=request.user, changes='SpamAssassin rule update completed') except Exception as e: return Response({'message' : str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response({}, status=status.HTTP_204_NO_CONTENT)