PB-1115: Validate all models on save

app/bod/management/commands/bod_migrate.py

@@ -81,8 +81,19 @@ def import_providers(self) -> None:
             processed.add(legacy_id)
 
             # Get or create provider
-            provider, is_new_model = Provider.objects.get_or_create(_legacy_id=legacy_id)
-            if is_new_model:
+            is_new_model = False
+            provider = Provider.objects.filter(_legacy_id=legacy_id).first()
+            if not provider:
+                is_new_model = True
+                provider = Provider.objects.create(
+                    _legacy_id=legacy_id,
+                    acronym_de="undefined",
+                    acronym_fr="undefined",
+                    acronym_en="undefined",
+                    name_de="undefined",
+                    name_fr="undefined",
+                    name_en="undefined"
+                )
                 self.increment_counter('provider', 'added')
                 self.print(f"Added provider '{organization.name_en}'")
 
@@ -141,9 +152,19 @@ def import_attribution(
         """
 
         # Get or create attribution
-        attribution, is_new_model = provider.attribution_set.get_or_create(_legacy_id=legacy_id)
-
-        if is_new_model:
+        is_new_model = False
+        attribution = provider.attribution_set.filter(_legacy_id=legacy_id).first()
+        if not attribution:
+            is_new_model = True
+            attribution = provider.attribution_set.create(
+                _legacy_id=legacy_id,
+                name_de="undefined",
+                name_fr="undefined",
+                name_en="undefined",
+                description_de="undefined",
+                description_fr="undefined",
+                description_en="undefined"
+            )
             self.increment_counter('attribution', 'added')
             self.print(f"Added attribution '{organization.attribution}'")
 
@@ -183,7 +204,8 @@ def update_attribution(
             changed = self.update_model(
                 attribution,
                 attribution_attribute,
-                getattr(translation, translation_attribute, '') or '',
+                getattr(translation, translation_attribute, '') or organization.attribution or
+                'undefined',
                 is_new_model
             )
             any_changed = any_changed or changed
@@ -208,10 +230,18 @@ def import_datasets(self, provider: Provider, attribution: Attribution, legacy_i
             processed.add(legacy_id)
 
             # Get or create dataset
-            dataset, is_new_model = Dataset.objects.get_or_create(
+            is_new_model = False
+            dataset = Dataset.objects.filter(
                 provider=provider, attribution=attribution, _legacy_id=legacy_id
-            )
-            if is_new_model:
+            ).first()
+            if not dataset:
+                is_new_model = True
+                dataset = Dataset.objects.create(
+                    provider=provider,
+                    attribution=attribution,
+                    _legacy_id=legacy_id,
+                    slug='undefined'
+                )
                 self.increment_counter('dataset', 'added')
                 self.print(f"Added dataset '{bod_dataset.id_dataset}'")
 

app/distributions/migrations/0006_alter_dataset_slug.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.0.9 on 2024-11-27 06:48
+
+import utils.fields
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('distributions', '0005_dataset__legacy_id_alter_dataset_slug_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataset',
+            name='slug',
+            field=utils.fields.CustomSlugField(max_length=100, verbose_name='Slug'),
+        ),
+    ]

app/distributions/models.py

@@ -1,4 +1,9 @@
+from typing import Iterable
+
+from utils.fields import CustomSlugField
+
 from django.db import models
+from django.db.models.base import ModelBase
 from django.utils.translation import pgettext_lazy as _
 
 
@@ -31,6 +36,22 @@ def __str__(self) -> str:
         help_text="This field is used to track objects imported from the BOD"
     )
 
+    def save(
+        self,
+        force_insert: bool | tuple[ModelBase, ...] = False,
+        force_update: bool = False,
+        using: str | None = None,
+        update_fields: Iterable[str] | None = None
+    ) -> None:
+
+        self.full_clean()
+        super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields
+        )
+
 
 class Dataset(models.Model):
 
@@ -40,7 +61,7 @@ def __str__(self) -> str:
         return str(self.slug)
 
 
-    slug = models.SlugField(_(_context, "Slug"), max_length=100)
+    slug = CustomSlugField(_(_context, "Slug"), max_length=100)
     created = models.DateTimeField(_(_context, "Created"), auto_now_add=True)
     updated = models.DateTimeField(_(_context, "Updated"), auto_now=True)
 
@@ -54,3 +75,19 @@ def __str__(self) -> str:
         db_index=False,
         help_text="This field is used to track objects imported from the BOD"
     )
+
+    def save(
+        self,
+        force_insert: bool | tuple[ModelBase, ...] = False,
+        force_update: bool = False,
+        using: str | None = None,
+        update_fields: Iterable[str] | None = None
+    ) -> None:
+
+        self.full_clean()
+        super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields
+        )

app/provider/models.py

@@ -1,4 +1,7 @@
+from typing import Iterable
+
 from django.db import models
+from django.db.models.base import ModelBase
 from django.utils.translation import pgettext_lazy as _
 
 
@@ -32,3 +35,19 @@ def __str__(self) -> str:
         db_index=False,
         help_text="This field is used to track objects imported from the BOD"
     )
+
+    def save(
+        self,
+        force_insert: bool | tuple[ModelBase, ...] = False,
+        force_update: bool = False,
+        using: str | None = None,
+        update_fields: Iterable[str] | None = None
+    ) -> None:
+
+        self.full_clean()
+        super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields
+        )

app/tests/access/test_access_api.py

@@ -10,7 +10,14 @@
 
 @fixture(name='provider')
 def fixture_provider(db):
-    yield Provider.objects.create()
+    yield Provider.objects.create(
+        acronym_de="BAFU",
+        acronym_fr="OFEV",
+        acronym_en="FOEN",
+        name_de="Bundesamt für Umwelt",
+        name_fr="Office fédéral de l'environnement",
+        name_en="Federal Office for the Environment"
+    )
 
 
 @fixture(name='user')

app/tests/access/test_access_models.py

@@ -13,7 +13,14 @@
 
 @fixture(name='provider')
 def fixture_provider(db):
-    yield Provider.objects.create()
+    yield Provider.objects.create(
+        acronym_de="BAFU",
+        acronym_fr="OFEV",
+        acronym_en="FOEN",
+        name_de="Bundesamt für Umwelt",
+        name_fr="Office fédéral de l'environnement",
+        name_en="Federal Office for the Environment"
+    )
 
 
 @patch('access.models.Client')

app/tests/bod/test_bod_migrate_command.py

@@ -92,22 +92,22 @@ def test_command_updates(bod_contact_organisation, bod_dataset):
         name_fr="XXX",
         name_en="XXX",
         acronym_de="BAFU",
-        acronym_fr="",
-        acronym_en="",
+        acronym_fr="XXX",
+        acronym_en="XXX",
         _legacy_id=bod_contact_organisation.pk_contactorganisation_id
     )
     attribution = Attribution.objects.create(
         name_de="XXX",
         name_fr="XXX",
         name_en="XXX",
         description_de="BAFU",
-        description_fr="",
-        description_en="",
+        description_fr="XXX",
+        description_en="XXX",
         provider=provider,
         _legacy_id=bod_contact_organisation.pk_contactorganisation_id
     )
     dataset = Dataset.objects.create(
-        slug="XXX", provider=provider, attribution=attribution, _legacy_id=bod_dataset.id
+        slug="xxx", provider=provider, attribution=attribution, _legacy_id=bod_dataset.id
     )
 
     out = StringIO()
@@ -173,7 +173,7 @@ def test_command_removes_orphaned(bod_dataset):
         provider=provider,
         _legacy_id=16
     )
-    Dataset.objects.create(slug="XXX", provider=provider, attribution=attribution, _legacy_id=160)
+    Dataset.objects.create(slug="xxx", provider=provider, attribution=attribution, _legacy_id=160)
 
     # Add objects which will not be removed
     provider = Provider.objects.create(
@@ -193,7 +193,7 @@ def test_command_removes_orphaned(bod_dataset):
         description_en="YYY",
         provider=provider
     )
-    Dataset.objects.create(slug="YYYY", provider=provider, attribution=attribution)
+    Dataset.objects.create(slug="yyyy", provider=provider, attribution=attribution)
 
     out = StringIO()
     call_command("bod_migrate", verbosity=2, stdout=out)
@@ -209,7 +209,7 @@ def test_command_removes_orphaned(bod_dataset):
     assert {'BAFU', 'YYYY'} == set(Provider.objects.values_list('acronym_de', flat=True))
     assert {'BAFU', 'YYYY'} == set(Attribution.objects.values_list('name_de', flat=True))
     assert {'ch.bafu.auen-vegetationskarten',
-            'YYYY'} == set(Dataset.objects.values_list('slug', flat=True))
+            'yyyy'} == set(Dataset.objects.values_list('slug', flat=True))
 
 
 def test_command_does_not_import_if_dry_run(bod_dataset):
@@ -243,7 +243,7 @@ def test_command_clears_existing_data(bod_dataset):
         description_en="XXX",
         provider=provider
     )
-    Dataset.objects.create(slug="YYYY", provider=provider, attribution=attribution)
+    Dataset.objects.create(slug="yyyy", provider=provider, attribution=attribution)
 
     out = StringIO()
     call_command("bod_migrate", clear=True, stdout=out)

app/tests/distributions/test_attribution_model.py

@@ -3,13 +3,20 @@
 from pytest import fixture
 from pytest import raises
 
-from django.db import IntegrityError
+from django.core.exceptions import ValidationError
 from django.forms import ModelForm
 
 
 @fixture(name='provider')
 def fixture_provider(db):
-    yield Provider.objects.create(acronym_de="ENSI")
+    yield Provider.objects.create(
+        acronym_de="BAFU",
+        acronym_fr="OFEV",
+        acronym_en="FOEN",
+        name_de="Bundesamt für Umwelt",
+        name_fr="Office fédéral de l'environnement",
+        name_en="Federal Office for the Environment"
+    )
 
 
 def test_object_created_in_db_with_all_fields_defined(provider):
@@ -45,7 +52,7 @@ def test_object_created_in_db_with_all_fields_defined(provider):
     assert actual.description_it == attribution["description_it"]
     assert actual.description_rm == attribution["description_rm"]
 
-    assert actual.provider.acronym_de == "ENSI"
+    assert actual.provider.acronym_de == "BAFU"
 
 
 def test_object_created_in_db_with_optional_fields_null(provider):
@@ -83,7 +90,7 @@ def test_object_created_in_db_with_optional_fields_null(provider):
 
 
 def test_raises_exception_when_creating_db_object_with_mandatory_field_null(provider):
-    with raises(IntegrityError):
+    with raises(ValidationError):
         Attribution.objects.create(name_de=None, provider=provider)