From a9bbf31b7952f2969cfb01d1b2d14255bafff9ac Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 7 May 2024 09:25:45 +0300 Subject: [PATCH 01/20] Generate syllables for finnish addresses --- services/management/commands/index_search_columns.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index b038b9116..23c6c2e4e 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -5,7 +5,7 @@ from munigeo.models import Address, AdministrativeDivision from services.models import Service, ServiceNode, Unit -from services.search.utils import hyphenate +from services.search.utils import get_foreign_key_attr, hyphenate logger = logging.getLogger("search") @@ -37,7 +37,7 @@ def generate_syllables(model): for row in model.objects.all(): row.syllables_fi = [] for column in model.get_syllable_fi_columns(): - row_content = getattr(row, column, None) + row_content = get_foreign_key_attr(row, column) if row_content: # Rows migth be of type str or Array, if str # cast to array by splitting. @@ -86,12 +86,15 @@ def index_servicenodes(lang): class Command(BaseCommand): def handle(self, *args, **kwargs): - for lang in ["fi", "sv", "en"]: + for lang in ["fi", "sv", "en"][0:1]: key = "search_column_%s" % lang # Only generate syllables for the finnish language if lang == "fi": logger.info(f"Generating syllables for language: {lang}.") logger.info(f"Syllables generated for {generate_syllables(Unit)} Units") + logger.info( + f"Syllables generated for {generate_syllables(Address)} Addresses" + ) logger.info( f"Syllables generated for {generate_syllables(Service)} Services" ) From 7ab1e9f0ba3a7f77f26fdc9f1d03b1d035a99fb4 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 13:32:04 +0300 Subject: [PATCH 02/20] Add Address Kellonsoittajankatu 1 --- services/search/tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/search/tests/conftest.py b/services/search/tests/conftest.py index 020e1a5d8..6c3476745 100644 --- a/services/search/tests/conftest.py +++ b/services/search/tests/conftest.py @@ -277,6 +277,14 @@ def addresses(streets, municipality): number=1, full_name="Tarkk'ampujankatu 1", ) + Address.objects.create( + municipality_id=municipality.id, + location=Point(60.44879002342721, 22.283629416961055), + id=7, + street_id=46, + number=1, + full_name="Kellonsoittajankatu 1", + ) Address.objects.update(search_column_fi=get_search_column(Address, "fi")) return Address.objects.all() @@ -314,6 +322,7 @@ def streets(): Street.objects.create(id=43, name="Markulantie", municipality_id="turku") Street.objects.create(id=44, name="Yliopistonkatu", municipality_id="turku") Street.objects.create(id=45, name="Tarkk'ampujankatu", municipality_id="turku") + Street.objects.create(id=46, name="Kellonsoittajankatu", municipality_id="turku") return Street.objects.all() From 1b7bdad878fadf6efd27c343e538d652d406a9ae Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 13:34:56 +0300 Subject: [PATCH 03/20] Generate syllables for addresses --- services/search/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/search/tests/conftest.py b/services/search/tests/conftest.py index 6c3476745..554c8e931 100644 --- a/services/search/tests/conftest.py +++ b/services/search/tests/conftest.py @@ -285,6 +285,7 @@ def addresses(streets, municipality): number=1, full_name="Kellonsoittajankatu 1", ) + generate_syllables(Address) Address.objects.update(search_column_fi=get_search_column(Address, "fi")) return Address.objects.all() From 0ac6cf61d64a83627db3ab4773a4634efaaba77c Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 13:35:46 +0300 Subject: [PATCH 04/20] Test search with 'kello' --- services/search/tests/test_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/search/tests/test_api.py b/services/search/tests/test_api.py index 99d8b2c3d..d7e510cbd 100644 --- a/services/search/tests/test_api.py +++ b/services/search/tests/test_api.py @@ -121,6 +121,12 @@ def test_search( assert kurrapolku["location"]["type"] == "Point" assert kurrapolku["location"]["coordinates"][0] == 60.479032 assert kurrapolku["location"]["coordinates"][1] == 22.25417 + # Test search with 'kello' + url = reverse("search") + "?q=kello&type=address" + response = api_client.get(url) + results = response.json()["results"] + assert len(results) == 1 + assert results[0]["name"]["fi"] == "Kellonsoittajankatu 1" # Test address search with apostrophe in query url = reverse("search") + "?q=tarkk'ampujankatu&type=address" response = api_client.get(url) From 91b319a7fdec4b7ca6664686f238316f02425e52 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 13:36:25 +0300 Subject: [PATCH 05/20] Fix typo --- services/search/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/search/constants.py b/services/search/constants.py index 967e37a78..75111fed3 100644 --- a/services/search/constants.py +++ b/services/search/constants.py @@ -11,7 +11,7 @@ "Address", ) QUERY_PARAM_TYPE_NAMES = [m.lower() for m in SEARCHABLE_MODEL_TYPE_NAMES] -# None will slice to the end of list, e.g. no limit. +# None will slice to the end of list, i.e., no limit. DEFAULT_MODEL_LIMIT_VALUE = None # The limit value for the search query that search the search_view. "NULL" = no limit DEFAULT_SEARCH_SQL_LIMIT_VALUE = "NULL" From 2599bde9444e2ed882464ccb97fe65739971633d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 14:10:19 +0300 Subject: [PATCH 06/20] Conditonally select addresses to hyphenate --- .../commands/index_search_columns.py | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 23c6c2e4e..12b31b350 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -1,10 +1,13 @@ import logging +from datetime import datetime, timedelta from django.contrib.postgres.search import SearchVector from django.core.management.base import BaseCommand +from django.utils import timezone from munigeo.models import Address, AdministrativeDivision from services.models import Service, ServiceNode, Unit +from services.search.constants import HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS from services.search.utils import get_foreign_key_attr, hyphenate logger = logging.getLogger("search") @@ -27,14 +30,26 @@ def get_search_column(model, lang): return search_column -def generate_syllables(model): +def generate_syllables( + model, hyphenate_all_addresses=False, hyphenate_addresses_from=None +): """ Generates syllables for the given model. """ # Disable sending of signals model._meta.auto_created = True + save_kwargs = {} num_populated = 0 - for row in model.objects.all(): + if model.__name__ == "Address" and not hyphenate_all_addresses: + save_kwargs["skip_modified_at"] = True + if not hyphenate_addresses_from: + hyphenate_addresses_from = Address.objects.latest( + "modified_at" + ).modified_at - timedelta(days=HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS) + qs = model.objects.filter(modified_at__gte=hyphenate_addresses_from) + else: + qs = model.objects.all() + for row in qs: row.syllables_fi = [] for column in model.get_syllable_fi_columns(): row_content = get_foreign_key_attr(row, column) @@ -47,7 +62,7 @@ def generate_syllables(model): syllables = hyphenate(word) for s in syllables: row.syllables_fi.append(s) - row.save() + row.save(**save_kwargs) num_populated += 1 # Enable sending of signals model._meta.auto_created = False @@ -85,16 +100,45 @@ def index_servicenodes(lang): class Command(BaseCommand): - def handle(self, *args, **kwargs): - for lang in ["fi", "sv", "en"][0:1]: + + def add_arguments(self, parser): + parser.add_argument( + "--hyphenate_addresses_from", + nargs="?", + type=str, + help="Hyphenate addresses whose modified_at timestamp starts at given timestamp YYYY-MM-DDTHH:MM:SS", + ) + + parser.add_argument( + "--hyphenate_all_addresses", + action="store_true", + help="Hyphenate all addresses", + ) + + def handle(self, *args, **options): + hyphenate_all_addresses = options.get("hyphenate_all_addresses", None) + hyphenate_addresses_from = options.get("hyphenate_addresses_from", None) + + if hyphenate_addresses_from: + try: + hyphenate_addresses_from = timezone.make_aware( + datetime.strptime(hyphenate_addresses_from, "%Y-%m-%dT%H:%M:%S") + ) + except ValueError as err: + raise ValueError(err) + + for lang in ["fi", "sv", "en"]: key = "search_column_%s" % lang # Only generate syllables for the finnish language if lang == "fi": logger.info(f"Generating syllables for language: {lang}.") logger.info(f"Syllables generated for {generate_syllables(Unit)} Units") - logger.info( - f"Syllables generated for {generate_syllables(Address)} Addresses" + num_populated = generate_syllables( + Address, + hyphenate_all_addresses=hyphenate_all_addresses, + hyphenate_addresses_from=hyphenate_addresses_from, ) + logger.info(f"Syllables generated for {num_populated} Addresses") logger.info( f"Syllables generated for {generate_syllables(Service)} Services" ) From 630f089f3037c8df86474f5fc7e1957f23298640 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 14:11:22 +0300 Subject: [PATCH 07/20] Add param args to task index_search_columns --- smbackend_turku/tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/smbackend_turku/tasks.py b/smbackend_turku/tasks.py index 6999f01fe..59782188e 100644 --- a/smbackend_turku/tasks.py +++ b/smbackend_turku/tasks.py @@ -46,8 +46,11 @@ def geo_import_municipalities(name="geo_import_municipalities"): @shared_task_email -def index_search_columns(name="index_search_columns"): - management.call_command("index_search_columns") +def index_search_columns(*args, name="index_search_columns"): + if args: + management.call_command("index_search_columns", args) + else: + management.call_command("index_search_columns") @shared_task_email From f88d3952fc930f9c89cd8d3b6b8844e7be9df2d3 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 14:12:28 +0300 Subject: [PATCH 08/20] Add constant HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS --- services/search/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/search/constants.py b/services/search/constants.py index 75111fed3..fbb2e0937 100644 --- a/services/search/constants.py +++ b/services/search/constants.py @@ -17,3 +17,5 @@ DEFAULT_SEARCH_SQL_LIMIT_VALUE = "NULL" DEFAULT_TRIGRAM_THRESHOLD = 0.15 DEFAULT_RANK_THRESHOLD = 1 + +HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS = 7 From 163620c6f5a973749310eae945736d4126d1fb5c Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 8 May 2024 14:13:35 +0300 Subject: [PATCH 09/20] Add function 'get_foreign_key_attr' --- services/search/utils.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/services/search/utils.py b/services/search/utils.py index 42def8bf3..8674fb575 100644 --- a/services/search/utils.py +++ b/services/search/utils.py @@ -16,6 +16,22 @@ voikko.setNoUglyHyphenation(True) +def get_foreign_key_attr(obj, field): + """Get attr recursively by following foreign key relations + For example: + get_foreign_key_attr( + , "street__name_fi" + ) + """ + fields = field.split("__") + if len(fields) == 1: + return getattr(obj, fields[0], None) + else: + first_field = fields[0] + remaining_fields = "__".join(fields[1:]) + return get_foreign_key_attr(getattr(obj, first_field), remaining_fields) + + def is_compound_word(word): result = voikko.analyze(word) if len(result) == 0: @@ -25,7 +41,7 @@ def is_compound_word(word): def hyphenate(word): """ - Returns a list of syllables of the word if it is a compound word. + Returns a list of syllables of the word, if it is a compound word. """ word = word.strip() if is_compound_word(word): From 18db3b80990adc356afa30d8c49bb56e3d645b34 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 10:49:45 +0300 Subject: [PATCH 10/20] Add words only if more than one syllables --- services/management/commands/index_search_columns.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 12b31b350..723113dc4 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -60,10 +60,12 @@ def generate_syllables( row_content = row_content.split() for word in row_content: syllables = hyphenate(word) - for s in syllables: - row.syllables_fi.append(s) + if len(syllables) > 1: + for s in syllables: + row.syllables_fi.append(s) row.save(**save_kwargs) num_populated += 1 + # Enable sending of signals model._meta.auto_created = False return num_populated @@ -126,7 +128,6 @@ def handle(self, *args, **options): ) except ValueError as err: raise ValueError(err) - for lang in ["fi", "sv", "en"]: key = "search_column_%s" % lang # Only generate syllables for the finnish language From c366ee06d81a0d4a282945c36aae26a590a49121 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 14:03:52 +0300 Subject: [PATCH 11/20] Add model ExclusionWord --- services/migrations/0101_exclusionword.py | 37 +++++++++++++++++++++++ services/models/__init__.py | 2 +- services/models/search_rule.py | 13 ++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 services/migrations/0101_exclusionword.py diff --git a/services/migrations/0101_exclusionword.py b/services/migrations/0101_exclusionword.py new file mode 100644 index 000000000..f4edcc0fd --- /dev/null +++ b/services/migrations/0101_exclusionword.py @@ -0,0 +1,37 @@ +# Generated by Django 4.1.13 on 2024-05-13 10:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("services", "0100_alter_unitconnection_section_type"), + ] + + operations = [ + migrations.CreateModel( + name="ExclusionWord", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("word", models.CharField(max_length=100, verbose_name="Word")), + ( + "language_short", + models.CharField(max_length=2, verbose_name="Language short"), + ), + ], + options={ + "verbose_name": "Exclusion word", + "verbose_name_plural": "Exclusion words", + "ordering": ["-id"], + }, + ), + ] diff --git a/services/models/__init__.py b/services/models/__init__.py index 57037a8e9..a722e6851 100644 --- a/services/models/__init__.py +++ b/services/models/__init__.py @@ -2,7 +2,7 @@ from .department import Department from .keyword import Keyword from .notification import Announcement, ErrorMessage -from .search_rule import ExclusionRule +from .search_rule import ExclusionRule, ExclusionWord from .service import Service, UnitServiceDetails from .service_mapping import ServiceMapping from .service_node import ServiceNode diff --git a/services/models/search_rule.py b/services/models/search_rule.py index 78c9c32b9..e1f0d8fe1 100644 --- a/services/models/search_rule.py +++ b/services/models/search_rule.py @@ -13,3 +13,16 @@ class Meta: def __str__(self): return "%s : %s" % (self.word, self.exclusion) + + +class ExclusionWord(models.Model): + word = models.CharField(max_length=100, verbose_name=_("Word")) + language_short = models.CharField(max_length=2, verbose_name=_("Language short")) + + class Meta: + ordering = ["-id"] + verbose_name = _("Exclusion word") + verbose_name_plural = _("Exclusion words") + + def __str__(self): + return self.word From d6c68709ca8370b5876da53f4fbf24b4d064e32f Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 14:06:36 +0300 Subject: [PATCH 12/20] Add exclusion words fixture --- services/fixtures/exclusion_words.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 services/fixtures/exclusion_words.json diff --git a/services/fixtures/exclusion_words.json b/services/fixtures/exclusion_words.json new file mode 100644 index 000000000..ec2891920 --- /dev/null +++ b/services/fixtures/exclusion_words.json @@ -0,0 +1,18 @@ +[ + { + "model": "services.exclusionword", + "pk": 1, + "fields": { + "word": "katu", + "language_short": "fi" + } + }, + { + "model": "services.exclusionword", + "pk": 2, + "fields": { + "word": "tie", + "language_short": "fi" + } + } + ] \ No newline at end of file From 03ce0cf866c37ebacaeeb7f0defd13bd760f6dc0 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 14:07:21 +0300 Subject: [PATCH 13/20] Add has_exclusion_word_in_query function --- services/search/utils.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/services/search/utils.py b/services/search/utils.py index 8674fb575..02db2b1f2 100644 --- a/services/search/utils.py +++ b/services/search/utils.py @@ -3,9 +3,16 @@ import libvoikko from django.db import connection from django.db.models import Case, When +from django.db.models.functions import Lower from rest_framework.exceptions import ParseError -from services.models import ExclusionRule, ServiceNode, ServiceNodeUnitCount, Unit +from services.models import ( + ExclusionRule, + ExclusionWord, + ServiceNode, + ServiceNodeUnitCount, + Unit, +) from services.search.constants import ( DEFAULT_TRIGRAM_THRESHOLD, SEARCHABLE_MODEL_TYPE_NAMES, @@ -232,3 +239,16 @@ def get_search_exclusions(q): if rule: return rule.exclusion return "" + + +def has_exclusion_word_in_query(q_vals, language_short): + """ + To add/modify search exclusion words edit: services/fixtures/exclusion_words.json + To import words: ./manage.py loaddata services/fixtures/exclusion_words.json + """ + return ( + ExclusionWord.objects.filter(language_short=language_short) + .annotate(word_lower=Lower("word")) + .filter(word_lower__in=[q.lower() for q in q_vals]) + .exists() + ) From 7ab44e63c3ae9d3a6b798ce80382d63f0f60ba01 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 14:08:15 +0300 Subject: [PATCH 14/20] Return Bad request if exclusion word in search query --- services/search/api.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/services/search/api.py b/services/search/api.py index 39046fe67..8a3ce9187 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -27,9 +27,10 @@ from drf_spectacular.utils import extend_schema, OpenApiParameter from munigeo import api as munigeo_api from munigeo.models import Address, AdministrativeDivision -from rest_framework import serializers +from rest_framework import serializers, status from rest_framework.exceptions import ParseError from rest_framework.generics import GenericAPIView +from rest_framework.response import Response from services.api import ( TranslatedModelSerializer, @@ -60,6 +61,7 @@ get_search_exclusions, get_service_node_results, get_trigram_results, + has_exclusion_word_in_query, set_address_fields, set_service_node_unit_count, set_service_unit_count, @@ -331,7 +333,6 @@ def get(self, request): raise ParseError("Supply search terms with 'q=' ' or input=' '") if not re.match(r"^[\w\såäö.'+&|-]+$", q_val): - raise ParseError( "Invalid search terms, only letters, numbers, spaces and .'+-&| allowed." ) @@ -447,6 +448,13 @@ def get(self, request): search_query_str += f"& {q}:*" else: search_query_str = f"{q}:*" + + if has_exclusion_word_in_query(q_vals, language_short): + return Response( + f"Search query {q_vals} would return too many results", + status=status.HTTP_400_BAD_REQUEST, + ) + search_fn = "to_tsquery" if use_websearch: exclusions = get_search_exclusions(q) From a86ff0fc8a9843d6fb3462c7323c0d351005f4db Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 14:09:10 +0300 Subject: [PATCH 15/20] Add ExclusionWord fixture --- services/search/tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/search/tests/conftest.py b/services/search/tests/conftest.py index 554c8e931..852f585dc 100644 --- a/services/search/tests/conftest.py +++ b/services/search/tests/conftest.py @@ -24,6 +24,7 @@ from services.models import ( Department, ExclusionRule, + ExclusionWord, Service, ServiceNode, Unit, @@ -331,3 +332,9 @@ def streets(): def exclusion_rules(): ExclusionRule.objects.create(id=1, word="tekojää", exclusion="-nurmi") return ExclusionRule.objects.all() + + +@pytest.fixture +def exclusion_words(): + ExclusionWord.objects.create(id=1, word="katu", language_short="fi") + return ExclusionWord.objects.all() From 27e1501b05ee64448370e1bb86fdd85c4f7465a9 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 13 May 2024 14:09:30 +0300 Subject: [PATCH 16/20] Test exclusion word --- services/search/tests/test_api.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/search/tests/test_api.py b/services/search/tests/test_api.py index d7e510cbd..d24653dee 100644 --- a/services/search/tests/test_api.py +++ b/services/search/tests/test_api.py @@ -14,6 +14,7 @@ def test_search( accessibility_shortcoming, municipality, exclusion_rules, + exclusion_words, ): # Search for "museo" in entities: units,services and servicenods url = reverse("search") + "?q=museo&type=unit,service,servicenode" @@ -121,6 +122,16 @@ def test_search( assert kurrapolku["location"]["type"] == "Point" assert kurrapolku["location"]["coordinates"][0] == 60.479032 assert kurrapolku["location"]["coordinates"][1] == 22.25417 + # Test search with excluded word + url = reverse("search") + "?q=katu" + response = api_client.get(url) + assert response.status_code == 400 + url = reverse("search") + "?q=Katu" + response = api_client.get(url) + assert response.status_code == 400 + url = reverse("search") + "?q=koti katu" + response = api_client.get(url) + assert response.status_code == 400 # Test search with 'kello' url = reverse("search") + "?q=kello&type=address" response = api_client.get(url) From e2acbbd493c2013753e2ba7386ad22169627f41a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 16 May 2024 08:48:35 +0300 Subject: [PATCH 17/20] Bump django-munigeo from v0.2.76 to v0.2.83 --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index d8abc2146..981ca033a 100644 --- a/requirements.in +++ b/requirements.in @@ -7,7 +7,7 @@ django-modeltranslation flake8 requests requests_cache -git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.76#egg=django-munigeo +git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83#egg=django-munigeo pytz django-cors-headers django-extensions diff --git a/requirements.txt b/requirements.txt index d73186327..c9ce571b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -87,7 +87,7 @@ django-mptt==0.13.4 # via # -r requirements.in # django-munigeo -django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.76 +django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83 # via -r requirements.in django-polymorphic==3.1.0 # via -r requirements.in From 9023257c706d95d04d2634e7d4b48a6d71a7fef2 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:47:37 +0300 Subject: [PATCH 18/20] Use Quryset.iterator() when generating syllables --- services/management/commands/index_search_columns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 723113dc4..423c09288 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -49,7 +49,7 @@ def generate_syllables( qs = model.objects.filter(modified_at__gte=hyphenate_addresses_from) else: qs = model.objects.all() - for row in qs: + for row in qs.iterator(chunk_size=10000): row.syllables_fi = [] for column in model.get_syllable_fi_columns(): row_content = get_foreign_key_attr(row, column) @@ -128,6 +128,7 @@ def handle(self, *args, **options): ) except ValueError as err: raise ValueError(err) + for lang in ["fi", "sv", "en"]: key = "search_column_%s" % lang # Only generate syllables for the finnish language From a366907169e9977db18aa14ef457ebd4ae6eda50 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:27:02 +0300 Subject: [PATCH 19/20] Add cache --- services/search/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/search/api.py b/services/search/api.py index 8a3ce9187..502c80ebd 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -24,6 +24,8 @@ from django.db import connection, reset_queries from django.db.models import Count +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page from drf_spectacular.utils import extend_schema, OpenApiParameter from munigeo import api as munigeo_api from munigeo.models import Address, AdministrativeDivision @@ -320,6 +322,7 @@ def to_representation(self, obj): class SearchViewSet(GenericAPIView): queryset = Unit.objects.all() + @method_decorator(cache_page(60 * 60)) def get(self, request): model_limits = {} show_only_address = False From 2a4580b9c8f3b5c12fb4b1d62419295c5b5d3842 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:27:14 +0300 Subject: [PATCH 20/20] Add munigeo 0.2.84 --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 981ca033a..130f77761 100644 --- a/requirements.in +++ b/requirements.in @@ -7,7 +7,7 @@ django-modeltranslation flake8 requests requests_cache -git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83#egg=django-munigeo +git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.84#egg=django-munigeo pytz django-cors-headers django-extensions diff --git a/requirements.txt b/requirements.txt index c9ce571b6..b84008c5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -87,7 +87,7 @@ django-mptt==0.13.4 # via # -r requirements.in # django-munigeo -django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83 +django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.84 # via -r requirements.in django-polymorphic==3.1.0 # via -r requirements.in