Feature/update project settings

README.md

@@ -141,8 +141,12 @@ For Turku specific imports see smbackend_turku/README.md.
 ./manage.py geo_import helsinki --divisions
 ./manage.py index_search_columns
 ```
-
+Import exclude rules fixtures used by the search:
+```
+./manage.py loaddata services/fixtures/exclusion_rules.json
+```
 7. Redis
+
 Redis is used for caching and as a message broker for Celery.
 Install Redis. Ubuntu: `sudo apt-get install redis-server`
 
@@ -198,3 +202,6 @@ psql template1 -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 Mobility platform
 -----------------
 The mobility data platform of the service map is being developed as part of European Union Horizon 2020 programme funded SCALE-UP project (grant agreement no. 955332).
+
+For more information see: mobility_data/README.mk
+

config_dev.env.example

@@ -145,6 +145,23 @@ EMAIL_HOST_USER=example@example.com
 EMAIL_PORT=25
 EMAIL_USE_TLS=True
 
+# Django project log level, default INFO
+DJANGO_LOG_LEVEL=
+# Turku services importers log level, default DEBUG
+TURKU_SERVICES_IMPORT_LOG_LEVEL=
+# Search log level, default INFO
+SEARCH_LOG_LEVEL=
+# IoT APP, default INFO 
+IOT_LOG_LEVEL=
+# Eco counter, default INFO
+ECO_COUNTER_LOG_LEVEL=
+# Mobility data (includes importers), default INFO
+MOBILITY_DATA_LOG_LEVEL=
+# Bicycle networks APP, default INFO
+BICYCLE_NETWORK_LOG_LEVEL=
+# Street maintenance, default INFO
+STREET_MAINTENANCE_LOG_LEVEL=
+
 # Settings needed for enabling Turku area:
 #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv
 #TURKU_API_KEY=secret
@@ -184,4 +201,4 @@ YIT_TOKEN_URL=https://login.microsoftonline.com/86792d09-0d81-4899-8d66-95dfc96c
 KUNTEC_KEY=
 # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py)
 # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api
-TELRAAM_TOKEN=
+TELRAAM_TOKEN=

requirements.in

@@ -14,7 +14,7 @@ django-extensions
 psycopg2-binary<2.9
 django-mptt
 lxml>=4.9.1
-raven~=6.10.0
+sentry-sdk
 pip-tools
 python-dateutil
 pytest-django

requirements.txt

@@ -30,7 +30,9 @@ celery==5.2.3
  # django-celery-beat
  # django-celery-results
 certifi==2022.12.7
- # via requests
+ # via
+ # requests
+ # sentry-sdk
 charset-normalizer==2.0.6
  # via requests
 click==8.0.3
@@ -201,8 +203,6 @@ pyyaml==5.3.1
  # via
  # django-munigeo
  # drf-spectacular
-raven==6.10.0
- # via -r requirements.in
 redis==4.4.4
  # via -r requirements.in
 requests==2.26.0
@@ -215,6 +215,8 @@ requests-cache==0.8.1
  # via -r requirements.in
 requests-mock==1.9.3
  # via -r requirements.in
+sentry-sdk==1.9.0
+ # via -r requirements.in
 shapely==1.8.0
  # via -r requirements.in
 six==1.16.0
@@ -233,7 +235,9 @@ toml==0.10.2
  # pytest
  # pytest-cov
 tomli==1.2.1
- # via pep517
+ # via
+ # black
+ # pep517
 tqdm==4.62.3
  # via -r requirements.in
 tzdata==2022.1
@@ -248,6 +252,7 @@ urllib3==1.26.7
  # via
  # requests
  # requests-cache
+ # sentry-sdk
 vine==5.0.0
  # via
  # amqp

services/fixtures/exclusion_rules.json

@@ -0,0 +1,10 @@
+[
+ {
+ "model": "services.exclusionrule",
+ "pk": 1,
+ "fields": {
+ "word": "tekojää",
+ "exclusion": "-nurmi"
+ }
+ }
+ ]

services/migrations/0095_exclusionrule.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.1.7 on 2023-07-20 05:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("services", "0094_create_syllables_fi_columns"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ExclusionRule",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("word", models.CharField(max_length=100, verbose_name="Word")),
+ (
+ "exclusion",
+ models.CharField(max_length=100, verbose_name="Exclusion"),
+ ),
+ ],
+ options={
+ "verbose_name": "Exclusion rule",
+ "verbose_name_plural": "Exclusion rules",
+ "ordering": ["-id"],
+ },
+ ),
+ ]

services/models/__init__.py

@@ -2,6 +2,7 @@
 from .department import Department
 from .keyword import Keyword
 from .notification import Announcement, ErrorMessage
+from .search_rule import ExclusionRule
 from .service import Service, UnitServiceDetails
 from .service_mapping import ServiceMapping
 from .service_node import ServiceNode

services/models/search_rule.py

@@ -0,0 +1,15 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+
+class ExclusionRule(models.Model):
+ word = models.CharField(max_length=100, verbose_name=_("Word"))
+ exclusion = models.CharField(max_length=100, verbose_name=_("Exclusion"))
+
+ class Meta:
+ ordering = ["-id"]
+ verbose_name = _("Exclusion rule")
+ verbose_name_plural = _("Exclusion rules")
+
+ def __str__(self):
+ return "%s : %s" % (self.word, self.exclusion)

services/search/api.py

@@ -54,6 +54,7 @@
 from .utils import (
  get_all_ids_from_sql_results,
  get_preserved_order,
+ get_search_exclusions,
  get_service_node_results,
  get_trigram_results,
  set_address_fields,
@@ -212,6 +213,14 @@
  else:
  trigram_threshold = DEFAULT_TRIGRAM_THRESHOLD
 
+ if "use_websearch" in params:
+ try:
+ use_websearch = strtobool(params["use_websearch"])
+ except ValueError:
+ raise ParseError("'use_websearch' needs to be a boolean")
+ else:
+ use_websearch = True
+
  if "geometry" in params:
  try:
  show_geometry = strtobool(params["geometry"])
@@ -266,7 +275,7 @@
  config_language = LANGUAGES[language_short]
  search_query_str = None # Used in the raw sql
  # Build conditional query string that is used in the SQL query.
- # split my "," or whitespace
+ # split by "," or whitespace
  q_vals = re.split(r",\s+|\s+", q_val)
  q_vals = [s.strip().replace("'", "") for s in q_vals]
  for q in q_vals:
@@ -279,12 +288,17 @@
  search_query_str += f"& {q}:*"
  else:
  search_query_str = f"{q}:*"
-
+ search_fn = "to_tsquery"
+ if use_websearch:
+ exclusions = get_search_exclusions(q)
+ if exclusions:
+ search_fn = "websearch_to_tsquery"
+ search_query_str += f" {exclusions}"
  # This is ~100 times faster than using Djangos SearchRank and allows searching using wildard "|*"
  # and by rankig gives better results, e.g. extra fields weight is counted.
  sql = f"""
  SELECT id, type_name, name_{language_short}, ts_rank_cd(search_column_{language_short}, search_query)
- AS rank FROM search_view, to_tsquery('{config_language}','{search_query_str}') search_query
+ AS rank FROM search_view, {search_fn}('{config_language}','{search_query_str}') search_query
  WHERE search_query @@ search_column_{language_short}
  ORDER BY rank DESC LIMIT {sql_query_limit};
  """

services/search/specification.swagger.yaml

@@ -34,6 +34,16 @@ components:
  type: string
  example: fi
  default: fi
+ use_websearch_param:
+ name: use_websearch
+ in: query
+ schema:
+ type: boolean
+ default: true
+ description: >
+ "websearch_to_tsquery is a simplified version of to_tsquery with an alternative syntax, similar to the one used by web search engines." 
+ If disabled, uses 'to_tsquery' function to convert the query to 'tsquery'.
+ If enabled, exclusion rules are used when generating the query as it support the not "-" operator.
  order_units_by_num_services_param:
  name: order_units_by_num_services
  in: query
@@ -173,6 +183,7 @@ paths:
  - $ref: "#/components/parameters/q_param"
  - $ref: "#/components/parameters/language_param"
  - $ref: "#/components/parameters/use_trigram_param"
+ - $ref: "#/components/parameters/use_websearch_param"
  - $ref: "#/components/parameters/trigram_threshold_param"
  - $ref: "#/components/parameters/order_units_by_num_services_param"
  - $ref: "#/components/parameters/geometry_param"

services/search/tests/conftest.py

@@ -12,14 +12,18 @@
 )
 from rest_framework.test import APIClient
 
-from services.management.commands.index_search_columns import get_search_column
+from services.management.commands.index_search_columns import (
+ generate_syllables,
+ get_search_column,
+)
 from services.management.commands.services_import.services import (
  update_service_counts,
  update_service_node_counts,
  update_service_root_service_nodes,
 )
 from services.models import (
  Department,
+ ExclusionRule,
  Service,
  ServiceNode,
  Unit,
@@ -80,9 +84,32 @@ def units(
  )
  unit.services.add(3)
  unit.save()
+ # id=4 is the "Tekonurmikentät" service
+ service = Service.objects.get(id=4)
+ unit = Unit.objects.create(
+ id=4,
+ name="Kupittaan tekonurmikentät",
+ service_names_fi=[service.name_fi],
+ last_modified_time=now(),
+ municipality=municipality,
+ )
+ unit.services.add(4)
+ unit.save()
+ # id=5 is the "tekojääradat" service
+ service = Service.objects.get(id=5)
+ unit = Unit.objects.create(
+ id=5,
+ name="Parkin kenttä",
+ service_names_fi=[service.name_fi],
+ last_modified_time=now(),
+ municipality=municipality,
+ )
+ unit.services.add(5)
+ unit.save()
  update_service_root_service_nodes()
  update_service_counts()
  update_service_node_counts()
+ generate_syllables(Unit)
  Unit.objects.update(search_column_fi=get_search_column(Unit, "fi"))
  return Unit.objects.all()
 
@@ -101,8 +128,9 @@ def department(municipality):
 @pytest.mark.django_db
 @pytest.fixture
 def accessibility_shortcoming(units):
+ unit = Unit.objects.get(name="Biologinen museo")
  return UnitAccessibilityShortcomings.objects.create(
- unit=units[1], accessibility_shortcoming_count={"rollator": 5, "stroller": 1}
+ unit=unit, accessibility_shortcoming_count={"rollator": 5, "stroller": 1}
  )
 
 
@@ -127,6 +155,19 @@ def services():
  name_sv="Simhall",
  last_modified_time=now(),
  )
+ Service.objects.create(
+ id=4,
+ name="Tekonurmikentät",
+ name_sv="Konstgräsplaner",
+ last_modified_time=now(),
+ )
+ Service.objects.create(
+ id=5,
+ name="tekojääkentät",
+ name_sv="konstisbanor",
+ last_modified_time=now(),
+ )
+ generate_syllables(Service)
  Service.objects.update(search_column_fi=get_search_column(Service, "fi"))
  return Service.objects.all()
 
@@ -244,3 +285,10 @@ def streets():
  Street.objects.create(id=43, name="Markulantie", municipality_id="turku")
  Street.objects.create(id=44, name="Yliopistonkatu", municipality_id="turku")
  return Street.objects.all()
+
+
+@pytest.mark.django_db
+@pytest.fixture
+def exclusion_rules():
+ ExclusionRule.objects.create(id=1, word="tekojää", exclusion="-nurmi")
+ return ExclusionRule.objects.all()

services/search/tests/test_api.py

@@ -13,6 +13,7 @@ def test_search(
  administrative_division,
  accessibility_shortcoming,
  municipality,
+ exclusion_rules,
 ):
  # Search for "museo" in entities: units,services and servicenods
  url = reverse("search") + "?q=museo&type=unit,service,servicenode"
@@ -30,7 +31,6 @@ def test_search(
  assert biological_museum_unit["street_address"] == "Neitsytpolku 1"
  assert biological_museum_unit["municipality"] == "turku"
  assert biological_museum_unit["contract_type"]["id"] == "municipal_service"
-
  assert (
  biological_museum_unit["contract_type"]["description"]["fi"]
  == "municipal_service"
@@ -133,3 +133,23 @@ def test_search(
  results = response.json()["results"]
  assert results[0]["object_type"] == "administrativedivision"
  assert results[0]["name"]["fi"] == "Turku"
+
+ # Test exclusion rules used with websearch. By default (use_websearch=True) should only find Parkin kenttä
+ url = reverse("search") + "?q=tekojää&type=unit,service,servicenode"
+ response = api_client.get(url)
+ results = response.json()["results"]
+ assert len(results) == 2
+ parkin_kentta = results[0]
+ assert parkin_kentta["object_type"] == "unit"
+ assert parkin_kentta["name"]["fi"] == "Parkin kenttä"
+ tekojaa_service = results[1]
+ assert tekojaa_service["object_type"] == "service"
+ assert tekojaa_service["name"]["fi"] == "tekojääkentät"
+ # Disabling use_websearch, should return both 'tekojääkentät', 'tekonurmikentät' services and their units.
+ # as syllable 'teko' is indexed from the compound words.
+ url = (
+ reverse("search")
+ + "?q=tekojää&type=unit,service,servicenode&use_websearch=false"
+ )
+ response = api_client.get(url)
+ assert len(response.json()["results"]) == 4