From f5bf3ac8e7bb8cbb49c56711ef730ed6e6dc6f99 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:14:11 +0300 Subject: [PATCH 01/19] Add info of importing exclude rules --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e86ba9c9c..91f269449 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,12 @@ For Turku specific imports see smbackend_turku/README.md. ./manage.py geo_import helsinki --divisions ./manage.py index_search_columns ``` - +Import exclude rules fixtures used by the search: +``` +./manage.py loaddata services/fixtures/exclusion_rules.json +``` 7. Redis + Redis is used for caching and as a message broker for Celery. Install Redis. Ubuntu: `sudo apt-get install redis-server` @@ -198,3 +202,6 @@ psql template1 -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' Mobility platform ----------------- The mobility data platform of the service map is being developed as part of European Union Horizon 2020 programme funded SCALE-UP project (grant agreement no. 955332). + +For more information see: mobility_data/README.mk + From 12fa6fd2b19dde0419021820f60e6e19022bc24a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:15:22 +0300 Subject: [PATCH 02/19] Add initial exclusion rules --- services/fixtures/exclusion_rules.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 services/fixtures/exclusion_rules.json diff --git a/services/fixtures/exclusion_rules.json b/services/fixtures/exclusion_rules.json new file mode 100644 index 000000000..2880b98a8 --- /dev/null +++ b/services/fixtures/exclusion_rules.json @@ -0,0 +1,10 @@ +[ + { + "model": "services.exclusionrule", + "pk": 1, + "fields": { + "word": "tekojää", + "exclusion": "-nurmi" + } + } + ] \ No newline at end of file From 5fe61376406fcaa6ad17cdca25f0b5d262a8862f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:20:06 +0300 Subject: [PATCH 03/19] Add model ExclusionRule --- services/migrations/0095_exclusionrule.py | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 services/migrations/0095_exclusionrule.py diff --git a/services/migrations/0095_exclusionrule.py b/services/migrations/0095_exclusionrule.py new file mode 100644 index 000000000..0e6b61e4a --- /dev/null +++ b/services/migrations/0095_exclusionrule.py @@ -0,0 +1,37 @@ +# Generated by Django 4.1.7 on 2023-07-20 05:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("services", "0094_create_syllables_fi_columns"), + ] + + operations = [ + migrations.CreateModel( + name="ExclusionRule", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("word", models.CharField(max_length=100, verbose_name="Word")), + ( + "exclusion", + models.CharField(max_length=100, verbose_name="Exclusion"), + ), + ], + options={ + "verbose_name": "Exclusion rule", + "verbose_name_plural": "Exclusion rules", + "ordering": ["-id"], + }, + ), + ] From bf044acfe4e2b45ce18ada48de3202982dcbd29d Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:20:46 +0300 Subject: [PATCH 04/19] Import ExclusionRule --- services/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/models/__init__.py b/services/models/__init__.py index be9d191a0..57037a8e9 100644 --- a/services/models/__init__.py +++ b/services/models/__init__.py @@ -2,6 +2,7 @@ from .department import Department from .keyword import Keyword from .notification import Announcement, ErrorMessage +from .search_rule import ExclusionRule from .service import Service, UnitServiceDetails from .service_mapping import ServiceMapping from .service_node import ServiceNode From 848d52fb1cc8d23801581287d538bb4e2f2a6443 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:21:27 +0300 Subject: [PATCH 05/19] Add model ExclusionRule --- services/models/search_rule.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 services/models/search_rule.py diff --git a/services/models/search_rule.py b/services/models/search_rule.py new file mode 100644 index 000000000..78c9c32b9 --- /dev/null +++ b/services/models/search_rule.py @@ -0,0 +1,15 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class ExclusionRule(models.Model): + word = models.CharField(max_length=100, verbose_name=_("Word")) + exclusion = models.CharField(max_length=100, verbose_name=_("Exclusion")) + + class Meta: + ordering = ["-id"] + verbose_name = _("Exclusion rule") + verbose_name_plural = _("Exclusion rules") + + def __str__(self): + return "%s : %s" % (self.word, self.exclusion) From a40e0e70f5202519464d8945ab7bf375350c12a3 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:22:38 +0300 Subject: [PATCH 06/19] Add websearch and exclusion rules --- services/search/api.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/services/search/api.py b/services/search/api.py index d4c681290..10408ef68 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -54,6 +54,7 @@ from .utils import ( get_all_ids_from_sql_results, get_preserved_order, + get_search_exclusions, get_service_node_results, get_trigram_results, set_address_fields, @@ -212,6 +213,14 @@ def get(self, request): else: trigram_threshold = DEFAULT_TRIGRAM_THRESHOLD + if "use_websearch" in params: + try: + use_websearch = strtobool(params["use_websearch"]) + except ValueError: + raise ParseError("'use_websearch' needs to be a boolean") + else: + use_websearch = True + if "geometry" in params: try: show_geometry = strtobool(params["geometry"]) @@ -266,7 +275,7 @@ def get(self, request): config_language = LANGUAGES[language_short] search_query_str = None # Used in the raw sql # Build conditional query string that is used in the SQL query. - # split my "," or whitespace + # split by "," or whitespace q_vals = re.split(r",\s+|\s+", q_val) q_vals = [s.strip().replace("'", "") for s in q_vals] for q in q_vals: @@ -279,12 +288,17 @@ def get(self, request): search_query_str += f"& {q}:*" else: search_query_str = f"{q}:*" - + search_fn = "to_tsquery" + if use_websearch: + exclusions = get_search_exclusions(q) + if exclusions: + search_fn = "websearch_to_tsquery" + search_query_str += f" {exclusions}" # This is ~100 times faster than using Djangos SearchRank and allows searching using wildard "|*" # and by rankig gives better results, e.g. extra fields weight is counted. sql = f""" SELECT id, type_name, name_{language_short}, ts_rank_cd(search_column_{language_short}, search_query) - AS rank FROM search_view, to_tsquery('{config_language}','{search_query_str}') search_query + AS rank FROM search_view, {search_fn}('{config_language}','{search_query_str}') search_query WHERE search_query @@ search_column_{language_short} ORDER BY rank DESC LIMIT {sql_query_limit}; """ From bf98d3d5f2a3ee15c4c249b1db0479bfb4897d5b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:24:07 +0300 Subject: [PATCH 07/19] Add information about use_websearch param --- services/search/specification.swagger.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/search/specification.swagger.yaml b/services/search/specification.swagger.yaml index c59950cba..e02142fbf 100644 --- a/services/search/specification.swagger.yaml +++ b/services/search/specification.swagger.yaml @@ -34,6 +34,16 @@ components: type: string example: fi default: fi + use_websearch_param: + name: use_websearch + in: query + schema: + type: boolean + default: true + description: > + "websearch_to_tsquery is a simplified version of to_tsquery with an alternative syntax, similar to the one used by web search engines." + If disabled, uses 'to_tsquery' function to convert the query to 'tsquery'. + If enabled, exclusion rules are used when generating the query as it support the not "-" operator. order_units_by_num_services_param: name: order_units_by_num_services in: query @@ -173,6 +183,7 @@ paths: - $ref: "#/components/parameters/q_param" - $ref: "#/components/parameters/language_param" - $ref: "#/components/parameters/use_trigram_param" + - $ref: "#/components/parameters/use_websearch_param" - $ref: "#/components/parameters/trigram_threshold_param" - $ref: "#/components/parameters/order_units_by_num_services_param" - $ref: "#/components/parameters/geometry_param" From dcd3f38e45905e538d325b2b8a421ad0085ceb41 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:27:20 +0300 Subject: [PATCH 08/19] Add get_search_exclusions function --- services/search/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/search/utils.py b/services/search/utils.py index 472849b18..e8611b990 100644 --- a/services/search/utils.py +++ b/services/search/utils.py @@ -2,7 +2,7 @@ from django.db import connection from django.db.models import Case, When -from services.models import ServiceNode, ServiceNodeUnitCount, Unit +from services.models import ExclusionRule, ServiceNode, ServiceNodeUnitCount, Unit from services.search.constants import ( DEFAULT_TRIGRAM_THRESHOLD, LENGTH_OF_HYPHENATED_WORDS, @@ -194,3 +194,14 @@ def get_trigram_results( ids = [row[0] for row in all_results] objs = model.objects.filter(id__in=ids) return objs + + +def get_search_exclusions(q): + """ + To add/modify search exclusion rules edit: services/fixtures/exclusion_rules + To import rules: ./manage.py loaddata services/fixtures/exclusion_rules.json + """ + rule = ExclusionRule.objects.filter(word__iexact=q).first() + if rule: + return rule.exclusion + return "" From 94dd7a2f6ba86abacb1d101ed6b76718c1b23aa5 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:32:43 +0300 Subject: [PATCH 09/19] Add fixtures for testing exclusion rules --- services/search/tests/conftest.py | 52 +++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/services/search/tests/conftest.py b/services/search/tests/conftest.py index cd4e5820b..3d7bd4cde 100644 --- a/services/search/tests/conftest.py +++ b/services/search/tests/conftest.py @@ -12,7 +12,10 @@ ) from rest_framework.test import APIClient -from services.management.commands.index_search_columns import get_search_column +from services.management.commands.index_search_columns import ( + generate_syllables, + get_search_column, +) from services.management.commands.services_import.services import ( update_service_counts, update_service_node_counts, @@ -20,6 +23,7 @@ ) from services.models import ( Department, + ExclusionRule, Service, ServiceNode, Unit, @@ -80,9 +84,32 @@ def units( ) unit.services.add(3) unit.save() + # id=4 is the "Tekonurmikentät" service + service = Service.objects.get(id=4) + unit = Unit.objects.create( + id=4, + name="Kupittaan tekonurmikentät", + service_names_fi=[service.name_fi], + last_modified_time=now(), + municipality=municipality, + ) + unit.services.add(4) + unit.save() + # id=5 is the "tekojääradat" service + service = Service.objects.get(id=5) + unit = Unit.objects.create( + id=5, + name="Parkin kenttä", + service_names_fi=[service.name_fi], + last_modified_time=now(), + municipality=municipality, + ) + unit.services.add(5) + unit.save() update_service_root_service_nodes() update_service_counts() update_service_node_counts() + generate_syllables(Unit) Unit.objects.update(search_column_fi=get_search_column(Unit, "fi")) return Unit.objects.all() @@ -101,8 +128,9 @@ def department(municipality): @pytest.mark.django_db @pytest.fixture def accessibility_shortcoming(units): + unit = Unit.objects.get(name="Biologinen museo") return UnitAccessibilityShortcomings.objects.create( - unit=units[1], accessibility_shortcoming_count={"rollator": 5, "stroller": 1} + unit=unit, accessibility_shortcoming_count={"rollator": 5, "stroller": 1} ) @@ -127,6 +155,19 @@ def services(): name_sv="Simhall", last_modified_time=now(), ) + Service.objects.create( + id=4, + name="Tekonurmikentät", + name_sv="Konstgräsplaner", + last_modified_time=now(), + ) + Service.objects.create( + id=5, + name="tekojääkentät", + name_sv="konstisbanor", + last_modified_time=now(), + ) + generate_syllables(Service) Service.objects.update(search_column_fi=get_search_column(Service, "fi")) return Service.objects.all() @@ -244,3 +285,10 @@ def streets(): Street.objects.create(id=43, name="Markulantie", municipality_id="turku") Street.objects.create(id=44, name="Yliopistonkatu", municipality_id="turku") return Street.objects.all() + + +@pytest.mark.django_db +@pytest.fixture +def exclusion_rules(): + ExclusionRule.objects.create(id=1, word="tekojää", exclusion="-nurmi") + return ExclusionRule.objects.all() From be19bb7574fbcd0ae48982275e943be886b4a990 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Jul 2023 14:34:50 +0300 Subject: [PATCH 10/19] Add exclusion rules tests --- services/search/tests/test_api.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/services/search/tests/test_api.py b/services/search/tests/test_api.py index 2d78d4419..d79c0cbb2 100644 --- a/services/search/tests/test_api.py +++ b/services/search/tests/test_api.py @@ -13,6 +13,7 @@ def test_search( administrative_division, accessibility_shortcoming, municipality, + exclusion_rules, ): # Search for "museo" in entities: units,services and servicenods url = reverse("search") + "?q=museo&type=unit,service,servicenode" @@ -30,7 +31,6 @@ def test_search( assert biological_museum_unit["street_address"] == "Neitsytpolku 1" assert biological_museum_unit["municipality"] == "turku" assert biological_museum_unit["contract_type"]["id"] == "municipal_service" - assert ( biological_museum_unit["contract_type"]["description"]["fi"] == "municipal_service" @@ -133,3 +133,23 @@ def test_search( results = response.json()["results"] assert results[0]["object_type"] == "administrativedivision" assert results[0]["name"]["fi"] == "Turku" + + # Test exclusion rules used with websearch. By default (use_websearch=True) should only find Parkin kenttä + url = reverse("search") + "?q=tekojää&type=unit,service,servicenode" + response = api_client.get(url) + results = response.json()["results"] + assert len(results) == 2 + parkin_kentta = results[0] + assert parkin_kentta["object_type"] == "unit" + assert parkin_kentta["name"]["fi"] == "Parkin kenttä" + tekojaa_service = results[1] + assert tekojaa_service["object_type"] == "service" + assert tekojaa_service["name"]["fi"] == "tekojääkentät" + # Disabling use_websearch, should return both 'tekojääkentät', 'tekonurmikentät' services and their units. + # as syllable 'teko' is indexed from the compound words. + url = ( + reverse("search") + + "?q=tekojää&type=unit,service,servicenode&use_websearch=false" + ) + response = api_client.get(url) + assert len(response.json()["results"]) == 4 From d9c8859b46b09c5803428aeca61d5b895f1f488a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 11:55:31 +0300 Subject: [PATCH 11/19] Read SECREY_KEY from env --- smbackend/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index 91306fdb0..a9709de17 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -12,6 +12,7 @@ DEBUG=(bool, False), LANGUAGES=(list, ["fi", "sv", "en"]), DATABASE_URL=(str, "postgis:///servicemap"), + SECRET_KEY=(str, ""), TRUST_X_FORWARDED_HOST=(bool, False), SECURE_PROXY_SSL_HEADER=(tuple, None), ALLOWED_HOSTS=(list, []), @@ -77,6 +78,7 @@ environ.Env.read_env(env_file_path) DEBUG = env("DEBUG") +SECRET_KEY = env("SECRET_KEY") TEMPLATE_DEBUG = False ALLOWED_HOSTS = env("ALLOWED_HOSTS") @@ -379,6 +381,7 @@ def preprocessing_filter_spec(endpoints): COOKIE_PREFIX = env("COOKIE_PREFIX") INTERNAL_IPS = env("INTERNAL_IPS") +# NOTE, Helsinki has removed generation of the SECRET_KEY if "SECRET_KEY" not in locals(): secret_file = os.path.join(BASE_DIR, ".django_secret") try: @@ -403,10 +406,11 @@ def preprocessing_filter_spec(endpoints): secret.write(SECRET_KEY) secret.close() except IOError: - Exception( + raise Exception( "Please create a %s file with random characters to generate your secret key!" % secret_file ) + TURKU_WFS_URL = env("TURKU_WFS_URL") PTV_ID_OFFSET = env("PTV_ID_OFFSET") GEO_SEARCH_LOCATION = env("GEO_SEARCH_LOCATION") From 86fe4a5be19ed92b09c7d4bba8872133872b1bcd Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 12:47:55 +0300 Subject: [PATCH 12/19] Remove secret key generation --- smbackend/settings.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index a9709de17..e9fca554a 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -380,37 +380,6 @@ def preprocessing_filter_spec(endpoints): COOKIE_PREFIX = env("COOKIE_PREFIX") INTERNAL_IPS = env("INTERNAL_IPS") - -# NOTE, Helsinki has removed generation of the SECRET_KEY -if "SECRET_KEY" not in locals(): - secret_file = os.path.join(BASE_DIR, ".django_secret") - try: - SECRET_KEY = open(secret_file).read().strip() - except IOError: - import random - - system_random = random.SystemRandom() - try: - SECRET_KEY = "".join( - [ - system_random.choice( - "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)" - ) - for i in range(64) - ] - ) - secret = open(secret_file, "w") - import os - - os.chmod(secret_file, 0o0600) - secret.write(SECRET_KEY) - secret.close() - except IOError: - raise Exception( - "Please create a %s file with random characters to generate your secret key!" - % secret_file - ) - TURKU_WFS_URL = env("TURKU_WFS_URL") PTV_ID_OFFSET = env("PTV_ID_OFFSET") GEO_SEARCH_LOCATION = env("GEO_SEARCH_LOCATION") From 2299269294dfbee786123a9a96cf869f1e1b0411 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 13:43:18 +0300 Subject: [PATCH 13/19] Add logging examples --- config_dev.env.example | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/config_dev.env.example b/config_dev.env.example index 296a49ad6..dc66fbe57 100644 --- a/config_dev.env.example +++ b/config_dev.env.example @@ -145,6 +145,23 @@ EMAIL_HOST_USER=example@example.com EMAIL_PORT=25 EMAIL_USE_TLS=True +# Django project log level, default INFO +DJANGO_LOG_LEVEL= +# Turku services importers log level, default DEBUG +TURKU_SERVICES_IMPORT_LOG_LEVEL= +# Search log level, default INFO +SEARCH_LOG_LEVEL= +# IoT APP, default INFO +IOT_LOG_LEVEL= +# Eco counter, default INFO +ECO_COUNTER_LOG_LEVEL= +# Mobility data (includes importers), default INFO +MOBILITY_DATA_LOG_LEVEL= +# Bicycle networks APP, default INFO +BICYCLE_NETWORK_LOG_LEVEL= +# Street maintenance, default INFO +STREET_MAINTENANCE_LOG_LEVEL= + # Settings needed for enabling Turku area: #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv #TURKU_API_KEY=secret @@ -184,4 +201,4 @@ YIT_TOKEN_URL=https://login.microsoftonline.com/86792d09-0d81-4899-8d66-95dfc96c KUNTEC_KEY= # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py) # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api -TELRAAM_TOKEN= \ No newline at end of file +TELRAAM_TOKEN= From 3c1e5b7b5ca8b7d9607ff09c9651ce5245cd3243 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 13:44:14 +0300 Subject: [PATCH 14/19] Read logging from env, improve default values --- smbackend/settings.py | 56 ++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index e9fca554a..914bc1243 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import environ from django.conf.global_settings import LANGUAGES as GLOBAL_LANGUAGES @@ -6,13 +7,13 @@ CONFIG_FILE_NAME = "config_dev.env" - -root = environ.Path(__file__) - 2 # two levels back in hierarchy +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = str(Path(__file__).resolve().parent.parent) env = environ.Env( DEBUG=(bool, False), LANGUAGES=(list, ["fi", "sv", "en"]), DATABASE_URL=(str, "postgis:///servicemap"), - SECRET_KEY=(str, ""), + SECRET_KEY=(str, "temp_key"), TRUST_X_FORWARDED_HOST=(bool, False), SECURE_PROXY_SSL_HEADER=(tuple, None), ALLOWED_HOSTS=(list, []), @@ -21,10 +22,10 @@ COOKIE_PREFIX=(str, "servicemap"), INTERNAL_IPS=(list, []), CELERY_BROKER_URL=(str, "amqp://guest:guest@localhost:5672"), - MEDIA_ROOT=(environ.Path(), root("media")), - STATIC_ROOT=(environ.Path(), root("static")), - MEDIA_URL=(str, "/media/"), + STATIC_ROOT=(str, BASE_DIR + "/static"), + MEDIA_ROOT=(str, BASE_DIR + "/media"), STATIC_URL=(str, "/static/"), + MEDIA_URL=(str, "/media/"), OPEN311_URL_BASE=(str, None), OPEN311_API_KEY=(str, None), OPEN311_INTERNAL_API_KEY=(str, None), @@ -64,10 +65,16 @@ EMAIL_PORT=(int, None), EMAIL_USE_TLS=(bool, None), TELRAAM_TOKEN=(str, None), + DJANGO_LOG_LEVEL=(str, "INFO"), + TURKU_SERVICES_IMPORT_LOG_LEVEL=(str, "INFO"), + SEARCH_LOG_LEVEL=(str, "INFO"), + IOT_LOG_LEVEL=(str, "INFO"), + ECO_COUNTER_LOG_LEVEL=(str, "INFO"), + MOBILITY_DATA_LOG_LEVEL=(str, "INFO"), + BICYCLE_NETWORK_LOG_LEVEL=(str, "INFO"), + STREET_MAINTENANCE_LOG_LEVEL=(str, "INFO"), ) -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = root() # Django environ has a nasty habit of complanining at level # WARN about env file not being preset. Here we pre-empt it. @@ -81,6 +88,14 @@ SECRET_KEY = env("SECRET_KEY") TEMPLATE_DEBUG = False ALLOWED_HOSTS = env("ALLOWED_HOSTS") +DJANGO_LOG_LEVEL = env("DJANGO_LOG_LEVEL") +TURKU_SERVICES_IMPORT_LOG_LEVEL = env("TURKU_SERVICES_IMPORT_LOG_LEVEL") +SEARCH_LOG_LEVEL = env("SEARCH_LOG_LEVEL") +IOT_LOG_LEVEL = env("IOT_LOG_LEVEL") +ECO_COUNTER_LOG_LEVEL = env("ECO_COUNTER_LOG_LEVEL") +MOBILITY_DATA_LOG_LEVEL = env("MOBILITY_DATA_LOG_LEVEL") +BICYCLE_NETWORK_LOG_LEVEL = env("BICYCLE_NETWORK_LOG_LEVEL") +STREET_MAINTENANCE_LOG_LEVEL = env("STREET_MAINTENANCE_LOG_LEVEL") # Application definition INSTALLED_APPS = [ @@ -294,14 +309,23 @@ def gettext(s): "blackhole": {"class": "logging.NullHandler"}, }, "loggers": { - "django": {"handlers": ["console"], "level": "INFO"}, - "turku_services_import": {"handlers": ["console"], "level": "DEBUG"}, - "search": {"handlers": ["console"], "level": "DEBUG"}, - "iot": {"handlers": ["console"], "level": "INFO"}, - "eco_counter": {"handlers": ["console"], "level": "INFO"}, - "mobility_data": {"handlers": ["console"], "level": "INFO"}, - "bicycle_network": {"handlers": ["console"], "level": "INFO"}, - "street_maintenance": {"handlers": ["console"], "level": "INFO"}, + "django": {"handlers": ["console"], "level": DJANGO_LOG_LEVEL}, + "turku_services_import": { + "handlers": ["console"], + "level": TURKU_SERVICES_IMPORT_LOG_LEVEL, + }, + "search": {"handlers": ["console"], "level": SEARCH_LOG_LEVEL}, + "iot": {"handlers": ["console"], "level": IOT_LOG_LEVEL}, + "eco_counter": {"handlers": ["console"], "level": ECO_COUNTER_LOG_LEVEL}, + "mobility_data": {"handlers": ["console"], "level": MOBILITY_DATA_LOG_LEVEL}, + "bicycle_network": { + "handlers": ["console"], + "level": BICYCLE_NETWORK_LOG_LEVEL, + }, + "street_maintenance": { + "handlers": ["console"], + "level": STREET_MAINTENANCE_LOG_LEVEL, + }, }, } From 53fcf1bbbcdcd4adbb7ebbb63d843eb04b7e799b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 13:53:03 +0300 Subject: [PATCH 15/19] Remove deprecated USE_L10N-setting --- smbackend/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index 914bc1243..9693881b0 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -178,7 +178,6 @@ def gettext(s): TIME_ZONE = "Europe/Helsinki" USE_I18N = True -USE_L10N = True USE_TZ = True USE_X_FORWARDED_HOST = env("TRUST_X_FORWARDED_HOST") From 960fae70a5282dd6872443848bbeff07847fe2dc Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:12:38 +0300 Subject: [PATCH 16/19] Replace Raven with Sentry SDK --- requirements.in | 2 +- requirements.txt | 13 +++++++++---- smbackend/settings.py | 28 +++++++++++----------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/requirements.in b/requirements.in index 6cdaf4f58..d29f20c45 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,7 @@ django-extensions psycopg2-binary<2.9 django-mptt lxml>=4.9.1 -raven~=6.10.0 +sentry-sdk pip-tools python-dateutil pytest-django diff --git a/requirements.txt b/requirements.txt index 163484fce..94f3a0300 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,9 @@ celery==5.2.3 # django-celery-beat # django-celery-results certifi==2022.12.7 - # via requests + # via + # requests + # sentry-sdk charset-normalizer==2.0.6 # via requests click==8.0.3 @@ -201,8 +203,6 @@ pyyaml==5.3.1 # via # django-munigeo # drf-spectacular -raven==6.10.0 - # via -r requirements.in redis==4.4.4 # via -r requirements.in requests==2.26.0 @@ -215,6 +215,8 @@ requests-cache==0.8.1 # via -r requirements.in requests-mock==1.9.3 # via -r requirements.in +sentry-sdk==1.9.0 + # via -r requirements.in shapely==1.8.0 # via -r requirements.in six==1.16.0 @@ -233,7 +235,9 @@ toml==0.10.2 # pytest # pytest-cov tomli==1.2.1 - # via pep517 + # via + # black + # pep517 tqdm==4.62.3 # via -r requirements.in tzdata==2022.1 @@ -248,6 +252,7 @@ urllib3==1.26.7 # via # requests # requests-cache + # sentry-sdk vine==5.0.0 # via # amqp diff --git a/smbackend/settings.py b/smbackend/settings.py index 9693881b0..83b4f4aeb 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -2,8 +2,10 @@ from pathlib import Path import environ +import sentry_sdk from django.conf.global_settings import LANGUAGES as GLOBAL_LANGUAGES from django.core.exceptions import ImproperlyConfigured +from sentry_sdk.integrations.django import DjangoIntegration CONFIG_FILE_NAME = "config_dev.env" @@ -17,8 +19,8 @@ TRUST_X_FORWARDED_HOST=(bool, False), SECURE_PROXY_SSL_HEADER=(tuple, None), ALLOWED_HOSTS=(list, []), - SENTRY_DSN=(str, None), - SENTRY_ENVIRONMENT=(str, "development"), + SENTRY_DSN=(str, ""), + SENTRY_ENVIRONMENT=(str, ""), COOKIE_PREFIX=(str, "servicemap"), INTERNAL_IPS=(list, []), CELERY_BROKER_URL=(str, "amqp://guest:guest@localhost:5672"), @@ -107,7 +109,6 @@ "django.contrib.staticfiles", "django.contrib.gis", "django.contrib.postgres", - "raven.contrib.django.raven_compat", "rest_framework.authtoken", "rest_framework", "corsheaders", @@ -359,11 +360,6 @@ def preprocessing_filter_spec(endpoints): LOCALE_PATHS = (os.path.join(BASE_DIR, "locale"),) -SENTRY_DSN = env("SENTRY_DSN") -SENTRY_ENVIRONMENT = env("SENTRY_ENVIRONMENT") - -import raven # noqa - # Celery CELERY_BROKER_URL = env("CELERY_BROKER_URL") CELERY_RESULT_BACKEND = "django-db" @@ -391,15 +387,13 @@ def preprocessing_filter_spec(endpoints): } } - -if SENTRY_DSN: - RAVEN_CONFIG = { - "dsn": SENTRY_DSN, - # Needs to change if settings.py is not in an immediate child of the project - "release": raven.fetch_git_sha(os.path.dirname(os.pardir)), - "environment": SENTRY_ENVIRONMENT, - } - +sentry_sdk.init( + dsn=env.str("SENTRY_DSN"), + environment=env.str("SENTRY_ENVIRONMENT"), + traces_sample_rate=1.0, + send_default_pii=True, + integrations=[DjangoIntegration()], +) COOKIE_PREFIX = env("COOKIE_PREFIX") INTERNAL_IPS = env("INTERNAL_IPS") From e667b8debb6b67b0d840f4ec81d5db912a6a8f38 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:17:17 +0300 Subject: [PATCH 17/19] Utilize GDAL and GEOS paths for Django --- smbackend/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 83b4f4aeb..dcaa13c07 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -8,6 +8,8 @@ from sentry_sdk.integrations.django import DjangoIntegration CONFIG_FILE_NAME = "config_dev.env" +GDAL_LIBRARY_PATH = os.environ.get("GDAL_LIBRARY_PATH") +GEOS_LIBRARY_PATH = os.environ.get("GEOS_LIBRARY_PATH") # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = str(Path(__file__).resolve().parent.parent) From fc176a349650bf5c698d4a47467f3d1573fe5c35 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:28:49 +0300 Subject: [PATCH 18/19] Update logging to work with dict config --- smbackend/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index dcaa13c07..1615fb9bd 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -1,3 +1,4 @@ +import logging.config import os from pathlib import Path @@ -330,6 +331,7 @@ def gettext(s): }, }, } +logging.config.dictConfig(LOGGING) # Define the endpoints for API documentation with drf-spectacular. DOC_ENDPOINTS = [ From 1f37f74215355c751829691ec38d1ad31846f45e Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:30:46 +0300 Subject: [PATCH 19/19] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff0dc934c..db52eb478 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="smbackend", - version="210929", + version="230717", license="AGPLv3", packages=find_packages(), include_package_data=True,