From d9c8859b46b09c5803428aeca61d5b895f1f488a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 11:55:31 +0300 Subject: [PATCH 1/9] Read SECREY_KEY from env --- smbackend/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index 91306fdb0..a9709de17 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -12,6 +12,7 @@ DEBUG=(bool, False), LANGUAGES=(list, ["fi", "sv", "en"]), DATABASE_URL=(str, "postgis:///servicemap"), + SECRET_KEY=(str, ""), TRUST_X_FORWARDED_HOST=(bool, False), SECURE_PROXY_SSL_HEADER=(tuple, None), ALLOWED_HOSTS=(list, []), @@ -77,6 +78,7 @@ environ.Env.read_env(env_file_path) DEBUG = env("DEBUG") +SECRET_KEY = env("SECRET_KEY") TEMPLATE_DEBUG = False ALLOWED_HOSTS = env("ALLOWED_HOSTS") @@ -379,6 +381,7 @@ def preprocessing_filter_spec(endpoints): COOKIE_PREFIX = env("COOKIE_PREFIX") INTERNAL_IPS = env("INTERNAL_IPS") +# NOTE, Helsinki has removed generation of the SECRET_KEY if "SECRET_KEY" not in locals(): secret_file = os.path.join(BASE_DIR, ".django_secret") try: @@ -403,10 +406,11 @@ def preprocessing_filter_spec(endpoints): secret.write(SECRET_KEY) secret.close() except IOError: - Exception( + raise Exception( "Please create a %s file with random characters to generate your secret key!" % secret_file ) + TURKU_WFS_URL = env("TURKU_WFS_URL") PTV_ID_OFFSET = env("PTV_ID_OFFSET") GEO_SEARCH_LOCATION = env("GEO_SEARCH_LOCATION") From 86fe4a5be19ed92b09c7d4bba8872133872b1bcd Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 12:47:55 +0300 Subject: [PATCH 2/9] Remove secret key generation --- smbackend/settings.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index a9709de17..e9fca554a 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -380,37 +380,6 @@ def preprocessing_filter_spec(endpoints): COOKIE_PREFIX = env("COOKIE_PREFIX") INTERNAL_IPS = env("INTERNAL_IPS") - -# NOTE, Helsinki has removed generation of the SECRET_KEY -if "SECRET_KEY" not in locals(): - secret_file = os.path.join(BASE_DIR, ".django_secret") - try: - SECRET_KEY = open(secret_file).read().strip() - except IOError: - import random - - system_random = random.SystemRandom() - try: - SECRET_KEY = "".join( - [ - system_random.choice( - "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)" - ) - for i in range(64) - ] - ) - secret = open(secret_file, "w") - import os - - os.chmod(secret_file, 0o0600) - secret.write(SECRET_KEY) - secret.close() - except IOError: - raise Exception( - "Please create a %s file with random characters to generate your secret key!" - % secret_file - ) - TURKU_WFS_URL = env("TURKU_WFS_URL") PTV_ID_OFFSET = env("PTV_ID_OFFSET") GEO_SEARCH_LOCATION = env("GEO_SEARCH_LOCATION") From 2299269294dfbee786123a9a96cf869f1e1b0411 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 13:43:18 +0300 Subject: [PATCH 3/9] Add logging examples --- config_dev.env.example | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/config_dev.env.example b/config_dev.env.example index 296a49ad6..dc66fbe57 100644 --- a/config_dev.env.example +++ b/config_dev.env.example @@ -145,6 +145,23 @@ EMAIL_HOST_USER=example@example.com EMAIL_PORT=25 EMAIL_USE_TLS=True +# Django project log level, default INFO +DJANGO_LOG_LEVEL= +# Turku services importers log level, default DEBUG +TURKU_SERVICES_IMPORT_LOG_LEVEL= +# Search log level, default INFO +SEARCH_LOG_LEVEL= +# IoT APP, default INFO +IOT_LOG_LEVEL= +# Eco counter, default INFO +ECO_COUNTER_LOG_LEVEL= +# Mobility data (includes importers), default INFO +MOBILITY_DATA_LOG_LEVEL= +# Bicycle networks APP, default INFO +BICYCLE_NETWORK_LOG_LEVEL= +# Street maintenance, default INFO +STREET_MAINTENANCE_LOG_LEVEL= + # Settings needed for enabling Turku area: #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv #TURKU_API_KEY=secret @@ -184,4 +201,4 @@ YIT_TOKEN_URL=https://login.microsoftonline.com/86792d09-0d81-4899-8d66-95dfc96c KUNTEC_KEY= # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py) # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api -TELRAAM_TOKEN= \ No newline at end of file +TELRAAM_TOKEN= From 3c1e5b7b5ca8b7d9607ff09c9651ce5245cd3243 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 13:44:14 +0300 Subject: [PATCH 4/9] Read logging from env, improve default values --- smbackend/settings.py | 56 ++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index e9fca554a..914bc1243 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import environ from django.conf.global_settings import LANGUAGES as GLOBAL_LANGUAGES @@ -6,13 +7,13 @@ CONFIG_FILE_NAME = "config_dev.env" - -root = environ.Path(__file__) - 2 # two levels back in hierarchy +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = str(Path(__file__).resolve().parent.parent) env = environ.Env( DEBUG=(bool, False), LANGUAGES=(list, ["fi", "sv", "en"]), DATABASE_URL=(str, "postgis:///servicemap"), - SECRET_KEY=(str, ""), + SECRET_KEY=(str, "temp_key"), TRUST_X_FORWARDED_HOST=(bool, False), SECURE_PROXY_SSL_HEADER=(tuple, None), ALLOWED_HOSTS=(list, []), @@ -21,10 +22,10 @@ COOKIE_PREFIX=(str, "servicemap"), INTERNAL_IPS=(list, []), CELERY_BROKER_URL=(str, "amqp://guest:guest@localhost:5672"), - MEDIA_ROOT=(environ.Path(), root("media")), - STATIC_ROOT=(environ.Path(), root("static")), - MEDIA_URL=(str, "/media/"), + STATIC_ROOT=(str, BASE_DIR + "/static"), + MEDIA_ROOT=(str, BASE_DIR + "/media"), STATIC_URL=(str, "/static/"), + MEDIA_URL=(str, "/media/"), OPEN311_URL_BASE=(str, None), OPEN311_API_KEY=(str, None), OPEN311_INTERNAL_API_KEY=(str, None), @@ -64,10 +65,16 @@ EMAIL_PORT=(int, None), EMAIL_USE_TLS=(bool, None), TELRAAM_TOKEN=(str, None), + DJANGO_LOG_LEVEL=(str, "INFO"), + TURKU_SERVICES_IMPORT_LOG_LEVEL=(str, "INFO"), + SEARCH_LOG_LEVEL=(str, "INFO"), + IOT_LOG_LEVEL=(str, "INFO"), + ECO_COUNTER_LOG_LEVEL=(str, "INFO"), + MOBILITY_DATA_LOG_LEVEL=(str, "INFO"), + BICYCLE_NETWORK_LOG_LEVEL=(str, "INFO"), + STREET_MAINTENANCE_LOG_LEVEL=(str, "INFO"), ) -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = root() # Django environ has a nasty habit of complanining at level # WARN about env file not being preset. Here we pre-empt it. @@ -81,6 +88,14 @@ SECRET_KEY = env("SECRET_KEY") TEMPLATE_DEBUG = False ALLOWED_HOSTS = env("ALLOWED_HOSTS") +DJANGO_LOG_LEVEL = env("DJANGO_LOG_LEVEL") +TURKU_SERVICES_IMPORT_LOG_LEVEL = env("TURKU_SERVICES_IMPORT_LOG_LEVEL") +SEARCH_LOG_LEVEL = env("SEARCH_LOG_LEVEL") +IOT_LOG_LEVEL = env("IOT_LOG_LEVEL") +ECO_COUNTER_LOG_LEVEL = env("ECO_COUNTER_LOG_LEVEL") +MOBILITY_DATA_LOG_LEVEL = env("MOBILITY_DATA_LOG_LEVEL") +BICYCLE_NETWORK_LOG_LEVEL = env("BICYCLE_NETWORK_LOG_LEVEL") +STREET_MAINTENANCE_LOG_LEVEL = env("STREET_MAINTENANCE_LOG_LEVEL") # Application definition INSTALLED_APPS = [ @@ -294,14 +309,23 @@ def gettext(s): "blackhole": {"class": "logging.NullHandler"}, }, "loggers": { - "django": {"handlers": ["console"], "level": "INFO"}, - "turku_services_import": {"handlers": ["console"], "level": "DEBUG"}, - "search": {"handlers": ["console"], "level": "DEBUG"}, - "iot": {"handlers": ["console"], "level": "INFO"}, - "eco_counter": {"handlers": ["console"], "level": "INFO"}, - "mobility_data": {"handlers": ["console"], "level": "INFO"}, - "bicycle_network": {"handlers": ["console"], "level": "INFO"}, - "street_maintenance": {"handlers": ["console"], "level": "INFO"}, + "django": {"handlers": ["console"], "level": DJANGO_LOG_LEVEL}, + "turku_services_import": { + "handlers": ["console"], + "level": TURKU_SERVICES_IMPORT_LOG_LEVEL, + }, + "search": {"handlers": ["console"], "level": SEARCH_LOG_LEVEL}, + "iot": {"handlers": ["console"], "level": IOT_LOG_LEVEL}, + "eco_counter": {"handlers": ["console"], "level": ECO_COUNTER_LOG_LEVEL}, + "mobility_data": {"handlers": ["console"], "level": MOBILITY_DATA_LOG_LEVEL}, + "bicycle_network": { + "handlers": ["console"], + "level": BICYCLE_NETWORK_LOG_LEVEL, + }, + "street_maintenance": { + "handlers": ["console"], + "level": STREET_MAINTENANCE_LOG_LEVEL, + }, }, } From 53fcf1bbbcdcd4adbb7ebbb63d843eb04b7e799b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 13:53:03 +0300 Subject: [PATCH 5/9] Remove deprecated USE_L10N-setting --- smbackend/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index 914bc1243..9693881b0 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -178,7 +178,6 @@ def gettext(s): TIME_ZONE = "Europe/Helsinki" USE_I18N = True -USE_L10N = True USE_TZ = True USE_X_FORWARDED_HOST = env("TRUST_X_FORWARDED_HOST") From 960fae70a5282dd6872443848bbeff07847fe2dc Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:12:38 +0300 Subject: [PATCH 6/9] Replace Raven with Sentry SDK --- requirements.in | 2 +- requirements.txt | 13 +++++++++---- smbackend/settings.py | 28 +++++++++++----------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/requirements.in b/requirements.in index 6cdaf4f58..d29f20c45 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,7 @@ django-extensions psycopg2-binary<2.9 django-mptt lxml>=4.9.1 -raven~=6.10.0 +sentry-sdk pip-tools python-dateutil pytest-django diff --git a/requirements.txt b/requirements.txt index 163484fce..94f3a0300 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,9 @@ celery==5.2.3 # django-celery-beat # django-celery-results certifi==2022.12.7 - # via requests + # via + # requests + # sentry-sdk charset-normalizer==2.0.6 # via requests click==8.0.3 @@ -201,8 +203,6 @@ pyyaml==5.3.1 # via # django-munigeo # drf-spectacular -raven==6.10.0 - # via -r requirements.in redis==4.4.4 # via -r requirements.in requests==2.26.0 @@ -215,6 +215,8 @@ requests-cache==0.8.1 # via -r requirements.in requests-mock==1.9.3 # via -r requirements.in +sentry-sdk==1.9.0 + # via -r requirements.in shapely==1.8.0 # via -r requirements.in six==1.16.0 @@ -233,7 +235,9 @@ toml==0.10.2 # pytest # pytest-cov tomli==1.2.1 - # via pep517 + # via + # black + # pep517 tqdm==4.62.3 # via -r requirements.in tzdata==2022.1 @@ -248,6 +252,7 @@ urllib3==1.26.7 # via # requests # requests-cache + # sentry-sdk vine==5.0.0 # via # amqp diff --git a/smbackend/settings.py b/smbackend/settings.py index 9693881b0..83b4f4aeb 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -2,8 +2,10 @@ from pathlib import Path import environ +import sentry_sdk from django.conf.global_settings import LANGUAGES as GLOBAL_LANGUAGES from django.core.exceptions import ImproperlyConfigured +from sentry_sdk.integrations.django import DjangoIntegration CONFIG_FILE_NAME = "config_dev.env" @@ -17,8 +19,8 @@ TRUST_X_FORWARDED_HOST=(bool, False), SECURE_PROXY_SSL_HEADER=(tuple, None), ALLOWED_HOSTS=(list, []), - SENTRY_DSN=(str, None), - SENTRY_ENVIRONMENT=(str, "development"), + SENTRY_DSN=(str, ""), + SENTRY_ENVIRONMENT=(str, ""), COOKIE_PREFIX=(str, "servicemap"), INTERNAL_IPS=(list, []), CELERY_BROKER_URL=(str, "amqp://guest:guest@localhost:5672"), @@ -107,7 +109,6 @@ "django.contrib.staticfiles", "django.contrib.gis", "django.contrib.postgres", - "raven.contrib.django.raven_compat", "rest_framework.authtoken", "rest_framework", "corsheaders", @@ -359,11 +360,6 @@ def preprocessing_filter_spec(endpoints): LOCALE_PATHS = (os.path.join(BASE_DIR, "locale"),) -SENTRY_DSN = env("SENTRY_DSN") -SENTRY_ENVIRONMENT = env("SENTRY_ENVIRONMENT") - -import raven # noqa - # Celery CELERY_BROKER_URL = env("CELERY_BROKER_URL") CELERY_RESULT_BACKEND = "django-db" @@ -391,15 +387,13 @@ def preprocessing_filter_spec(endpoints): } } - -if SENTRY_DSN: - RAVEN_CONFIG = { - "dsn": SENTRY_DSN, - # Needs to change if settings.py is not in an immediate child of the project - "release": raven.fetch_git_sha(os.path.dirname(os.pardir)), - "environment": SENTRY_ENVIRONMENT, - } - +sentry_sdk.init( + dsn=env.str("SENTRY_DSN"), + environment=env.str("SENTRY_ENVIRONMENT"), + traces_sample_rate=1.0, + send_default_pii=True, + integrations=[DjangoIntegration()], +) COOKIE_PREFIX = env("COOKIE_PREFIX") INTERNAL_IPS = env("INTERNAL_IPS") From e667b8debb6b67b0d840f4ec81d5db912a6a8f38 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:17:17 +0300 Subject: [PATCH 7/9] Utilize GDAL and GEOS paths for Django --- smbackend/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 83b4f4aeb..dcaa13c07 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -8,6 +8,8 @@ from sentry_sdk.integrations.django import DjangoIntegration CONFIG_FILE_NAME = "config_dev.env" +GDAL_LIBRARY_PATH = os.environ.get("GDAL_LIBRARY_PATH") +GEOS_LIBRARY_PATH = os.environ.get("GEOS_LIBRARY_PATH") # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = str(Path(__file__).resolve().parent.parent) From fc176a349650bf5c698d4a47467f3d1573fe5c35 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:28:49 +0300 Subject: [PATCH 8/9] Update logging to work with dict config --- smbackend/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index dcaa13c07..1615fb9bd 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -1,3 +1,4 @@ +import logging.config import os from pathlib import Path @@ -330,6 +331,7 @@ def gettext(s): }, }, } +logging.config.dictConfig(LOGGING) # Define the endpoints for API documentation with drf-spectacular. DOC_ENDPOINTS = [ From 1f37f74215355c751829691ec38d1ad31846f45e Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 21 Jul 2023 14:30:46 +0300 Subject: [PATCH 9/9] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff0dc934c..db52eb478 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="smbackend", - version="210929", + version="230717", license="AGPLv3", packages=find_packages(), include_package_data=True,