Merge pull request #297 from City-of-Turku/feature/update-project-set…

…tings

Feature/update project settings

config_dev.env.example

@@ -145,6 +145,23 @@ EMAIL_HOST_USER=example@example.com
 EMAIL_PORT=25
 EMAIL_USE_TLS=True
 
+# Django project log level, default INFO
+DJANGO_LOG_LEVEL=
+# Turku services importers log level, default DEBUG
+TURKU_SERVICES_IMPORT_LOG_LEVEL=
+# Search log level, default INFO
+SEARCH_LOG_LEVEL=
+# IoT APP, default INFO 
+IOT_LOG_LEVEL=
+# Eco counter, default INFO
+ECO_COUNTER_LOG_LEVEL=
+# Mobility data (includes importers), default INFO
+MOBILITY_DATA_LOG_LEVEL=
+# Bicycle networks APP, default INFO
+BICYCLE_NETWORK_LOG_LEVEL=
+# Street maintenance, default INFO
+STREET_MAINTENANCE_LOG_LEVEL=
+
 # Settings needed for enabling Turku area:
 #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv
 #TURKU_API_KEY=secret
@@ -184,4 +201,4 @@ YIT_TOKEN_URL=https://login.microsoftonline.com/86792d09-0d81-4899-8d66-95dfc96c
 KUNTEC_KEY=
 # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py)
 # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api
-TELRAAM_TOKEN=
+TELRAAM_TOKEN=

requirements.in

@@ -14,7 +14,7 @@ django-extensions
 psycopg2-binary<2.9
 django-mptt
 lxml>=4.9.1
-raven~=6.10.0
+sentry-sdk
 pip-tools
 python-dateutil
 pytest-django

requirements.txt

@@ -30,7 +30,9 @@ celery==5.2.3
     #   django-celery-beat
     #   django-celery-results
 certifi==2022.12.7
-    # via requests
+    # via
+    #   requests
+    #   sentry-sdk
 charset-normalizer==2.0.6
     # via requests
 click==8.0.3
@@ -201,8 +203,6 @@ pyyaml==5.3.1
     # via
     #   django-munigeo
     #   drf-spectacular
-raven==6.10.0
-    # via -r requirements.in
 redis==4.4.4
     # via -r requirements.in
 requests==2.31.0
@@ -215,6 +215,8 @@ requests-cache==0.8.1
     # via -r requirements.in
 requests-mock==1.9.3
     # via -r requirements.in
+sentry-sdk==1.9.0
+    # via -r requirements.in
 shapely==1.8.0
     # via -r requirements.in
 six==1.16.0
@@ -233,7 +235,9 @@ toml==0.10.2
     #   pytest
     #   pytest-cov
 tomli==1.2.1
-    # via pep517
+    # via
+    #   black
+    #   pep517
 tqdm==4.62.3
     # via -r requirements.in
 tzdata==2022.1
@@ -248,6 +252,7 @@ urllib3==1.26.7
     # via
     #   requests
     #   requests-cache
+    #   sentry-sdk
 vine==5.0.0
     # via
     #   amqp

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="smbackend",
-    version="210929",
+    version="230717",
     license="AGPLv3",
     packages=find_packages(),
     include_package_data=True,

smbackend/settings.py

@@ -1,29 +1,36 @@
+import logging.config
 import os
+from pathlib import Path
 
 import environ
+import sentry_sdk
 from django.conf.global_settings import LANGUAGES as GLOBAL_LANGUAGES
 from django.core.exceptions import ImproperlyConfigured
+from sentry_sdk.integrations.django import DjangoIntegration
 
 CONFIG_FILE_NAME = "config_dev.env"
+GDAL_LIBRARY_PATH = os.environ.get("GDAL_LIBRARY_PATH")
+GEOS_LIBRARY_PATH = os.environ.get("GEOS_LIBRARY_PATH")
 
-
-root = environ.Path(__file__) - 2  # two levels back in hierarchy
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = str(Path(__file__).resolve().parent.parent)
 env = environ.Env(
     DEBUG=(bool, False),
     LANGUAGES=(list, ["fi", "sv", "en"]),
     DATABASE_URL=(str, "postgis:///servicemap"),
+    SECRET_KEY=(str, "temp_key"),
     TRUST_X_FORWARDED_HOST=(bool, False),
     SECURE_PROXY_SSL_HEADER=(tuple, None),
     ALLOWED_HOSTS=(list, []),
-    SENTRY_DSN=(str, None),
-    SENTRY_ENVIRONMENT=(str, "development"),
+    SENTRY_DSN=(str, ""),
+    SENTRY_ENVIRONMENT=(str, ""),
     COOKIE_PREFIX=(str, "servicemap"),
     INTERNAL_IPS=(list, []),
     CELERY_BROKER_URL=(str, "amqp://guest:guest@localhost:5672"),
-    MEDIA_ROOT=(environ.Path(), root("media")),
-    STATIC_ROOT=(environ.Path(), root("static")),
-    MEDIA_URL=(str, "/media/"),
+    STATIC_ROOT=(str, BASE_DIR + "/static"),
+    MEDIA_ROOT=(str, BASE_DIR + "/media"),
     STATIC_URL=(str, "/static/"),
+    MEDIA_URL=(str, "/media/"),
     OPEN311_URL_BASE=(str, None),
     OPEN311_API_KEY=(str, None),
     OPEN311_INTERNAL_API_KEY=(str, None),
@@ -63,10 +70,16 @@
     EMAIL_PORT=(int, None),
     EMAIL_USE_TLS=(bool, None),
     TELRAAM_TOKEN=(str, None),
+    DJANGO_LOG_LEVEL=(str, "INFO"),
+    TURKU_SERVICES_IMPORT_LOG_LEVEL=(str, "INFO"),
+    SEARCH_LOG_LEVEL=(str, "INFO"),
+    IOT_LOG_LEVEL=(str, "INFO"),
+    ECO_COUNTER_LOG_LEVEL=(str, "INFO"),
+    MOBILITY_DATA_LOG_LEVEL=(str, "INFO"),
+    BICYCLE_NETWORK_LOG_LEVEL=(str, "INFO"),
+    STREET_MAINTENANCE_LOG_LEVEL=(str, "INFO"),
 )
 
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = root()
 
 # Django environ has a nasty habit of complanining at level
 # WARN about env file not being preset. Here we pre-empt it.
@@ -77,8 +90,17 @@
     environ.Env.read_env(env_file_path)
 
 DEBUG = env("DEBUG")
+SECRET_KEY = env("SECRET_KEY")
 TEMPLATE_DEBUG = False
 ALLOWED_HOSTS = env("ALLOWED_HOSTS")
+DJANGO_LOG_LEVEL = env("DJANGO_LOG_LEVEL")
+TURKU_SERVICES_IMPORT_LOG_LEVEL = env("TURKU_SERVICES_IMPORT_LOG_LEVEL")
+SEARCH_LOG_LEVEL = env("SEARCH_LOG_LEVEL")
+IOT_LOG_LEVEL = env("IOT_LOG_LEVEL")
+ECO_COUNTER_LOG_LEVEL = env("ECO_COUNTER_LOG_LEVEL")
+MOBILITY_DATA_LOG_LEVEL = env("MOBILITY_DATA_LOG_LEVEL")
+BICYCLE_NETWORK_LOG_LEVEL = env("BICYCLE_NETWORK_LOG_LEVEL")
+STREET_MAINTENANCE_LOG_LEVEL = env("STREET_MAINTENANCE_LOG_LEVEL")
 
 # Application definition
 INSTALLED_APPS = [
@@ -90,7 +112,6 @@
     "django.contrib.staticfiles",
     "django.contrib.gis",
     "django.contrib.postgres",
-    "raven.contrib.django.raven_compat",
     "rest_framework.authtoken",
     "rest_framework",
     "corsheaders",
@@ -161,7 +182,6 @@ def gettext(s):
 
 TIME_ZONE = "Europe/Helsinki"
 USE_I18N = True
-USE_L10N = True
 USE_TZ = True
 
 USE_X_FORWARDED_HOST = env("TRUST_X_FORWARDED_HOST")
@@ -292,16 +312,26 @@ def gettext(s):
         "blackhole": {"class": "logging.NullHandler"},
     },
     "loggers": {
-        "django": {"handlers": ["console"], "level": "INFO"},
-        "turku_services_import": {"handlers": ["console"], "level": "DEBUG"},
-        "search": {"handlers": ["console"], "level": "DEBUG"},
-        "iot": {"handlers": ["console"], "level": "INFO"},
-        "eco_counter": {"handlers": ["console"], "level": "INFO"},
-        "mobility_data": {"handlers": ["console"], "level": "INFO"},
-        "bicycle_network": {"handlers": ["console"], "level": "INFO"},
-        "street_maintenance": {"handlers": ["console"], "level": "INFO"},
+        "django": {"handlers": ["console"], "level": DJANGO_LOG_LEVEL},
+        "turku_services_import": {
+            "handlers": ["console"],
+            "level": TURKU_SERVICES_IMPORT_LOG_LEVEL,
+        },
+        "search": {"handlers": ["console"], "level": SEARCH_LOG_LEVEL},
+        "iot": {"handlers": ["console"], "level": IOT_LOG_LEVEL},
+        "eco_counter": {"handlers": ["console"], "level": ECO_COUNTER_LOG_LEVEL},
+        "mobility_data": {"handlers": ["console"], "level": MOBILITY_DATA_LOG_LEVEL},
+        "bicycle_network": {
+            "handlers": ["console"],
+            "level": BICYCLE_NETWORK_LOG_LEVEL,
+        },
+        "street_maintenance": {
+            "handlers": ["console"],
+            "level": STREET_MAINTENANCE_LOG_LEVEL,
+        },
     },
 }
+logging.config.dictConfig(LOGGING)
 
 # Define the endpoints for API documentation with drf-spectacular.
 DOC_ENDPOINTS = [
@@ -334,11 +364,6 @@ def preprocessing_filter_spec(endpoints):
 
 LOCALE_PATHS = (os.path.join(BASE_DIR, "locale"),)
 
-SENTRY_DSN = env("SENTRY_DSN")
-SENTRY_ENVIRONMENT = env("SENTRY_ENVIRONMENT")
-
-import raven  # noqa
-
 # Celery
 CELERY_BROKER_URL = env("CELERY_BROKER_URL")
 CELERY_RESULT_BACKEND = "django-db"
@@ -366,47 +391,16 @@ def preprocessing_filter_spec(endpoints):
     }
 }
 
-
-if SENTRY_DSN:
-    RAVEN_CONFIG = {
-        "dsn": SENTRY_DSN,
-        # Needs to change if settings.py is not in an immediate child of the project
-        "release": raven.fetch_git_sha(os.path.dirname(os.pardir)),
-        "environment": SENTRY_ENVIRONMENT,
-    }
-
+sentry_sdk.init(
+    dsn=env.str("SENTRY_DSN"),
+    environment=env.str("SENTRY_ENVIRONMENT"),
+    traces_sample_rate=1.0,
+    send_default_pii=True,
+    integrations=[DjangoIntegration()],
+)
 
 COOKIE_PREFIX = env("COOKIE_PREFIX")
 INTERNAL_IPS = env("INTERNAL_IPS")
-
-if "SECRET_KEY" not in locals():
-    secret_file = os.path.join(BASE_DIR, ".django_secret")
-    try:
-        SECRET_KEY = open(secret_file).read().strip()
-    except IOError:
-        import random
-
-        system_random = random.SystemRandom()
-        try:
-            SECRET_KEY = "".join(
-                [
-                    system_random.choice(
-                        "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)"
-                    )
-                    for i in range(64)
-                ]
-            )
-            secret = open(secret_file, "w")
-            import os
-
-            os.chmod(secret_file, 0o0600)
-            secret.write(SECRET_KEY)
-            secret.close()
-        except IOError:
-            Exception(
-                "Please create a %s file with random characters to generate your secret key!"
-                % secret_file
-            )
 TURKU_WFS_URL = env("TURKU_WFS_URL")
 PTV_ID_OFFSET = env("PTV_ID_OFFSET")
 GEO_SEARCH_LOCATION = env("GEO_SEARCH_LOCATION")