From 1826777eab4cce111b26c41265b5cabac80c7b15 Mon Sep 17 00:00:00 2001 From: Steven Bal Date: Fri, 7 Jun 2024 16:31:34 +0200 Subject: [PATCH] :sparkles: [open-zaak/open-zaak#1649] Command to document envvars --- open_api_framework/conf/base.py | 298 +++++++++++++++--- open_api_framework/conf/utils.py | 44 ++- open_api_framework/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/generate_envvar_docs.py | 27 ++ .../open_api_framework/env_config.rst | 18 ++ open_api_framework/templatetags/doc_tags.py | 35 ++ 7 files changed, 374 insertions(+), 48 deletions(-) create mode 100644 open_api_framework/management/__init__.py create mode 100644 open_api_framework/management/commands/__init__.py create mode 100644 open_api_framework/management/commands/generate_envvar_docs.py create mode 100644 open_api_framework/templates/open_api_framework/env_config.rst create mode 100644 open_api_framework/templatetags/doc_tags.py diff --git a/open_api_framework/conf/base.py b/open_api_framework/conf/base.py index 85df04a..aeb1fdb 100644 --- a/open_api_framework/conf/base.py +++ b/open_api_framework/conf/base.py @@ -30,19 +30,56 @@ # # Core Django settings # -SITE_ID = config("SITE_ID", default=1) +SITE_ID = config( + "SITE_ID", + default=1, + help_text="The database ID of the site object. You usually won't have to touch this.", +) # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = config("SECRET_KEY") +SECRET_KEY = config( + "SECRET_KEY", + help_text=( + "Secret key that's used for certain cryptographic utilities. " + "You should generate one via `miniwebtool `_" + ), +) # NEVER run with DEBUG=True in production-like environments -DEBUG = config("DEBUG", default=False) +DEBUG = config( + "DEBUG", + default=False, + help_text=( + "Only set this to True on a local development environment. " + "Various other security settings are derived from this setting!" + ), +) # = domains we're running on -ALLOWED_HOSTS = config("ALLOWED_HOSTS", default="", split=True) -USE_X_FORWARDED_HOST = config("USE_X_FORWARDED_HOST", default=False) +ALLOWED_HOSTS = config( + "ALLOWED_HOSTS", + default="", + split=True, + help_text=( + "a comma separated (without spaces!) list of domains that serve " + "the installation. Used to protect against Host header attacks." + ), +) +USE_X_FORWARDED_HOST = config( + "USE_X_FORWARDED_HOST", + default=False, + help_text=( + "whether to grab the domain/host from the X-Forwarded-Host header or not. " + "This header is typically set by reverse proxies (such as nginx, traefik, Apache...). " + "Note: this is a header that can be spoofed and you need to ensure you control it before enabling this." + ), +) -IS_HTTPS = config("IS_HTTPS", default=not DEBUG) +IS_HTTPS = config( + "IS_HTTPS", + default=not DEBUG, + help_text="Used to construct absolute URLs and controls a variety of security settings", +) # Internationalization # https://docs.djangoproject.com/en/2.0/topics/i18n/ @@ -66,11 +103,33 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", - "NAME": config("DB_NAME", PROJECT_DIRNAME), - "USER": config("DB_USER", PROJECT_DIRNAME), - "PASSWORD": config("DB_PASSWORD", PROJECT_DIRNAME), - "HOST": config("DB_HOST", "localhost"), - "PORT": config("DB_PORT", 5432), + "NAME": config( + "DB_NAME", + PROJECT_DIRNAME, + group="Database", + help_text="name of the PostgreSQL database.", + ), + "USER": config( + "DB_USER", + PROJECT_DIRNAME, + group="Database", + help_text="username of the database user.", + ), + "PASSWORD": config( + "DB_PASSWORD", + PROJECT_DIRNAME, + group="Database", + help_text="password of the database user.", + ), + "HOST": config( + "DB_HOST", + "localhost", + group="Database", + help_text="hostname of the PostgreSQL database", + ), + "PORT": config( + "DB_PORT", 5432, group="Database", help_text="port number of the database" + ), } } @@ -78,10 +137,21 @@ # https://docs.djangoproject.com/en/4.0/ref/settings/#std:setting-DEFAULT_AUTO_FIELD DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +CACHE_DEFAULT = config( + "CACHE_DEFAULT", + "localhost:6379/0", + help_text="redis cache address for the default cache", +) +CACHE_AXES = config( + "CACHE_AXES", + "localhost:6379/0", + help_text="redis cache address for the brute force login protection cache", +) + CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": f"redis://{config('CACHE_DEFAULT', 'localhost:6379/0')}", + "LOCATION": f"redis://{CACHE_DEFAULT}", "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", "IGNORE_EXCEPTIONS": True, @@ -89,7 +159,7 @@ }, "axes": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": f"redis://{config('CACHE_AXES', 'localhost:6379/0')}", + "LOCATION": f"redis://{CACHE_AXES}", "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", "IGNORE_EXCEPTIONS": True, @@ -97,7 +167,7 @@ }, "oidc": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": f"redis://{config('CACHE_DEFAULT', 'localhost:6379/0')}", + "LOCATION": f"redis://{CACHE_DEFAULT}", "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", "IGNORE_EXCEPTIONS": True, @@ -147,7 +217,10 @@ "log_outgoing_requests", "django_setup_configuration", "open_api_framework", +<<<<<<< HEAD PROJECT_DIRNAME, +======= +>>>>>>> :sparkles: [open-zaak/open-zaak#1649] Command to document envvars ] MIDDLEWARE = [ @@ -223,24 +296,65 @@ # # Sending EMAIL # -EMAIL_HOST = config("EMAIL_HOST", default="localhost") +EMAIL_HOST = config( + "EMAIL_HOST", + default="localhost", + help_text="hostname for the outgoing e-mail server", +) EMAIL_PORT = config( - "EMAIL_PORT", default=25 + "EMAIL_PORT", + default=25, + help_text=( + "port number of the outgoing e-mail server. Note that if you're on Google Cloud, " + "sending e-mail via port 25 is completely blocked and you should use 487 for TLS." + ), ) # disabled on Google Cloud, use 487 instead -EMAIL_HOST_USER = config("EMAIL_HOST_USER", default="") -EMAIL_HOST_PASSWORD = config("EMAIL_HOST_PASSWORD", default="") -EMAIL_USE_TLS = config("EMAIL_USE_TLS", default=False) +EMAIL_HOST_USER = config( + "EMAIL_HOST_USER", default="", help_text="username to connect to the mail server" +) +EMAIL_HOST_PASSWORD = config( + "EMAIL_HOST_PASSWORD", + default="", + help_text="password to connect to the mail server", +) +EMAIL_USE_TLS = config( + "EMAIL_USE_TLS", + default=False, + help_text=( + "whether to use TLS or not to connect to the mail server. " + "Should be True if you're changing the EMAIL_PORT to 487." + ), +) EMAIL_TIMEOUT = 10 -DEFAULT_FROM_EMAIL = config("DEFAULT_FROM_EMAIL", f"{PROJECT_DIRNAME}@example.com") +DEFAULT_FROM_EMAIL = config( + "DEFAULT_FROM_EMAIL", + f"{PROJECT_DIRNAME}@example.com", + help_text="The default email address from which emails are sent", +) # # LOGGING # -LOG_STDOUT = config("LOG_STDOUT", default=False) -LOG_LEVEL = config("LOG_LEVEL", default="WARNING") -LOG_QUERIES = config("LOG_QUERIES", default=False) -LOG_REQUESTS = config("LOG_REQUESTS", default=False) +LOG_STDOUT = config( + "LOG_STDOUT", default=False, help_text="whether to log to stdout or not" +) +LOG_LEVEL = config( + "LOG_LEVEL", + default="WARNING", + help_text="control the verbosity of logging output. Available values are CRITICAL, ERROR, WARNING, INFO and DEBUG", +) +LOG_QUERIES = config( + "LOG_QUERIES", + default=False, + help_text=( + "enable (query) logging at the database backend level. Note that you " + "must also set DEBUG=1, which should be done very sparingly!" + ), +) +LOG_REQUESTS = config( + "LOG_REQUESTS", default=False, help_text="enable logging of the outgoing requests" +) if LOG_QUERIES and not DEBUG: warnings.warn( "Requested LOG_QUERIES=1 but DEBUG is false, no query logs will be emited.", @@ -438,11 +552,21 @@ # # Custom settings # -ENVIRONMENT = config("ENVIRONMENT", "") +ENVIRONMENT = config( + "ENVIRONMENT", + "", + help_text=( + "An identifier for the environment, displayed in the admin depending on " + "the settings module used and included in the error monitoring (see SENTRY_DSN). " + "The default is set according to DJANGO_SETTINGS_MODULE." + ), +) ENVIRONMENT_SHOWN_IN_ADMIN = True # Generating the schema, depending on the component -subpath = config("SUBPATH", None) +subpath = config( + "SUBPATH", None, help_text="The subpath the application will be mounted on" +) if subpath: if not subpath.startswith("/"): subpath = f"/{subpath}" @@ -462,12 +586,21 @@ else: GIT_SHA = None -RELEASE = config("RELEASE", GIT_SHA) +RELEASE = config( + "RELEASE", + GIT_SHA, + help_text="The version number or commit hash of the application (this is also send to Sentry)", +) NUM_PROXIES = config( # TODO: this also is relevant for DRF settings if/when we have rate-limited endpoints "NUM_PROXIES", default=1, cast=lambda val: int(val) if val is not None else None, + help_text=( + "the number of reverse proxies in front of the application, as an integer. " + "This is used to determine the actual client IP adres. " + "On Kubernetes with an ingress you typically want to set this to 2." + ), ) ############################## @@ -516,10 +649,28 @@ # # DJANGO-CORS-MIDDLEWARE # -CORS_ALLOW_ALL_ORIGINS = config("CORS_ALLOW_ALL_ORIGINS", default=False) -CORS_ALLOWED_ORIGINS = config("CORS_ALLOWED_ORIGINS", split=True, default=[]) +CORS_ALLOW_ALL_ORIGINS = config( + "CORS_ALLOW_ALL_ORIGINS", + default=False, + group="Cross-Origin-Resource-Sharing", + help_text="allow cross-domain access from any client", +) +CORS_ALLOWED_ORIGINS = config( + "CORS_ALLOWED_ORIGINS", + split=True, + default=[], + group="Cross-Origin-Resource-Sharing", + help_text=( + "explicitly list the allowed origins for cross-domain requests. " + "Example: http://localhost:3000,https://some-app.gemeente.nl" + ), +) CORS_ALLOWED_ORIGIN_REGEXES = config( - "CORS_ALLOWED_ORIGIN_REGEXES", split=True, default=[] + "CORS_ALLOWED_ORIGIN_REGEXES", + split=True, + default=[], + group="Cross-Origin-Resource-Sharing", + help_text="same as ``CORS_ALLOWED_ORIGINS``, but supports regular expressions", ) # Authorization is included in default_cors_headers CORS_ALLOW_HEADERS = ( @@ -528,7 +679,17 @@ "accept-crs", "content-crs", ] - + config("CORS_EXTRA_ALLOW_HEADERS", split=True, default=[]) + + config( + "CORS_EXTRA_ALLOW_HEADERS", + split=True, + default=[], + group="Cross-Origin-Resource-Sharing", + help_text=( + "headers that are allowed to be sent as part of the cross-domain request. " + "By default, Authorization, Accept-Crs and Content-Crs are already included. " + "The value of this variable is added to these already included headers." + ), + ) ) CORS_EXPOSE_HEADERS = [ "content-crs", @@ -545,6 +706,7 @@ "CSRF_TRUSTED_ORIGINS", split=True, default=[strip_protocol_from_origin(origin) for origin in CORS_ALLOWED_ORIGINS], + help_text="A list of trusted origins for unsafe requests (e.g. POST)", ) # # DJANGO-PRIVATES -- safely serve files after authorization @@ -556,7 +718,11 @@ # # NOTIFICATIONS-API-COMMON # -NOTIFICATIONS_DISABLED = config("NOTIFICATIONS_DISABLED", default=False) +NOTIFICATIONS_DISABLED = config( + "NOTIFICATIONS_DISABLED", + default=False, + help_text="if this variable is set to true, yes or 1, the notification mechanism will be disabled.", +) # # SENTRY - error monitoring @@ -564,7 +730,14 @@ def init_sentry(before_send: Callable | None = None): - SENTRY_DSN = config("SENTRY_DSN", None) + SENTRY_DSN = config( + "SENTRY_DSN", + None, + help_text=( + "URL of the sentry project to send error reports to. Default empty, " + "i.e. -> no monitoring set up. Highly recommended to configure this." + ), + ) if SENTRY_DSN: SENTRY_CONFIG = { @@ -587,8 +760,18 @@ def init_sentry(before_send: Callable | None = None): # # CELERY # -CELERY_BROKER_URL = config("CELERY_RESULT_BACKEND", "redis://localhost:6379/1") -CELERY_RESULT_BACKEND = config("CELERY_RESULT_BACKEND", "redis://localhost:6379/1") +CELERY_BROKER_URL = config( + "CELERY_RESULT_BACKEND", + "redis://localhost:6379/1", + group="Celery", + help_text="the URL of the broker that will be used to actually send the notifications", +) +CELERY_RESULT_BACKEND = config( + "CELERY_RESULT_BACKEND", + "redis://localhost:6379/1", + group="Celery", + help_text="the backend where the results of tasks will be stored", +) # @@ -611,15 +794,36 @@ def init_sentry(before_send: Callable | None = None): # # Elastic APM # -ELASTIC_APM_SERVER_URL = config("ELASTIC_APM_SERVER_URL", None) +ELASTIC_APM_SERVER_URL = config( + "ELASTIC_APM_SERVER_URL", + None, + "URL where Elastic APM is hosted", + group="Elastic APM", +) ELASTIC_APM = { # FIXME this does change the default service name, because PROJECT_DIRNAME != PROJECT_NAME "SERVICE_NAME": config( - "ELASTIC_APM_SERVICE_NAME", f"{PROJECT_DIRNAME} - {ENVIRONMENT}" + "ELASTIC_APM_SERVICE_NAME", + f"{PROJECT_DIRNAME} - {ENVIRONMENT}", + help_text="Name of the service for this application in Elastic APM", + group="Elastic APM", + ), + "SECRET_TOKEN": config( + "ELASTIC_APM_SECRET_TOKEN", + "default", + "Token used to communicate with Elastic APM", + group="Elastic APM", ), - "SECRET_TOKEN": config("ELASTIC_APM_SECRET_TOKEN", "default"), "SERVER_URL": ELASTIC_APM_SERVER_URL, - "TRANSACTION_SAMPLE_RATE": config("ELASTIC_APM_TRANSACTION_SAMPLE_RATE", 0.1), + "TRANSACTION_SAMPLE_RATE": config( + "ELASTIC_APM_TRANSACTION_SAMPLE_RATE", + 0.1, + help_text=( + "By default, the agent will sample every transaction (e.g. request to your service). " + "To reduce overhead and storage requirements, set the sample rate to a value between 0.0 and 1.0" + ), + group="Elastic APM", + ), } if not ELASTIC_APM_SERVER_URL: ELASTIC_APM["ENABLED"] = False @@ -652,7 +856,11 @@ def init_sentry(before_send: Callable | None = None): # if DISABLE_2FA is true, fill the MAYKIN_2FA_ALLOW_MFA_BYPASS_BACKENDS with all # configured AUTHENTICATION_BACKENDS and thus disabeling the entire 2FA chain. -if config("DISABLE_2FA", default=False): # pragma: no cover +if config( + "DISABLE_2FA", + default=False, + help_text="Whether or not two factor authentication should be disabled", +): # pragma: no cover MAYKIN_2FA_ALLOW_MFA_BYPASS_BACKENDS = AUTHENTICATION_BACKENDS @@ -660,13 +868,15 @@ def init_sentry(before_send: Callable | None = None): # LOG OUTGOING REQUESTS # LOG_OUTGOING_REQUESTS_EMIT_BODY = config( - "LOG_OUTGOING_REQUESTS_EMIT_BODY", default=True + "LOG_OUTGOING_REQUESTS_EMIT_BODY", default=True, help_text="Whether or not outgoing request bodies should be logged" ) +LOG_OUTGOING_REQUESTS_DB_SAVE = config("LOG_OUTGOING_REQUESTS_DB_SAVE", default=False, help_text="Whether or not outgoing request logs should be saved to the database") LOG_OUTGOING_REQUESTS_DB_SAVE_BODY = config( - "LOG_OUTGOING_REQUESTS_DB_SAVE_BODY", default=True + "LOG_OUTGOING_REQUESTS_DB_SAVE_BODY", default=True, help_text="Whether or not outgoing request bodies should be saved to the database" ) -LOG_OUTGOING_REQUESTS_DB_SAVE = config("LOG_OUTGOING_REQUESTS_DB_SAVE", default=False) LOG_OUTGOING_REQUESTS_RESET_DB_SAVE_AFTER = None LOG_OUTGOING_REQUESTS_MAX_AGE = config( - "LOG_OUTGOING_REQUESTS_MAX_AGE", default=7 + "LOG_OUTGOING_REQUESTS_MAX_AGE", + default=7, + help_text="The amount of time after which request logs should be deleted from the database", ) # number of days diff --git a/open_api_framework/conf/utils.py b/open_api_framework/conf/utils.py index 167aa05..0f338e0 100644 --- a/open_api_framework/conf/utils.py +++ b/open_api_framework/conf/utils.py @@ -1,13 +1,42 @@ import sys +from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Optional from urllib.parse import urlparse -from decouple import Csv, config as _config, undefined +from decouple import Csv, Undefined, config as _config, undefined from sentry_sdk.integrations import DidNotEnable, django, redis -def config(option: str, default: Any = undefined, *args, **kwargs): +@dataclass +class EnvironmentVariable: + name: str + default: Any + help_text: str + group: Optional[str] = None + + def __post_init__(self): + if not self.group: + self.group = ( + "Required" if isinstance(self.default, Undefined) else "Optional" + ) + + def __eq__(self, other): + return isinstance(other, EnvironmentVariable) and self.name == other.name + + +ENVVAR_REGISTRY = [] + + +def config( + option: str, + default: Any = undefined, + help_text="", + group=None, + add_to_docs=True, + *args, + **kwargs, +): """ Pull a config parameter from the environment. @@ -17,6 +46,13 @@ def config(option: str, default: Any = undefined, *args, **kwargs): Pass ``split=True`` to split the comma-separated input into a list. """ + if add_to_docs: + variable = EnvironmentVariable( + name=option, default=default, help_text=help_text, group=group + ) + if variable not in ENVVAR_REGISTRY: + ENVVAR_REGISTRY.append(variable) + if "split" in kwargs: kwargs.pop("split") kwargs["cast"] = Csv() @@ -54,7 +90,7 @@ def strip_protocol_from_origin(origin: str) -> str: def get_project_dirname() -> str: - return config("DJANGO_SETTINGS_MODULE").split(".")[0] + return config("DJANGO_SETTINGS_MODULE", add_to_docs=False).split(".")[0] def get_django_project_dir() -> str: diff --git a/open_api_framework/management/__init__.py b/open_api_framework/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/open_api_framework/management/commands/__init__.py b/open_api_framework/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/open_api_framework/management/commands/generate_envvar_docs.py b/open_api_framework/management/commands/generate_envvar_docs.py new file mode 100644 index 0000000..8f3518a --- /dev/null +++ b/open_api_framework/management/commands/generate_envvar_docs.py @@ -0,0 +1,27 @@ +import warnings +from collections import defaultdict + +from django.core.management.base import BaseCommand +from django.template import loader + +from open_api_framework.conf.utils import EnvironmentVariable + + +def convert_group_to_rst(variables: set[EnvironmentVariable]) -> str: + template = loader.get_template("open_api_framework/env_config.rst") + grouped_vars = defaultdict(list) + for var in variables: + if not var.help_text: + warnings.warn(f"missing help_text for environment variable {var}") + grouped_vars[var.group].append(var) + return template.render({"vars": grouped_vars.items()}) + + +class Command(BaseCommand): + help = "Generate documentation for all used envvars" + + def handle(self, *args, **options): + from open_api_framework.conf.utils import ENVVAR_REGISTRY + + with open("docs/env_config.rst", "w") as f: + f.write(convert_group_to_rst(ENVVAR_REGISTRY)) diff --git a/open_api_framework/templates/open_api_framework/env_config.rst b/open_api_framework/templates/open_api_framework/env_config.rst new file mode 100644 index 0000000..6879756 --- /dev/null +++ b/open_api_framework/templates/open_api_framework/env_config.rst @@ -0,0 +1,18 @@ +{% load doc_tags %}.. _installation_env_config: + +=================================== +Environment configuration reference +=================================== + +{% block intro %}{% endblock %} + +Available environment variables +=============================== + +{% for group_name, vars in vars %} +{{group_name}} +{{group_name|repeat_char:"="}} + +{% for var in vars %}* ``{{var.name}}``: {% if var.help_text %}{{var.help_text|safe|ensure_endswith:"."}}{% endif %}{% if not var.default|is_undefined %} Defaults to: ``{{var.default|to_str}}``{% endif %} +{% endfor %} +{% endfor %} diff --git a/open_api_framework/templatetags/doc_tags.py b/open_api_framework/templatetags/doc_tags.py new file mode 100644 index 0000000..e1791f5 --- /dev/null +++ b/open_api_framework/templatetags/doc_tags.py @@ -0,0 +1,35 @@ +from django import template + +from decouple import Undefined + +register = template.Library() + + +@register.filter(name="repeat_char") +def repeat_char(value, char="-"): + try: + length = len(value) + return char * length + except TypeError: + return "" + + +@register.filter(name="is_undefined") +def is_undefined(value): + return isinstance(value, Undefined) + + +@register.filter(name="to_str") +def to_str(value): + if value == "": + return "(empty string)" + return str(value) + + +@register.filter(name="ensure_endswith") +def ensure_endswith(value, char): + if not isinstance(value, str): + value = str(value) + if not value.endswith(char): + value += char + return value