diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 00f5c3cb994..cccf4efbfd3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -14,6 +14,7 @@ services: - 9091:9000 environment: ALLOW_SIGNUP: "false" + LOG_LEVEL: "DEBUG" DB_ENGINE: sqlite # Optional: 'sqlite', 'postgres' # ===================================== diff --git a/docker/entry.sh b/docker/entry.sh index 976f4b8c511..98e11d447f2 100644 --- a/docker/entry.sh +++ b/docker/entry.sh @@ -40,10 +40,11 @@ init GUNICORN_PORT=${API_PORT:-9000} # Start API -hostip=`/sbin/ip route|awk '/default/ { print $3 }'` +HOST_IP=`/sbin/ip route|awk '/default/ { print $3 }'` + if [ "$WEB_GUNICORN" = 'true' ]; then echo "Starting Gunicorn" - exec gunicorn mealie.app:app -b 0.0.0.0:$GUNICORN_PORT --forwarded-allow-ips=$hostip -k uvicorn.workers.UvicornWorker -c /app/gunicorn_conf.py --preload + exec gunicorn mealie.app:app -b 0.0.0.0:$GUNICORN_PORT --forwarded-allow-ips=$HOST_IP -k uvicorn.workers.UvicornWorker -c /app/gunicorn_conf.py --preload else - exec uvicorn mealie.app:app --host 0.0.0.0 --forwarded-allow-ips=$hostip --port $GUNICORN_PORT + exec python /app/mealie/main.py fi diff --git a/docs/docs/documentation/getting-started/installation/backend-config.md b/docs/docs/documentation/getting-started/installation/backend-config.md index bcea9f41e0e..45c7f1949e2 100644 --- a/docs/docs/documentation/getting-started/installation/backend-config.md +++ b/docs/docs/documentation/getting-started/installation/backend-config.md @@ -15,6 +15,8 @@ | API_DOCS | True | Turns on/off access to the API documentation locally. | | TZ | UTC | Must be set to get correct date/time on the server | | ALLOW_SIGNUP\* | false | Allow user sign-up without token | +| LOG_CONFIG_OVERRIDE | | Override the config for logging with a custom path | +| LOG_LEVEL | info | logging level configured | \* Starting in v1.4.0 this was changed to default to `false` as apart of a security review of the application. @@ -27,15 +29,15 @@ ### Database -| Variables | Default | Description | -| ----------------- | :------: | -------------------------------- | -| DB_ENGINE | sqlite | Optional: 'sqlite', 'postgres' | -| POSTGRES_USER | mealie | Postgres database user | -| POSTGRES_PASSWORD | mealie | Postgres database password | -| POSTGRES_SERVER | postgres | Postgres database server address | -| POSTGRES_PORT | 5432 | Postgres database port | -| POSTGRES_DB | mealie | Postgres database name | -| POSTGRES_URL_OVERRIDE | None | Optional Postgres URL override to use instead of POSTGRES_* variables | +| Variables | Default | Description | +| --------------------- | :------: | ----------------------------------------------------------------------- | +| DB_ENGINE | sqlite | Optional: 'sqlite', 'postgres' | +| POSTGRES_USER | mealie | Postgres database user | +| POSTGRES_PASSWORD | mealie | Postgres database password | +| POSTGRES_SERVER | postgres | Postgres database server address | +| POSTGRES_PORT | 5432 | Postgres database port | +| POSTGRES_DB | mealie | Postgres database name | +| POSTGRES_URL_OVERRIDE | None | Optional Postgres URL override to use instead of POSTGRES\_\* variables | ### Email @@ -96,7 +98,7 @@ For usage, see [Usage - OpenID Connect](../authentication/oidc.md) | OIDC_PROVIDER_NAME | OAuth | The provider name is shown in SSO login button. "Login with " | | OIDC_REMEMBER_ME | False | Because redirects bypass the login screen, you cant extend your session by clicking the "Remember Me" checkbox. By setting this value to true, a session will be extended as if "Remember Me" was checked | | OIDC_SIGNING_ALGORITHM | RS256 | The algorithm used to sign the id token (examples: RS256, HS256) | -| OIDC_USER_CLAIM | email | Optional: 'email', 'preferred_username' +| OIDC_USER_CLAIM | email | Optional: 'email', 'preferred_username' | ### Themeing diff --git a/docs/docs/documentation/getting-started/installation/logs.md b/docs/docs/documentation/getting-started/installation/logs.md new file mode 100644 index 00000000000..1065a44224a --- /dev/null +++ b/docs/docs/documentation/getting-started/installation/logs.md @@ -0,0 +1,16 @@ +# Logs + +:octicons-tag-24: v1.5.0 + +## Highlighs + +- Logs are written to `/app/data/mealie.log` by default in the container. +- Logs are also written to stdout and stderr. +- You can adjust the log level using the `LOG_LEVEL` environment variable. + +## Configuration + +Starting in v1.5.0 logging is now highly configurable. Using the `LOG_CONFIG_OVERRIDE` you can provide the application with a custom configuration to log however you'd like. This configuration file is based off the [Python Logging Config](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig). It can be difficult to understand the configuration at first, so here are some resources to help get started. + +- This [YouTube Video](https://www.youtube.com/watch?v=9L77QExPmI0) for a great walkthrough on the logging file format. +- Our [Logging Config](https://github.com/mealie-recipes/mealie/blob/mealie-next/mealie/core/logger/logconf.prod.json) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 77b05c3370c..4d667d3d3a0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -73,6 +73,7 @@ nav: - PostgreSQL: "documentation/getting-started/installation/postgres.md" - Backend Configuration: "documentation/getting-started/installation/backend-config.md" - Security: "documentation/getting-started/installation/security.md" + - Logs: "documentation/getting-started/installation/logs.md" - Usage: - Backup and Restoring: "documentation/getting-started/usage/backups-and-restoring.md" - Permissions and Public Access: "documentation/getting-started/usage/permissions-and-public-access.md" diff --git a/frontend/lib/api/types/admin.ts b/frontend/lib/api/types/admin.ts index bdf98d9e9a0..36c255b3c22 100644 --- a/frontend/lib/api/types/admin.ts +++ b/frontend/lib/api/types/admin.ts @@ -203,7 +203,6 @@ export interface MaintenanceStorageDetails { } export interface MaintenanceSummary { dataDirSize: string; - logFileSize: string; cleanableImages: number; cleanableDirs: number; } diff --git a/frontend/pages/admin/maintenance/index.vue b/frontend/pages/admin/maintenance/index.vue index 1b6ccaa372c..4c45b696c7e 100644 --- a/frontend/pages/admin/maintenance/index.vue +++ b/frontend/pages/admin/maintenance/index.vue @@ -22,10 +22,6 @@ -
- -
-
@@ -110,7 +106,6 @@ export default defineComponent({ const infoResults = ref({ dataDirSize: i18n.tc("about.unknown-version"), - logFileSize: i18n.tc("about.unknown-version"), cleanableDirs: 0, cleanableImages: 0, }); @@ -121,7 +116,6 @@ export default defineComponent({ infoResults.value = data ?? { dataDirSize: i18n.tc("about.unknown-version"), - logFileSize: i18n.tc("about.unknown-version"), cleanableDirs: 0, cleanableImages: 0, }; @@ -129,17 +123,12 @@ export default defineComponent({ state.fetchingInfo = false; } - const info = computed(() => { return [ { name: i18n.t("admin.maintenance.info-description-data-dir-size"), value: infoResults.value.dataDirSize, }, - { - name: i18n.t("admin.maintenance.info-description-log-file-size"), - value: infoResults.value.logFileSize, - }, { name: i18n.t("admin.maintenance.info-description-cleanable-directories"), value: infoResults.value.cleanableDirs, @@ -184,12 +173,6 @@ export default defineComponent({ // ========================================================================== // Actions - async function handleDeleteLogFile() { - state.actionLoading = true; - await adminApi.maintenance.cleanLogFile(); - state.actionLoading = false; - } - async function handleCleanDirectories() { state.actionLoading = true; await adminApi.maintenance.cleanRecipeFolders(); @@ -209,11 +192,6 @@ export default defineComponent({ } const actions = [ - { - name: i18n.t("admin.maintenance.action-delete-log-files-name"), - handler: handleDeleteLogFile, - subtitle: i18n.t("admin.maintenance.action-delete-log-files-description"), - }, { name: i18n.t("admin.maintenance.action-clean-directories-name"), handler: handleCleanDirectories, diff --git a/frontend/pages/admin/maintenance/logs.vue b/frontend/pages/admin/maintenance/logs.vue deleted file mode 100644 index ca3b4b8445c..00000000000 --- a/frontend/pages/admin/maintenance/logs.vue +++ /dev/null @@ -1,109 +0,0 @@ - - - - - diff --git a/mealie/core/logger/config.py b/mealie/core/logger/config.py new file mode 100644 index 00000000000..fc2bde52772 --- /dev/null +++ b/mealie/core/logger/config.py @@ -0,0 +1,67 @@ +import json +import logging +import pathlib +import typing +from logging import config as logging_config + +__dir = pathlib.Path(__file__).parent +__conf: dict[str, str] | None = None + + +def _load_config(path: pathlib.Path, substitutions: dict[str, str] | None = None) -> dict[str, typing.Any]: + with open(path) as file: + if substitutions: + contents = file.read() + for key, value in substitutions.items(): + # Replaces the key matches + # + # Example: + # {"key": "value"} + # "/path/to/${key}/file" -> "/path/to/value/file" + contents = contents.replace(f"${{{key}}}", value) + + json_data = json.loads(contents) + + else: + json_data = json.load(file) + + return json_data + + +def log_config() -> dict[str, str]: + if __conf is None: + raise ValueError("logger not configured, must call configured_logger first") + + return __conf + + +def configured_logger( + *, + mode: str, + config_override: pathlib.Path | None = None, + substitutions: dict[str, str] | None = None, +) -> logging.Logger: + """ + Configure the logger based on the mode and return the root logger + + Args: + mode (str): The mode to configure the logger for (production, development, testing) + config_override (pathlib.Path, optional): A path to a custom logging config. Defaults to None. + substitutions (dict[str, str], optional): A dictionary of substitutions to apply to the logging config. + """ + global __conf + + if config_override: + __conf = _load_config(config_override, substitutions) + else: + if mode == "production": + __conf = _load_config(__dir / "logconf.prod.json", substitutions) + elif mode == "development": + __conf = _load_config(__dir / "logconf.dev.json", substitutions) + elif mode == "testing": + __conf = _load_config(__dir / "logconf.test.json", substitutions) + else: + raise ValueError(f"Invalid mode: {mode}") + + logging_config.dictConfig(config=__conf) + return logging.getLogger() diff --git a/mealie/core/logger/logconf.dev.json b/mealie/core/logger/logconf.dev.json new file mode 100644 index 00000000000..7f30c99a185 --- /dev/null +++ b/mealie/core/logger/logconf.dev.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "handlers": { + "rich": { + "class": "rich.logging.RichHandler" + } + }, + "loggers": { + "root": { + "level": "DEBUG", + "handlers": [ + "rich" + ] + } + } +} diff --git a/mealie/core/logger/logconf.prod.json b/mealie/core/logger/logconf.prod.json new file mode 100644 index 00000000000..383cbd2d569 --- /dev/null +++ b/mealie/core/logger/logconf.prod.json @@ -0,0 +1,74 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(levelname)-8s %(asctime)s - %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%S" + }, + "detailed": { + "format": "[%(levelname)s|%(module)s|L%(lineno)d] %(asctime)s: %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%S" + }, + "access": { + "()": "uvicorn.logging.AccessFormatter", + "fmt": "%(levelname)-8s %(asctime)s - [%(client_addr)s] %(status_code)s \"%(request_line)s\"", + "datefmt": "%Y-%m-%dT%H:%M:%S" + } + }, + "handlers": { + "stderr": { + "class": "logging.StreamHandler", + "level": "WARNING", + "formatter": "simple", + "stream": "ext://sys.stderr" + }, + "stdout": { + "class": "logging.StreamHandler", + "level": "${LOG_LEVEL}", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "access": { + "class": "logging.StreamHandler", + "level": "${LOG_LEVEL}", + "formatter": "access", + "stream": "ext://sys.stdout" + }, + "file": { + "class": "logging.handlers.RotatingFileHandler", + "level": "DEBUG", + "formatter": "detailed", + "filename": "${DATA_DIR}/mealie.log", + "maxBytes": 10000, + "backupCount": 3 + } + }, + "loggers": { + "root": { + "level": "${LOG_LEVEL}", + "handlers": [ + "stderr", + "file", + "stdout" + ] + }, + "uvicorn.error": { + "handlers": [ + "stderr", + "file", + "stdout" + ], + "level": "${LOG_LEVEL}", + "propagate": false + }, + "uvicorn.access": { + "handlers": [ + "access", + "file" + ], + "level": "${LOG_LEVEL}", + "propagate": false + } + } +} diff --git a/mealie/core/logger/logconf.test.json b/mealie/core/logger/logconf.test.json new file mode 100644 index 00000000000..2bd0484f714 --- /dev/null +++ b/mealie/core/logger/logconf.test.json @@ -0,0 +1,26 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "detailed": { + "format": "[%(levelname)s|%(module)s|L%(lineno)d] %(asctime)s: %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%S" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "detailed", + "stream": "ext://sys.stdout" + } + }, + "loggers": { + "root": { + "level": "${LOG_LEVEL}", + "handlers": [ + "stdout" + ] + } + } +} diff --git a/mealie/core/root_logger.py b/mealie/core/root_logger.py index 29db504fee7..a9dc4546a12 100644 --- a/mealie/core/root_logger.py +++ b/mealie/core/root_logger.py @@ -1,85 +1,46 @@ import logging -import sys -from dataclasses import dataclass -from functools import lru_cache -from mealie.core.config import determine_data_dir +from .config import get_app_dirs, get_app_settings +from .logger.config import configured_logger -DATA_DIR = determine_data_dir() +__root_logger: None | logging.Logger = None -from .config import get_app_settings # noqa E402 -LOGGER_FILE = DATA_DIR.joinpath("mealie.log") -DATE_FORMAT = "%d-%b-%y %H:%M:%S" -LOGGER_FORMAT = "%(levelname)s: %(asctime)s \t%(message)s" +def get_logger(module=None) -> logging.Logger: + """ + Get a logger instance for a module, in most cases module should not be + provided. Simply using the root logger is sufficient. + Cases where you would want to use a module specific logger might be a background + task or a long running process where you want to easily identify the source of + those messages + """ + global __root_logger -@dataclass -class LoggerConfig: - handlers: list - format: str - date_format: str - logger_file: str - level: int = logging.INFO + if __root_logger is None: + app_settings = get_app_settings() + mode = "development" -@lru_cache -def get_logger_config(): - settings = get_app_settings() + if app_settings.TESTING: + mode = "testing" + elif app_settings.PRODUCTION: + mode = "production" - log_level = logging._nameToLevel[settings.LOG_LEVEL] + dirs = get_app_dirs() - if not settings.PRODUCTION: - from rich.logging import RichHandler + substitutions = { + "DATA_DIR": dirs.DATA_DIR.as_posix(), + "LOG_LEVEL": app_settings.LOG_LEVEL.upper(), + } - return LoggerConfig( - handlers=[RichHandler(rich_tracebacks=True, tracebacks_show_locals=True)], - format=None, - date_format=None, - logger_file=None, - level=log_level, + __root_logger = configured_logger( + mode=mode, + config_override=app_settings.LOG_CONFIG_OVERRIDE, + substitutions=substitutions, ) - output_file_handler = logging.FileHandler(LOGGER_FILE) - handler_format = logging.Formatter(LOGGER_FORMAT, datefmt=DATE_FORMAT) - output_file_handler.setFormatter(handler_format) - - # Stdout - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.setFormatter(handler_format) - - return LoggerConfig( - handlers=[output_file_handler, stdout_handler], - format="%(levelname)s: %(asctime)s \t%(message)s", - date_format="%d-%b-%y %H:%M:%S", - logger_file=LOGGER_FILE, - level=log_level, - ) - - -logger_config = get_logger_config() - -logging.basicConfig( - level=logger_config.level, - format=logger_config.format, - datefmt=logger_config.date_format, - handlers=logger_config.handlers, -) - - -def logger_init() -> logging.Logger: - """Returns the Root Logging Object for Mealie""" - return logging.getLogger("mealie") - - -root_logger = logger_init() - - -def get_logger(module=None) -> logging.Logger: - """Returns a child logger for mealie""" - global root_logger - if module is None: - return root_logger + return __root_logger - return root_logger.getChild(module) + return __root_logger.getChild(module) diff --git a/mealie/core/settings/settings.py b/mealie/core/settings/settings.py index ba52432d023..f035bbbc795 100644 --- a/mealie/core/settings/settings.py +++ b/mealie/core/settings/settings.py @@ -36,13 +36,21 @@ class AppSettings(BaseSettings): """path to static files directory (ex. `mealie/dist`)""" IS_DEMO: bool = False + + HOST_IP: str = "*" + + API_HOST: str = "0.0.0.0" API_PORT: int = 9000 API_DOCS: bool = True TOKEN_TIME: int = 48 """time in hours""" SECRET: str - LOG_LEVEL: str = "INFO" + + LOG_CONFIG_OVERRIDE: Path | None = None + """path to custom logging configuration file""" + + LOG_LEVEL: str = "info" """corresponds to standard Python log levels""" GIT_COMMIT_HASH: str = "unknown" diff --git a/mealie/main.py b/mealie/main.py new file mode 100644 index 00000000000..f18d4dcdcf8 --- /dev/null +++ b/mealie/main.py @@ -0,0 +1,20 @@ +import uvicorn + +from mealie.app import settings +from mealie.core.logger.config import log_config + + +def main(): + uvicorn.run( + "app:app", + host=settings.API_HOST, + port=settings.API_PORT, + log_level=settings.LOG_LEVEL.lower(), + log_config=log_config(), + workers=1, + forwarded_allow_ips=settings.HOST_IP, + ) + + +if __name__ == "__main__": + main() diff --git a/mealie/routes/admin/__init__.py b/mealie/routes/admin/__init__.py index 77ea8a9957d..81766370cd6 100644 --- a/mealie/routes/admin/__init__.py +++ b/mealie/routes/admin/__init__.py @@ -5,7 +5,6 @@ admin_analytics, admin_backups, admin_email, - admin_log, admin_maintenance, admin_management_groups, admin_management_users, @@ -15,7 +14,6 @@ router = AdminAPIRouter(prefix="/admin") router.include_router(admin_about.router, tags=["Admin: About"]) -router.include_router(admin_log.router, tags=["Admin: Log"]) router.include_router(admin_management_users.router, tags=["Admin: Manage Users"]) router.include_router(admin_management_groups.router, tags=["Admin: Manage Groups"]) router.include_router(admin_email.router, tags=["Admin: Email"]) diff --git a/mealie/routes/admin/admin_log.py b/mealie/routes/admin/admin_log.py deleted file mode 100644 index ac12c12ed0e..00000000000 --- a/mealie/routes/admin/admin_log.py +++ /dev/null @@ -1,44 +0,0 @@ -from fastapi import APIRouter - -from mealie.core.root_logger import LOGGER_FILE -from mealie.core.security import create_file_token - -router = APIRouter(prefix="/logs") - - -@router.get("/{num}") -async def get_log(num: int): - """Doc Str""" - with open(LOGGER_FILE, "rb") as f: - log_text = tail(f, num) - return log_text - - -@router.get("") -async def get_log_file(): - """Returns a token to download a file""" - return {"fileToken": create_file_token(LOGGER_FILE)} - - -def tail(f, lines=20): - total_lines_wanted = lines - - BLOCK_SIZE = 1024 - f.seek(0, 2) - block_end_byte = f.tell() - lines_to_go = total_lines_wanted - block_number = -1 - blocks = [] - while lines_to_go > 0 and block_end_byte > 0: - if block_end_byte - BLOCK_SIZE > 0: - f.seek(block_number * BLOCK_SIZE, 2) - blocks.append(f.read(BLOCK_SIZE)) - else: - f.seek(0, 0) - blocks.append(f.read(block_end_byte)) - lines_found = blocks[-1].count(b"\n") - lines_to_go -= lines_found - block_end_byte -= BLOCK_SIZE - block_number -= 1 - all_read_text = b"".join(reversed(blocks)) - return b"/n".join(all_read_text.splitlines()[-total_lines_wanted:]) diff --git a/mealie/routes/admin/admin_maintenance.py b/mealie/routes/admin/admin_maintenance.py index 23ef8369dd3..eab3b982c54 100644 --- a/mealie/routes/admin/admin_maintenance.py +++ b/mealie/routes/admin/admin_maintenance.py @@ -1,16 +1,13 @@ -import contextlib -import os import shutil import uuid from pathlib import Path from fastapi import APIRouter, HTTPException -from mealie.core.root_logger import LOGGER_FILE from mealie.pkgs.stats import fs_stats from mealie.routes._base import BaseAdminController, controller from mealie.schema.admin import MaintenanceSummary -from mealie.schema.admin.maintenance import MaintenanceLogs, MaintenanceStorageDetails +from mealie.schema.admin.maintenance import MaintenanceStorageDetails from mealie.schema.response import ErrorResponse, SuccessResponse router = APIRouter(prefix="/maintenance") @@ -72,21 +69,13 @@ def get_maintenance_summary(self): """ Get the maintenance summary """ - log_file_size = 0 - with contextlib.suppress(FileNotFoundError): - log_file_size = os.path.getsize(LOGGER_FILE) return MaintenanceSummary( data_dir_size=fs_stats.pretty_size(fs_stats.get_dir_size(self.folders.DATA_DIR)), - log_file_size=fs_stats.pretty_size(log_file_size), cleanable_images=clean_images(self.folders.RECIPE_DATA_DIR, dry_run=True), cleanable_dirs=clean_recipe_folders(self.folders.RECIPE_DATA_DIR, dry_run=True), ) - @router.get("/logs", response_model=MaintenanceLogs) - def get_logs(self, lines: int = 200): - return MaintenanceLogs(logs=tail_log(LOGGER_FILE, lines)) - @router.get("/storage", response_model=MaintenanceStorageDetails) def get_storage_details(self): return MaintenanceStorageDetails( @@ -130,16 +119,3 @@ def clean_recipe_folders(self): return SuccessResponse.respond(f"{cleaned_dirs} Recipe folders removed") except Exception as e: raise HTTPException(status_code=500, detail=ErrorResponse.respond("Failed to clean directories")) from e - - @router.post("/clean/logs", response_model=SuccessResponse) - def clean_logs(self): - """ - Purges the logs - """ - try: - with contextlib.suppress(FileNotFoundError): - os.remove(LOGGER_FILE) - LOGGER_FILE.touch() - return SuccessResponse.respond("Logs cleaned") - except Exception as e: - raise HTTPException(status_code=500, detail=ErrorResponse.respond("Failed to clean logs")) from e diff --git a/mealie/schema/admin/maintenance.py b/mealie/schema/admin/maintenance.py index d5d28bdf8a0..2d1e61b679e 100644 --- a/mealie/schema/admin/maintenance.py +++ b/mealie/schema/admin/maintenance.py @@ -3,7 +3,6 @@ class MaintenanceSummary(MealieModel): data_dir_size: str - log_file_size: str cleanable_images: int cleanable_dirs: int