diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 262c348f3..e331d8d5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,9 +22,6 @@ repos: rev: 1.1.2 hooks: - id: copyright - - id: poetry-check - additional_dependencies: - - poetry==1.8.4 # pypi - id: poetry-lock additional_dependencies: - poetry==1.8.4 # pypi diff --git a/Dockerfile b/Dockerfile index b09ff9754..1bfa018aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PATH=/venv/bin:$PATH # Used to convert the locked packages by poetry to pip requirements format # We don't directly use `poetry install` because it force to use a virtual environment. -FROM base-all-0 as poetry +FROM base-all-0 AS poetry # Install Poetry WORKDIR /tmp @@ -28,7 +28,7 @@ RUN poetry export --extras=all --output=requirements.txt \ && poetry export --extras=all --with=dev --output=requirements-dev.txt # Base, the biggest thing is to install the Python packages -FROM base-all-0 as base-all +FROM base-all-0 AS base-all # The /poetry/requirements.txt file is build with the command # poetry export --extras=all --output=requirements.txt, see above @@ -102,7 +102,7 @@ CMD ["/venv/bin/gunicorn"] COPY production.ini /app/ -FROM base-lint as tests +FROM base-lint AS tests WORKDIR /opt/c2cwsgiutils COPY c2cwsgiutils ./c2cwsgiutils @@ -113,4 +113,4 @@ RUN --mount=type=cache,target=/root/.cache \ POETRY_DYNAMIC_VERSIONING_BYPASS=${VERSION} python3 -m pip install --disable-pip-version-check --no-deps --editable=. \ && python3 -m pip freeze > /requirements.txt -FROM base as standard +FROM base AS standard diff --git a/Makefile b/Makefile index 279d0106f..911c478b4 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ tests: build_docker_test ## Run the unit tests acceptance: acceptance-in acceptance-out ## Run the acceptance tests .PHONY: acceptance-run -acceptance-run: tests ## Start the application used to run the acceptance tests +acceptance-run: tests build_test_app ## Start the application used to run the acceptance tests cd acceptance_tests/tests/; docker compose up --detach db db_slave cd acceptance_tests/tests/; docker compose run -T --no-deps app /app/scripts/wait-db cd acceptance_tests/tests/; docker compose up --detach @@ -60,8 +60,15 @@ build_docker_test: docker build --tag=$(DOCKER_BASE):tests --target=tests . .PHONY: build_test_app -build_test_app: build_docker - docker build --tag=$(DOCKER_BASE)_test_app --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/app +build_test_app: build_test_app_gunicorn build_test_app_waitress + +.PHONY: build_test_app_gunicorn +build_test_app_gunicorn: build_docker + docker build --tag=$(DOCKER_BASE)_test_app --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/gunicorn_app + +.PHONY: build_test_app_waitress +build_test_app_waitress: build_docker + docker build --tag=$(DOCKER_BASE)_test_app_waitress --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/waitress_app .PHONY: checks checks: prospector ## Run the checks diff --git a/acceptance_tests/app/run_alembic.sh b/acceptance_tests/app/run_alembic.sh deleted file mode 100755 index ddf5aec7b..000000000 --- a/acceptance_tests/app/run_alembic.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Upgrade the DB -set -e - -# wait for the DB to be UP -while ! echo "import sqlalchemy; sqlalchemy.create_engine('$SQLALCHEMY_URL').connect()" | python3 2> /dev/null; do - echo "Waiting for the DB to be reachable" - sleep 1 -done - -for ini in *alembic*.ini; do - if [[ -f $ini ]]; then - echo "$ini ===========================" - alembic -c "$ini" upgrade head - fi -done diff --git a/acceptance_tests/app/.coveragerc b/acceptance_tests/gunicorn_app/.coveragerc similarity index 100% rename from acceptance_tests/app/.coveragerc rename to acceptance_tests/gunicorn_app/.coveragerc diff --git a/acceptance_tests/app/.dockerignore b/acceptance_tests/gunicorn_app/.dockerignore similarity index 100% rename from acceptance_tests/app/.dockerignore rename to acceptance_tests/gunicorn_app/.dockerignore diff --git a/acceptance_tests/gunicorn_app/Dockerfile b/acceptance_tests/gunicorn_app/Dockerfile new file mode 100644 index 000000000..eb56f829b --- /dev/null +++ b/acceptance_tests/gunicorn_app/Dockerfile @@ -0,0 +1,70 @@ +FROM camptocamp/c2cwsgiutils AS base-all +LABEL maintainer Camptocamp "info@camptocamp.com" +SHELL ["/bin/bash", "-o", "pipefail", "-cux"] + +# Used to convert the locked packages by poetry to pip requirements format +# We don't directly use `poetry install` because it force to use a virtual environment. +FROM base-all AS poetry + +RUN --mount=type=cache,target=/var/lib/apt/lists \ + --mount=type=cache,target=/var/cache,sharing=locked \ + apt-get update \ + && apt-get install --assume-yes --no-install-recommends python-is-python3 + +# Install Poetry +WORKDIR /tmp +COPY requirements.txt ./ +RUN python3 -m pip install --disable-pip-version-check --requirement=requirements.txt + +# Do the conversion +COPY poetry.lock pyproject.toml ./ +RUN poetry export --output=requirements.txt \ + && poetry export --with=dev --output=requirements-dev.txt + +# Base, the biggest thing is to install the Python packages +FROM base-all AS base + +WORKDIR /app + +EXPOSE 8080 +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=bind,from=poetry,source=/tmp,target=/poetry \ + python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt + +COPY . /app + +ARG GIT_HASH + +RUN --mount=type=cache,target=/root/.cache \ + python3 -m pip install --disable-pip-version-check --no-deps --editable=. \ + && python3 -m pip freeze > /requirements.txt +RUN ./models_graph.py > models.dot \ + && ./models_graph.py Hello > models-hello.dot \ + && c2cwsgiutils-genversion $GIT_HASH \ + && python3 -m compileall -q . + +ENV \ + DOCKER_RUN=1 \ + DEVELOPMENT=0 \ + SQLALCHEMY_POOL_RECYCLE=30 \ + SQLALCHEMY_POOL_SIZE=5 \ + SQLALCHEMY_MAX_OVERFLOW=25 \ + SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \ + SQLALCHEMY_SLAVE_POOL_SIZE=5 \ + SQLALCHEMY_SLAVE_MAX_OVERFLOW=25 \ + LOG_TYPE=console \ + OTHER_LOG_LEVEL=WARNING \ + GUNICORN_LOG_LEVEL=WARNING \ + SQL_LOG_LEVEL=WARNING \ + C2CWSGIUTILS_LOG_LEVEL=WARNING \ + LOG_LEVEL=INFO \ + VISIBLE_ENTRY_POINT=/ + +RUN mkdir -p /prometheus-metrics \ + && chmod a+rwx /prometheus-metrics +ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics + +# www-data +USER 33 + +CMD ["/venv/bin/gunicorn", "--paste=/app/production.ini"] diff --git a/acceptance_tests/app/README.md b/acceptance_tests/gunicorn_app/README.md similarity index 100% rename from acceptance_tests/app/README.md rename to acceptance_tests/gunicorn_app/README.md diff --git a/acceptance_tests/app/alembic.ini b/acceptance_tests/gunicorn_app/alembic.ini similarity index 100% rename from acceptance_tests/app/alembic.ini rename to acceptance_tests/gunicorn_app/alembic.ini diff --git a/acceptance_tests/app/app_alembic/__init__.py b/acceptance_tests/gunicorn_app/app_alembic/__init__.py similarity index 100% rename from acceptance_tests/app/app_alembic/__init__.py rename to acceptance_tests/gunicorn_app/app_alembic/__init__.py diff --git a/acceptance_tests/app/app_alembic/env.py b/acceptance_tests/gunicorn_app/app_alembic/env.py similarity index 100% rename from acceptance_tests/app/app_alembic/env.py rename to acceptance_tests/gunicorn_app/app_alembic/env.py diff --git a/acceptance_tests/app/app_alembic/script.py.mako b/acceptance_tests/gunicorn_app/app_alembic/script.py.mako similarity index 100% rename from acceptance_tests/app/app_alembic/script.py.mako rename to acceptance_tests/gunicorn_app/app_alembic/script.py.mako diff --git a/acceptance_tests/app/app_alembic/versions/4a8c1bb4e775_initial_version.py b/acceptance_tests/gunicorn_app/app_alembic/versions/4a8c1bb4e775_initial_version.py similarity index 100% rename from acceptance_tests/app/app_alembic/versions/4a8c1bb4e775_initial_version.py rename to acceptance_tests/gunicorn_app/app_alembic/versions/4a8c1bb4e775_initial_version.py diff --git a/acceptance_tests/app/app_alembic/versions/__init__.py b/acceptance_tests/gunicorn_app/app_alembic/versions/__init__.py similarity index 100% rename from acceptance_tests/app/app_alembic/versions/__init__.py rename to acceptance_tests/gunicorn_app/app_alembic/versions/__init__.py diff --git a/acceptance_tests/app/c2cwsgiutils_app/__init__.py b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/__init__.py similarity index 99% rename from acceptance_tests/app/c2cwsgiutils_app/__init__.py rename to acceptance_tests/gunicorn_app/c2cwsgiutils_app/__init__.py index ca7fc6aba..cf0bcd543 100644 --- a/acceptance_tests/app/c2cwsgiutils_app/__init__.py +++ b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/__init__.py @@ -1,4 +1,3 @@ -from c2cwsgiutils_app import models from pyramid.config import Configurator from pyramid.httpexceptions import HTTPInternalServerError @@ -6,6 +5,8 @@ from c2cwsgiutils import broadcast, db from c2cwsgiutils.health_check import HealthCheck, JsonCheckException +from c2cwsgiutils_app import models + def _failure(_request): raise HTTPInternalServerError("failing check") diff --git a/acceptance_tests/app/c2cwsgiutils_app/get_hello.py b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/get_hello.py similarity index 99% rename from acceptance_tests/app/c2cwsgiutils_app/get_hello.py rename to acceptance_tests/gunicorn_app/c2cwsgiutils_app/get_hello.py index 4446c79d3..06ac9f022 100644 --- a/acceptance_tests/app/c2cwsgiutils_app/get_hello.py +++ b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/get_hello.py @@ -2,11 +2,12 @@ import psycopg2 import transaction -from c2cwsgiutils_app import models import c2cwsgiutils.db import c2cwsgiutils.setup_process +from c2cwsgiutils_app import models + def _fill_db(): for db, value in (("db", "master"), ("db_slave", "slave")): diff --git a/acceptance_tests/app/c2cwsgiutils_app/models.py b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/models.py similarity index 100% rename from acceptance_tests/app/c2cwsgiutils_app/models.py rename to acceptance_tests/gunicorn_app/c2cwsgiutils_app/models.py diff --git a/acceptance_tests/app/c2cwsgiutils_app/services.py b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/services.py similarity index 99% rename from acceptance_tests/app/c2cwsgiutils_app/services.py rename to acceptance_tests/gunicorn_app/c2cwsgiutils_app/services.py index d2d551fd0..084ec31f7 100644 --- a/acceptance_tests/app/c2cwsgiutils_app/services.py +++ b/acceptance_tests/gunicorn_app/c2cwsgiutils_app/services.py @@ -2,7 +2,6 @@ import prometheus_client import requests -from c2cwsgiutils_app import models from pyramid.httpexceptions import ( HTTPBadRequest, HTTPForbidden, @@ -13,6 +12,8 @@ from c2cwsgiutils import sentry, services +from c2cwsgiutils_app import models + _PROMETHEUS_TEST_COUNTER = prometheus_client.Counter("test_counter", "Test counter") _PROMETHEUS_TEST_GAUGE = prometheus_client.Gauge("test_gauge", "Test gauge", ["value", "toto"]) _PROMETHEUS_TEST_SUMMARY = prometheus_client.Summary("test_summary", "Test summary") diff --git a/acceptance_tests/app/gunicorn.conf.py b/acceptance_tests/gunicorn_app/gunicorn.conf.py similarity index 100% rename from acceptance_tests/app/gunicorn.conf.py rename to acceptance_tests/gunicorn_app/gunicorn.conf.py diff --git a/acceptance_tests/app/models_graph.py b/acceptance_tests/gunicorn_app/models_graph.py similarity index 100% rename from acceptance_tests/app/models_graph.py rename to acceptance_tests/gunicorn_app/models_graph.py index 85936a3cf..2fedeb0a8 100755 --- a/acceptance_tests/app/models_graph.py +++ b/acceptance_tests/gunicorn_app/models_graph.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -from c2cwsgiutils_app import models - from c2cwsgiutils.models_graph import generate_model_graph +from c2cwsgiutils_app import models + def main(): generate_model_graph(models) diff --git a/acceptance_tests/app/poetry.lock b/acceptance_tests/gunicorn_app/poetry.lock similarity index 100% rename from acceptance_tests/app/poetry.lock rename to acceptance_tests/gunicorn_app/poetry.lock diff --git a/acceptance_tests/app/production.ini b/acceptance_tests/gunicorn_app/production.ini similarity index 89% rename from acceptance_tests/app/production.ini rename to acceptance_tests/gunicorn_app/production.ini index ea8d86138..81cdad93e 100644 --- a/acceptance_tests/app/production.ini +++ b/acceptance_tests/gunicorn_app/production.ini @@ -1,11 +1,12 @@ ### -# app configuration +# Application configuration # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/environment.html # this file should be used by gunicorn. ### [app:app] use = egg:c2cwsgiutils_app +filter-with = proxy-prefix pyramid.reload_templates = %(DEVELOPMENT)s pyramid.debug_authorization = %(DEVELOPMENT)s @@ -27,15 +28,23 @@ sqlalchemy_slave.max_overflow = %(SQLALCHEMY_SLAVE_MAX_OVERFLOW)s c2c.sql_request_id = True c2c.requests_default_timeout = 2 +[filter:proxy-prefix] +use = egg:PasteDeploy#prefix +prefix = %(VISIBLE_ENTRY_POINT)s + +[filter:translogger] +use = egg:Paste#translogger +setup_console_handler = False + [pipeline:main] -pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry app +pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry translogger app [server:main] use = egg:waitress#main listen = *:8080 ### -# logging configuration +# Logging configuration # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html ### diff --git a/acceptance_tests/app/pyproject.toml b/acceptance_tests/gunicorn_app/pyproject.toml similarity index 100% rename from acceptance_tests/app/pyproject.toml rename to acceptance_tests/gunicorn_app/pyproject.toml diff --git a/acceptance_tests/app/requirements.txt b/acceptance_tests/gunicorn_app/requirements.txt similarity index 100% rename from acceptance_tests/app/requirements.txt rename to acceptance_tests/gunicorn_app/requirements.txt diff --git a/acceptance_tests/gunicorn_app/run-alembic b/acceptance_tests/gunicorn_app/run-alembic new file mode 100755 index 000000000..d17f1d8be --- /dev/null +++ b/acceptance_tests/gunicorn_app/run-alembic @@ -0,0 +1,24 @@ +#!/bin/bash +# Upgrade the DB +set -eu + +# wait for the DB to be UP +while ! echo "import sqlalchemy; sqlalchemy.create_engine('${SQLALCHEMY_URL}').connect()" | python3 2> /dev/null; do + echo "Waiting for the DB to be reachable" + sleep 1 +done + +for ini in *alembic*.ini; do + if [[ -f "${ini}" ]]; then + echo "${ini} ===========================" + echo History + alembic -c "${ini}" history + echo Head + alembic -c "${ini}" head + echo Upgrade + alembic -c "${ini}" upgrade head + echo Current + alembic -c "${ini}" current + echo "===========================" + fi +done diff --git a/acceptance_tests/app/scripts/wait-db b/acceptance_tests/gunicorn_app/scripts/wait-db similarity index 98% rename from acceptance_tests/app/scripts/wait-db rename to acceptance_tests/gunicorn_app/scripts/wait-db index 187505565..5d7f29076 100755 --- a/acceptance_tests/app/scripts/wait-db +++ b/acceptance_tests/gunicorn_app/scripts/wait-db @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2023, Camptocamp SA +# Copyright (c) 2023-2024, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without diff --git a/acceptance_tests/tests/docker-compose.override.sample.yaml b/acceptance_tests/tests/docker-compose.override.sample.yaml index 2c870a982..d11727938 100644 --- a/acceptance_tests/tests/docker-compose.override.sample.yaml +++ b/acceptance_tests/tests/docker-compose.override.sample.yaml @@ -4,7 +4,7 @@ services: app: # Uncomment to use pserve # command: - # - pserve + # - /venv/bin/pserve # - --reload # - c2c:///app/production.ini volumes: @@ -12,12 +12,20 @@ services: # the views are automatically reloaded when a file change - ../app/c2cwsgiutils_app/:/app/c2cwsgiutils_app/:ro - ../../c2cwsgiutils/:/opt/c2cwsgiutils/c2cwsgiutils/:ro + environment: + - DEVELOPMENT=TRUE ports: - 9090:9090 app2: + # command: + # - /venv/bin/pserve + # - --reload + # - c2c:///app/production.ini ports: - 9092:9090 + environment: + - DEVELOPMENT=TRUE db: ports: diff --git a/acceptance_tests/tests/docker-compose.yaml b/acceptance_tests/tests/docker-compose.yaml index b07362635..91576311c 100644 --- a/acceptance_tests/tests/docker-compose.yaml +++ b/acceptance_tests/tests/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '2.2' - services: app: &app image: camptocamp/c2cwsgiutils_test_app @@ -34,7 +32,6 @@ services: # Test problematic environment variable (values contains % and duplicated with different cass) - TEST='%1' - test='%2' - - PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics - C2C_PROMETHEUS_PORT=9090 - C2C_PROMETHEUS_APPLICATION_PACKAGE=c2cwsgiutils_app links: @@ -46,6 +43,7 @@ services: app2: <<: *app + image: camptocamp/c2cwsgiutils_test_app_waitress # Same as app but with 2 workers (and different Redis DB 2, broadcast_prefix, ports, and JSON log format) environment: - SQLALCHEMY_URL @@ -61,7 +59,6 @@ services: - C2C_PROFILER_PATH=/api_profiler - C2C_PROFILER_MODULES=c2cwsgiutils c2cwsgiutils_app sqlalchemy request - C2C_ENABLE_EXCEPTION_HANDLING=1 - - GUNICORN_CMD_ARGS="--reload" # don't use this in production - C2CWSGIUTILS_LOG_LEVEL=DEBUG - SQL_LOG_LEVEL=DEBUG - OTHER_LOG_LEVEL=INFO @@ -75,12 +72,9 @@ services: - C2C_BROADCAST_PREFIX=app2 - PYTHONMALLOC=debug - DEBUG_LOGCONFIG - - GUNICORN_WORKERS=2 - - GUNICORN_THREADS=10 # Test problematic environment variable (values contains % and duplicated with different cass) - TEST='%1' - test='%2' - - PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics - C2C_PROMETHEUS_PORT=9090 - C2C_PROMETHEUS_APPLICATION_PACKAGES=c2cwsgiutils_app ports: @@ -101,10 +95,10 @@ services: links: - db command: - - /app/run_alembic.sh + - /app/run-alembic alembic_slave: - image: camptocamp/c2cwsgiutils_test_app + image: camptocamp/c2cwsgiutils_test_app_waitress environment: - SQLALCHEMY_URL - SQLALCHEMY_SLAVE_URL @@ -118,7 +112,7 @@ services: links: - db_slave:db command: - - /app/run_alembic.sh + - /app/run-alembic db: &db image: camptocamp/postgres:14-postgis-3 diff --git a/acceptance_tests/tests/tests/conftest.py b/acceptance_tests/tests/tests/conftest.py index b34c3bf8c..cae878100 100644 --- a/acceptance_tests/tests/tests/conftest.py +++ b/acceptance_tests/tests/tests/conftest.py @@ -8,7 +8,8 @@ _BASE_URL = "http://app:8080/api/" _BASE_URL_APP2 = "http://app2:8080/api/" -_PROMETHEUS_URL = "http://app2:9090/metrics" +_PROMETHEUS_URL_1 = "http://app:9090/metrics" +_PROMETHEUS_URL_2 = "http://app2:9090/metrics" _PROMETHEUS_TEST_URL = "http://run_test:9090/metrics" _PROMETHEUS_STATS_DB_URL = "http://stats_db:9090/metrics" _LOG = logging.getLogger(__name__) @@ -44,13 +45,23 @@ def app2_connection(composition): @pytest.fixture -def prometheus_connection(composition): +def prometheus_1_connection(composition): """ Fixture that returns a connection to a running batch container. """ del composition - return Connection(base_url=_PROMETHEUS_URL, origin="http://example.com/") + return Connection(base_url=_PROMETHEUS_URL_1, origin="http://example.com/") + + +@pytest.fixture +def prometheus_2_connection(composition): + """ + Fixture that returns a connection to a running batch container. + """ + + del composition + return Connection(base_url=_PROMETHEUS_URL_2, origin="http://example.com/") @pytest.fixture diff --git a/acceptance_tests/tests/tests/test_prometheus_client.py b/acceptance_tests/tests/tests/test_prometheus_client.py index e88944f85..0ea8c3d06 100644 --- a/acceptance_tests/tests/tests/test_prometheus_client.py +++ b/acceptance_tests/tests/tests/test_prometheus_client.py @@ -3,8 +3,17 @@ _PID_RE = re.compile(r',pid="([0-9]+)"') -def test_prometheus(prometheus_connection): +def test_prometheus_1(prometheus_1_connection): # One for the root process, one for each workers assert ( - len(set(_PID_RE.findall(prometheus_connection.get("metrics", cache_expected=False, cors=False)))) == 3 + len(set(_PID_RE.findall(prometheus_1_connection.get("metrics", cache_expected=False, cors=False)))) + == 3 + ) + + +def test_prometheus_2(prometheus_2_connection): + # One for the root process, one for each workers + assert ( + len(set(_PID_RE.findall(prometheus_2_connection.get("metrics", cache_expected=False, cors=False)))) + == 3 ) diff --git a/acceptance_tests/waitress_app/.coveragerc b/acceptance_tests/waitress_app/.coveragerc new file mode 100644 index 000000000..e1e8251a2 --- /dev/null +++ b/acceptance_tests/waitress_app/.coveragerc @@ -0,0 +1,9 @@ +[run] +source= + /opt/c2cwsgiutils/c2cwsgiutils + /app/c2cwsgiutils_app +branch=True +omit= + /opt/c2cwsgiutils/c2cwsgiutils/acceptance/* + /opt/c2cwsgiutils/c2cwsgiutils/models_graph.py + /opt/c2cwsgiutils/c2cwsgiutils/coverage_setup.py diff --git a/acceptance_tests/waitress_app/.dockerignore b/acceptance_tests/waitress_app/.dockerignore new file mode 100644 index 000000000..f38fe123a --- /dev/null +++ b/acceptance_tests/waitress_app/.dockerignore @@ -0,0 +1,3 @@ +**/__pycache__ +Dockerfile +c2cwsgiutils_app.egg-info diff --git a/acceptance_tests/app/Dockerfile b/acceptance_tests/waitress_app/Dockerfile similarity index 85% rename from acceptance_tests/app/Dockerfile rename to acceptance_tests/waitress_app/Dockerfile index 4e1851e79..ac659d611 100644 --- a/acceptance_tests/app/Dockerfile +++ b/acceptance_tests/waitress_app/Dockerfile @@ -1,10 +1,10 @@ -FROM camptocamp/c2cwsgiutils as base-all +FROM camptocamp/c2cwsgiutils AS base-all LABEL maintainer Camptocamp "info@camptocamp.com" SHELL ["/bin/bash", "-o", "pipefail", "-cux"] # Used to convert the locked packages by poetry to pip requirements format # We don't directly use `poetry install` because it force to use a virtual environment. -FROM base-all as poetry +FROM base-all AS poetry RUN --mount=type=cache,target=/var/lib/apt/lists \ --mount=type=cache,target=/var/cache,sharing=locked \ @@ -22,7 +22,7 @@ RUN poetry export --output=requirements.txt \ && poetry export --with=dev --output=requirements-dev.txt # Base, the biggest thing is to install the Python packages -FROM base-all as base +FROM base-all AS base WORKDIR /app @@ -51,12 +51,17 @@ ENV \ SQLALCHEMY_MAX_OVERFLOW=25 \ SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \ SQLALCHEMY_SLAVE_POOL_SIZE=5 \ - SQLALCHEMY_SLAVE_MAX_OVERFLOW=25 LOG_TYPE=console \ + SQLALCHEMY_SLAVE_MAX_OVERFLOW=25 \ + LOG_TYPE=console \ OTHER_LOG_LEVEL=WARNING \ - GUNICORN_LOG_LEVEL=WARNING \ + WAITRESS_LOG_LEVEL=WARNING \ SQL_LOG_LEVEL=WARNING \ C2CWSGIUTILS_LOG_LEVEL=WARNING \ - LOG_LEVEL=INFO + LOG_LEVEL=INFO \ + WAITRESS_THREADS=10 \ + VISIBLE_ENTRY_POINT=/ # www-data USER 33 + +CMD ["/venv/bin/pserve", "c2c:///app/production.ini"] diff --git a/acceptance_tests/waitress_app/README.md b/acceptance_tests/waitress_app/README.md new file mode 100644 index 000000000..790162a66 --- /dev/null +++ b/acceptance_tests/waitress_app/README.md @@ -0,0 +1,8 @@ +# A sample application + +This is a sample application shipped as a Docker Image. + +It's used for testing the library. Because we want to test the currently +checked out version of the library, it is not in the requirements.txt +file. The root Makefile copies the library inline with the application +before building the Docker image. diff --git a/acceptance_tests/waitress_app/alembic.ini b/acceptance_tests/waitress_app/alembic.ini new file mode 100644 index 000000000..956d1ab51 --- /dev/null +++ b/acceptance_tests/waitress_app/alembic.ini @@ -0,0 +1,32 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = app_alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = %(SQLALCHEMY_URL)s diff --git a/acceptance_tests/waitress_app/app_alembic/__init__.py b/acceptance_tests/waitress_app/app_alembic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance_tests/waitress_app/app_alembic/env.py b/acceptance_tests/waitress_app/app_alembic/env.py new file mode 100644 index 000000000..c2b987e82 --- /dev/null +++ b/acceptance_tests/waitress_app/app_alembic/env.py @@ -0,0 +1,70 @@ +import os + +from alembic import context +from sqlalchemy import engine_from_config, pool + +import c2cwsgiutils.setup_process + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +config.set_main_option("sqlalchemy.url", os.environ["SQLALCHEMY_URL"]) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """ + Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """ + Run migrations in 'online' mode. + + In this scenario we need to create an Engine and associate a connection with the context. + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + # To allow to run "UPDATE" in a migration and do an "ALTER" in another one, later. + transaction_per_migration=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +c2cwsgiutils.setup_process.bootstrap_application() +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/acceptance_tests/waitress_app/app_alembic/script.py.mako b/acceptance_tests/waitress_app/app_alembic/script.py.mako new file mode 100644 index 000000000..0814b42c2 --- /dev/null +++ b/acceptance_tests/waitress_app/app_alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +import sqlalchemy as sa +from alembic import op + +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/acceptance_tests/waitress_app/app_alembic/versions/4a8c1bb4e775_initial_version.py b/acceptance_tests/waitress_app/app_alembic/versions/4a8c1bb4e775_initial_version.py new file mode 100644 index 000000000..0ee5cb89e --- /dev/null +++ b/acceptance_tests/waitress_app/app_alembic/versions/4a8c1bb4e775_initial_version.py @@ -0,0 +1,31 @@ +""" +Initial version. + +Revision ID: 4a8c1bb4e775 +Revises: +Create Date: 2016-09-14 09:23:27.466418 +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "4a8c1bb4e775" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS postgis;") + op.execute( + """ + CREATE TABLE hello ( + id SERIAL PRIMARY KEY, + value TEXT UNIQUE INITIALLY DEFERRED + ) + """ + ) + + +def downgrade(): + pass diff --git a/acceptance_tests/waitress_app/app_alembic/versions/__init__.py b/acceptance_tests/waitress_app/app_alembic/versions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance_tests/waitress_app/c2cwsgiutils_app/__init__.py b/acceptance_tests/waitress_app/c2cwsgiutils_app/__init__.py new file mode 100644 index 000000000..6a58799fe --- /dev/null +++ b/acceptance_tests/waitress_app/c2cwsgiutils_app/__init__.py @@ -0,0 +1,52 @@ +from pyramid.config import Configurator +from pyramid.httpexceptions import HTTPInternalServerError + +import c2cwsgiutils.prometheus +import c2cwsgiutils.pyramid +from c2cwsgiutils import broadcast, db +from c2cwsgiutils.health_check import HealthCheck, JsonCheckException + +from c2cwsgiutils_app import models + + +def _failure(_request): + raise HTTPInternalServerError("failing check") + + +def _failure_json(_request): + raise JsonCheckException("failing check", {"some": "json"}) + + +@broadcast.decorator(expect_answers=True) +def broadcast_view(): + return 42 + + +def main(_, **settings): + """ + This function returns a Pyramid WSGI application. + """ + + c2cwsgiutils.prometheus.start_single_process() + + config = Configurator(settings=settings, route_prefix="/api") + + # Initialize the broadcast view before c2cwsgiutils is initialized. This allows to test the + # reconfiguration on the fly of the broadcast framework + config.add_route("broadcast", r"/broadcast", request_method="GET") + config.add_view( + lambda request: broadcast_view(), route_name="broadcast", renderer="fast_json", http_cache=0 + ) + + config.include(c2cwsgiutils.pyramid.includeme) + dbsession = db.init(config, "sqlalchemy", "sqlalchemy_slave", force_slave=["POST /api/hello"]) + config.scan("c2cwsgiutils_app.services") + health_check = HealthCheck(config) + health_check.add_db_session_check(dbsession, at_least_one_model=models.Hello) + health_check.add_url_check("http://localhost:8080/api/hello") + health_check.add_url_check(name="fun_url", url=lambda _request: "http://localhost:8080/api/hello") + health_check.add_custom_check("fail", _failure, 2) + health_check.add_custom_check("fail_json", _failure_json, 2) + health_check.add_alembic_check(dbsession, "/app/alembic.ini", 1) + + return config.make_wsgi_app() diff --git a/acceptance_tests/waitress_app/c2cwsgiutils_app/get_hello.py b/acceptance_tests/waitress_app/c2cwsgiutils_app/get_hello.py new file mode 100644 index 000000000..06ac9f022 --- /dev/null +++ b/acceptance_tests/waitress_app/c2cwsgiutils_app/get_hello.py @@ -0,0 +1,42 @@ +import argparse + +import psycopg2 +import transaction + +import c2cwsgiutils.db +import c2cwsgiutils.setup_process + +from c2cwsgiutils_app import models + + +def _fill_db(): + for db, value in (("db", "master"), ("db_slave", "slave")): + connection = psycopg2.connect( + database="test", user="www-data", password="www-data", host=db, port=5432 + ) + with connection.cursor() as curs: + curs.execute("DELETE FROM hello") + curs.execute("INSERT INTO hello (value) VALUES (%s)", (value,)) + connection.commit() + + +def main() -> None: + """Get the fist hello value.""" + + parser = argparse.ArgumentParser(description="Get the first hello value.") + c2cwsgiutils.setup_process.fill_arguments(parser) + args = parser.parse_args() + env = c2cwsgiutils.setup_process.bootstrap_application_from_options(args) + + engine = c2cwsgiutils.db.get_engine(env["registry"].settings) + session_factory = c2cwsgiutils.db.get_session_factory(engine) + with transaction.manager: + dbsession = c2cwsgiutils.db.get_tm_session(session_factory, transaction.manager) + if len(dbsession.query(models.Hello).all()) == 0: + _fill_db() + hello = dbsession.query(models.Hello).first() + print(hello.value) + + +if __name__ == "__main__": + main() diff --git a/acceptance_tests/waitress_app/c2cwsgiutils_app/models.py b/acceptance_tests/waitress_app/c2cwsgiutils_app/models.py new file mode 100644 index 000000000..d48583218 --- /dev/null +++ b/acceptance_tests/waitress_app/c2cwsgiutils_app/models.py @@ -0,0 +1,10 @@ +import sqlalchemy as sa +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + + +class Hello(Base): + __tablename__ = "hello" + id = sa.Column(sa.Integer, primary_key=True, autoincrement=True) + value = sa.Column(sa.Text, nullable=False) diff --git a/acceptance_tests/waitress_app/c2cwsgiutils_app/services.py b/acceptance_tests/waitress_app/c2cwsgiutils_app/services.py new file mode 100644 index 000000000..084ec31f7 --- /dev/null +++ b/acceptance_tests/waitress_app/c2cwsgiutils_app/services.py @@ -0,0 +1,122 @@ +import logging + +import prometheus_client +import requests +from pyramid.httpexceptions import ( + HTTPBadRequest, + HTTPForbidden, + HTTPMovedPermanently, + HTTPNoContent, + HTTPUnauthorized, +) + +from c2cwsgiutils import sentry, services + +from c2cwsgiutils_app import models + +_PROMETHEUS_TEST_COUNTER = prometheus_client.Counter("test_counter", "Test counter") +_PROMETHEUS_TEST_GAUGE = prometheus_client.Gauge("test_gauge", "Test gauge", ["value", "toto"]) +_PROMETHEUS_TEST_SUMMARY = prometheus_client.Summary("test_summary", "Test summary") + + +ping_service = services.create("ping", "/ping") +hello_service = services.create("hello", "/hello", cors_credentials=True) +error_service = services.create("error", "/error") +tracking_service = services.create("tracking", "/tracking/{depth:[01]}") +empty_service = services.create("empty", "/empty") +timeout_service = services.create("timeout", "timeout/{where:sql}") +leaked_objects = [] + + +class LeakedObject: + pass + + +@ping_service.get() +def ping(_): + leaked_objects.append(LeakedObject()) # A memory leak to test debug/memory_diff + logging.getLogger(__name__ + ".ping").info("Ping!") + return {"pong": True} + + +@hello_service.get() +def hello_get(request): + """ + Will use the slave. + """ + with _PROMETHEUS_TEST_SUMMARY.time(): + hello = request.dbsession.query(models.Hello).first() + _PROMETHEUS_TEST_COUNTER.inc() + _PROMETHEUS_TEST_GAUGE.labels(value=24, toto="tutu").set(42) + return {"value": hello.value} + + +@hello_service.put() +def hello_put(request): + """ + Will use the master. + """ + with sentry.capture_exceptions(): + hello = request.dbsession.query(models.Hello).first() + return {"value": hello.value} + + +@hello_service.post() +def hello_post(request): + """ + Will use the slave (overridden by the config). + """ + return hello_put(request) + + +@error_service.get() +def error(request): + code = int(request.params.get("code", "500")) + if code == 400: + raise HTTPBadRequest("arg") + if code == 403: + raise HTTPForbidden("bam") + if code == 401: + e = HTTPUnauthorized() + e.headers["WWW-Authenticate"] = 'Basic realm="Access to staging site"' + raise e + if code == 301: + raise HTTPMovedPermanently(location="http://www.camptocamp.com/en/") + if code == 204: + raise HTTPNoContent() + if request.params.get("db", "0") == "dup": + for _ in range(2): + request.dbsession.add(models.Hello(value="toto")) + elif request.params.get("db", "0") == "data": + request.dbsession.add(models.Hello(id="abcd", value="toto")) + else: + raise Exception("boom") + return {"status": 200} + + +@tracking_service.get() +def tracking(request): + depth = int(request.matchdict.get("depth")) + result = {"request_id": request.c2c_request_id} + if depth > 0: + result["sub"] = requests.get(f"http://localhost:8080/api/tracking/{depth - 1}").json() + return result + + +@empty_service.put() +def empty_put(request): + request.response.status_code = 204 + return request.response + + +@empty_service.patch() +def empty_patch(request): + request.response.status_code = 204 + return request.response + + +@timeout_service.get(match_param="where=sql") +def timeout_sql(request): + request.dbsession.execute(sqlalchemy.sql.expression.text("SELECT pg_sleep(2)")) + request.response.status_code = 204 + return request.response diff --git a/acceptance_tests/waitress_app/models_graph.py b/acceptance_tests/waitress_app/models_graph.py new file mode 100755 index 000000000..2fedeb0a8 --- /dev/null +++ b/acceptance_tests/waitress_app/models_graph.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +from c2cwsgiutils.models_graph import generate_model_graph + +from c2cwsgiutils_app import models + + +def main(): + generate_model_graph(models) + + +main() diff --git a/acceptance_tests/waitress_app/poetry.lock b/acceptance_tests/waitress_app/poetry.lock new file mode 100644 index 000000000..9d0e3c591 --- /dev/null +++ b/acceptance_tests/waitress_app/poetry.lock @@ -0,0 +1,726 @@ +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "astroid" +version = "3.3.5" +description = "An abstract syntax tree for Python with inference support." +optional = false +python-versions = ">=3.9.0" +files = [ + {file = "astroid-3.3.5-py3-none-any.whl", hash = "sha256:a9d1c946ada25098d790e079ba2a1b112157278f3fb7e718ae6a9252f5835dc8"}, + {file = "astroid-3.3.5.tar.gz", hash = "sha256:5cfc40ae9f68311075d27ef68a4841bdc5cc7f6cf86671b49f00607d30188e2d"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "bandit" +version = "1.7.10" +description = "Security oriented static analyser for python code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bandit-1.7.10-py3-none-any.whl", hash = "sha256:665721d7bebbb4485a339c55161ac0eedde27d51e638000d91c8c2d68343ad02"}, + {file = "bandit-1.7.10.tar.gz", hash = "sha256:59ed5caf5d92b6ada4bf65bc6437feea4a9da1093384445fed4d472acc6cff7b"}, +] + +[package.dependencies] +colorama = {version = ">=0.3.9", markers = "platform_system == \"Windows\""} +PyYAML = ">=5.3.1" +rich = "*" +stevedore = ">=1.20.0" + +[package.extras] +baseline = ["GitPython (>=3.1.30)"] +sarif = ["jschema-to-python (>=1.2.3)", "sarif-om (>=1.0.4)"] +test = ["beautifulsoup4 (>=4.8.0)", "coverage (>=4.5.4)", "fixtures (>=3.0.0)", "flake8 (>=4.0.0)", "pylint (==1.9.4)", "stestr (>=2.5.0)", "testscenarios (>=0.5.0)", "testtools (>=2.3.0)"] +toml = ["tomli (>=1.1.0)"] +yaml = ["PyYAML"] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "dill" +version = "0.3.9" +description = "serialize all of Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, + {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] + +[[package]] +name = "dodgy" +version = "0.2.1" +description = "Dodgy: Searches for dodgy looking lines in Python code" +optional = false +python-versions = "*" +files = [ + {file = "dodgy-0.2.1-py3-none-any.whl", hash = "sha256:51f54c0fd886fa3854387f354b19f429d38c04f984f38bc572558b703c0542a6"}, + {file = "dodgy-0.2.1.tar.gz", hash = "sha256:28323cbfc9352139fdd3d316fa17f325cc0e9ac74438cbba51d70f9b48f86c3a"}, +] + +[[package]] +name = "flake8" +version = "6.1.0" +description = "the modular source code checker: pep8 pyflakes and co" +optional = false +python-versions = ">=3.8.1" +files = [ + {file = "flake8-6.1.0-py2.py3-none-any.whl", hash = "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5"}, + {file = "flake8-6.1.0.tar.gz", hash = "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23"}, +] + +[package.dependencies] +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.11.0,<2.12.0" +pyflakes = ">=3.1.0,<3.2.0" + +[[package]] +name = "flake8-polyfill" +version = "1.0.2" +description = "Polyfill package for Flake8 plugins" +optional = false +python-versions = "*" +files = [ + {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, + {file = "flake8_polyfill-1.0.2-py2.py3-none-any.whl", hash = "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9"}, +] + +[package.dependencies] +flake8 = "*" + +[[package]] +name = "gitdb" +version = "4.0.11" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.43" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] + +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "mypy" +version = "1.13.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, + {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, + {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, + {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, + {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, + {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, + {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, + {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, + {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, + {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, + {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, + {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, + {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, + {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, + {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, + {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pbr" +version = "6.1.0" +description = "Python Build Reasonableness" +optional = false +python-versions = ">=2.6" +files = [ + {file = "pbr-6.1.0-py2.py3-none-any.whl", hash = "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a"}, + {file = "pbr-6.1.0.tar.gz", hash = "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24"}, +] + +[[package]] +name = "pep8-naming" +version = "0.10.0" +description = "Check PEP-8 naming conventions, plugin for flake8" +optional = false +python-versions = "*" +files = [ + {file = "pep8-naming-0.10.0.tar.gz", hash = "sha256:f3b4a5f9dd72b991bf7d8e2a341d2e1aa3a884a769b5aaac4f56825c1763bf3a"}, + {file = "pep8_naming-0.10.0-py2.py3-none-any.whl", hash = "sha256:5d9f1056cb9427ce344e98d1a7f5665710e2f20f748438e308995852cfa24164"}, +] + +[package.dependencies] +flake8-polyfill = ">=1.0.2,<2" + +[[package]] +name = "platformdirs" +version = "4.3.6" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] + +[[package]] +name = "prospector" +version = "1.13.0" +description = "Prospector is a tool to analyse Python code by aggregating the result of other tools." +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "prospector-1.13.0-py3-none-any.whl", hash = "sha256:a4868809ef0e5e4d9837404fa4fd691d8f28621422e24340e2d7c63ad054cf55"}, + {file = "prospector-1.13.0.tar.gz", hash = "sha256:d7061ac2e0ee419266c7b958f59912ed39441769d4029250a7135a6ec7440420"}, +] + +[package.dependencies] +bandit = {version = ">=1.5.1", optional = true, markers = "extra == \"with-bandit\" or extra == \"with_everything\""} +dodgy = ">=0.2.1,<0.3.0" +GitPython = ">=3.1.27,<4.0.0" +mccabe = ">=0.7.0,<0.8.0" +mypy = {version = ">=0.600", optional = true, markers = "extra == \"with-mypy\" or extra == \"with_everything\""} +packaging = "*" +pep8-naming = ">=0.3.3,<=0.10.0" +pycodestyle = ">=2.9.0" +pydocstyle = ">=2.0.0" +pyflakes = ">=2.2.0" +pylint = ">=3.0" +pylint-celery = "0.3" +pylint-django = ">=2.6.1" +pylint-flask = "0.6" +PyYAML = "*" +requirements-detector = ">=1.3.2" +setoptconf-tmp = ">=0.3.1,<0.4.0" +toml = ">=0.10.2,<0.11.0" + +[package.extras] +with-bandit = ["bandit (>=1.5.1)"] +with-everything = ["bandit (>=1.5.1)", "mypy (>=0.600)", "pyright (>=1.1.3)", "pyroma (>=2.4)", "ruff", "vulture (>=1.5)"] +with-mypy = ["mypy (>=0.600)"] +with-pyright = ["pyright (>=1.1.3)"] +with-pyroma = ["pyroma (>=2.4)"] +with-ruff = ["ruff"] +with-vulture = ["vulture (>=1.5)"] + +[[package]] +name = "prospector-profile-duplicated" +version = "1.6.0" +description = "Profile that can be used to disable the duplicated or conflict rules between Prospector and other tools" +optional = false +python-versions = "*" +files = [ + {file = "prospector_profile_duplicated-1.6.0-py2.py3-none-any.whl", hash = "sha256:bf6a6aae0c7de48043b95e4d42e23ccd090c6c7115b6ee8c8ca472ffb1a2022b"}, + {file = "prospector_profile_duplicated-1.6.0.tar.gz", hash = "sha256:9c2d541076537405e8b2484cb6222276a2df17492391b6af1b192695770aab83"}, +] + +[[package]] +name = "prospector-profile-utils" +version = "1.9.1" +description = "Some utility Prospector profiles." +optional = false +python-versions = "*" +files = [ + {file = "prospector_profile_utils-1.9.1-py2.py3-none-any.whl", hash = "sha256:b458d8c4d59bdb1547e4630a2c6de4971946c4f0999443db6a9eef6d216b26b8"}, + {file = "prospector_profile_utils-1.9.1.tar.gz", hash = "sha256:008efa6797a85233fd8093dcb9d86f5fa5d89673e431c15cb1496a91c9b2c601"}, +] + +[[package]] +name = "pycodestyle" +version = "2.11.1" +description = "Python style guide checker" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, +] + +[[package]] +name = "pydocstyle" +version = "6.3.0" +description = "Python docstring style checker" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pydocstyle-6.3.0-py3-none-any.whl", hash = "sha256:118762d452a49d6b05e194ef344a55822987a462831ade91ec5c06fd2169d019"}, + {file = "pydocstyle-6.3.0.tar.gz", hash = "sha256:7ce43f0c0ac87b07494eb9c0b462c0b73e6ff276807f204d6b53edc72b7e44e1"}, +] + +[package.dependencies] +snowballstemmer = ">=2.2.0" + +[package.extras] +toml = ["tomli (>=1.2.3)"] + +[[package]] +name = "pyflakes" +version = "3.1.0" +description = "passive checker of Python programs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyflakes-3.1.0-py2.py3-none-any.whl", hash = "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774"}, + {file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"}, +] + +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pylint" +version = "3.3.1" +description = "python code static checker" +optional = false +python-versions = ">=3.9.0" +files = [ + {file = "pylint-3.3.1-py3-none-any.whl", hash = "sha256:2f846a466dd023513240bc140ad2dd73bfc080a5d85a710afdb728c420a5a2b9"}, + {file = "pylint-3.3.1.tar.gz", hash = "sha256:9f3dcc87b1203e612b78d91a896407787e708b3f189b5fa0b307712d49ff0c6e"}, +] + +[package.dependencies] +astroid = ">=3.3.4,<=3.4.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, +] +isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" +mccabe = ">=0.6,<0.8" +platformdirs = ">=2.2.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" + +[package.extras] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] + +[[package]] +name = "pylint-celery" +version = "0.3" +description = "pylint-celery is a Pylint plugin to aid Pylint in recognising and understandingerrors caused when using the Celery library" +optional = false +python-versions = "*" +files = [ + {file = "pylint-celery-0.3.tar.gz", hash = "sha256:41e32094e7408d15c044178ea828dd524beedbdbe6f83f712c5e35bde1de4beb"}, +] + +[package.dependencies] +astroid = ">=1.0" +pylint = ">=1.0" +pylint-plugin-utils = ">=0.2.1" + +[[package]] +name = "pylint-django" +version = "2.6.1" +description = "A Pylint plugin to help Pylint understand the Django web framework" +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "pylint-django-2.6.1.tar.gz", hash = "sha256:19e8c85a8573a04e3de7be2ba91e9a7c818ebf05e1b617be2bbae67a906b725f"}, + {file = "pylint_django-2.6.1-py3-none-any.whl", hash = "sha256:359f68fe8c810ee6bc8e1ab4c83c19b15a43b234a24b08978f47a23462b5ce28"}, +] + +[package.dependencies] +pylint = ">=3.0,<4" +pylint-plugin-utils = ">=0.8" + +[package.extras] +with-django = ["Django (>=2.2)"] + +[[package]] +name = "pylint-flask" +version = "0.6" +description = "pylint-flask is a Pylint plugin to aid Pylint in recognizing and understanding errors caused when using Flask" +optional = false +python-versions = "*" +files = [ + {file = "pylint-flask-0.6.tar.gz", hash = "sha256:f4d97de2216bf7bfce07c9c08b166e978fe9f2725de2a50a9845a97de7e31517"}, +] + +[package.dependencies] +pylint-plugin-utils = ">=0.2.1" + +[[package]] +name = "pylint-plugin-utils" +version = "0.8.2" +description = "Utilities and helpers for writing Pylint plugins" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "pylint_plugin_utils-0.8.2-py3-none-any.whl", hash = "sha256:ae11664737aa2effbf26f973a9e0b6779ab7106ec0adc5fe104b0907ca04e507"}, + {file = "pylint_plugin_utils-0.8.2.tar.gz", hash = "sha256:d3cebf68a38ba3fba23a873809155562571386d4c1b03e5b4c4cc26c3eee93e4"}, +] + +[package.dependencies] +pylint = ">=1.7" + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "requirements-detector" +version = "1.3.2" +description = "Python tool to find and list requirements of a Python project" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "requirements_detector-1.3.2-py3-none-any.whl", hash = "sha256:e7595a32a21e5273dd54d3727bfef4591bbb96de341f6d95c9671981440876ee"}, + {file = "requirements_detector-1.3.2.tar.gz", hash = "sha256:af5a3ea98ca703d14cf7b66751b2aeb3656d02d9e5fc1c97d7d4da02b057b601"}, +] + +[package.dependencies] +astroid = ">=3.0,<4.0" +packaging = ">=21.3" +semver = ">=3.0.0,<4.0.0" +toml = {version = ">=0.10.2,<0.11.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "rich" +version = "13.9.4" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, + {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "semver" +version = "3.0.2" +description = "Python helper for Semantic Versioning (https://semver.org)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, +] + +[[package]] +name = "setoptconf-tmp" +version = "0.3.1" +description = "A module for retrieving program settings from various sources in a consistant method." +optional = false +python-versions = "*" +files = [ + {file = "setoptconf-tmp-0.3.1.tar.gz", hash = "sha256:e0480addd11347ba52f762f3c4d8afa3e10ad0affbc53e3ffddc0ca5f27d5778"}, + {file = "setoptconf_tmp-0.3.1-py3-none-any.whl", hash = "sha256:76035d5cd1593d38b9056ae12d460eca3aaa34ad05c315b69145e138ba80a745"}, +] + +[package.extras] +yaml = ["pyyaml"] + +[[package]] +name = "smmap" +version = "5.0.1" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, +] + +[[package]] +name = "snowballstemmer" +version = "2.2.0" +description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +optional = false +python-versions = "*" +files = [ + {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, + {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, +] + +[[package]] +name = "stevedore" +version = "5.3.0" +description = "Manage dynamic plugins for Python applications" +optional = false +python-versions = ">=3.8" +files = [ + {file = "stevedore-5.3.0-py3-none-any.whl", hash = "sha256:1efd34ca08f474dad08d9b19e934a22c68bb6fe416926479ba29e5013bcc8f78"}, + {file = "stevedore-5.3.0.tar.gz", hash = "sha256:9a64265f4060312828151c204efbe9b7a9852a0d9228756344dbc7e4023e375a"}, +] + +[package.dependencies] +pbr = ">=2.0.0" + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "tomli" +version = "2.0.2" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, +] + +[[package]] +name = "tomlkit" +version = "0.13.2" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[metadata] +lock-version = "2.0" +python-versions = ">=3.10,<4.0" +content-hash = "6a5bccbc4947bb07c85a8d577629ffe2cfe84a72140d387c81a695013c84fbc9" diff --git a/acceptance_tests/waitress_app/production.ini b/acceptance_tests/waitress_app/production.ini new file mode 100644 index 000000000..595b7c549 --- /dev/null +++ b/acceptance_tests/waitress_app/production.ini @@ -0,0 +1,104 @@ +### +# Application configuration +# http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/environment.html +### + +[app:app] +use = egg:c2cwsgiutils_app +filter-with = proxy-prefix + +pyramid.reload_templates = %(DEVELOPMENT)s +pyramid.debug_authorization = %(DEVELOPMENT)s +pyramid.debug_notfound = %(DEVELOPMENT)s +pyramid.debug_routematch = %(DEVELOPMENT)s + +pyramid.default_locale_name = en + +sqlalchemy.url = %(SQLALCHEMY_URL)s +sqlalchemy.pool_recycle = %(SQLALCHEMY_POOL_RECYCLE)s +sqlalchemy.pool_size = %(SQLALCHEMY_POOL_SIZE)s +sqlalchemy.max_overflow = %(SQLALCHEMY_MAX_OVERFLOW)s + +sqlalchemy_slave.url = %(SQLALCHEMY_SLAVE_URL)s +sqlalchemy_slave.pool_recycle = %(SQLALCHEMY_SLAVE_POOL_RECYCLE)s +sqlalchemy_slave.pool_size = %(SQLALCHEMY_SLAVE_POOL_SIZE)s +sqlalchemy_slave.max_overflow = %(SQLALCHEMY_SLAVE_MAX_OVERFLOW)s + +c2c.sql_request_id = True +c2c.requests_default_timeout = 2 + +[filter:proxy-prefix] +use = egg:PasteDeploy#prefix +prefix = %(VISIBLE_ENTRY_POINT)s + +[filter:translogger] +use = egg:Paste#translogger +setup_console_handler = False + +[pipeline:main] +pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry translogger app + +[server:main] +use = egg:waitress#main +listen = *:8080 +threads = %(WAITRESS_THREADS)s +trusted_proxy = True +clear_untrusted_proxy_headers = False + +### +# Logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, waitress, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app + +[handlers] +keys = console, json + +[formatters] +keys = generic + +[logger_root] +level = %(OTHER_LOG_LEVEL)s +handlers = %(LOG_TYPE)s + +[logger_waitress] +level = %(WAITRESS_LOG_LEVEL)s +handlers = +qualname = waitress + +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARNING" logs neither. (Recommended for production systems.) +[logger_sqlalchemy] +level = %(SQL_LOG_LEVEL)s +handlers = +qualname = sqlalchemy.engine + +[logger_c2cwsgiutils] +level = %(C2CWSGIUTILS_LOG_LEVEL)s +handlers = +qualname = c2cwsgiutils + +[logger_c2cwsgiutils_app] +level = %(LOG_LEVEL)s +handlers = +qualname = c2cwsgiutils_app + +[handler_console] +class = logging.StreamHandler +kwargs = {'stream': 'ext://sys.stdout'} +level = NOTSET +formatter = generic + +[handler_json] +class = c2cwsgiutils.pyramid_logging.JsonLogHandler +kwargs = {'stream': 'ext://sys.stdout'} +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s %(name)s %(message)s +datefmt = [%Y-%m-%d %H:%M:%S %z] +class = logging.Formatter diff --git a/acceptance_tests/waitress_app/pyproject.toml b/acceptance_tests/waitress_app/pyproject.toml new file mode 100644 index 000000000..8061364f1 --- /dev/null +++ b/acceptance_tests/waitress_app/pyproject.toml @@ -0,0 +1,48 @@ +[tool.black] +line-length = 110 +target-version = ["py39"] + +[tool.isort] +profile = "black" +line_length = 110 +known_local_folder = ["c2cwsgiutils_app"] + +[tool.mypy] +python_version = 3.9 +ignore_missing_imports = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = true +strict = true + +[tool.poetry] +name = "c2cwsgiutils_app" +version = "0.1.0" +description = "Test application for c2cwsgiutils" +authors = ["Camptocamp "] +classifiers = [ + "Programming Language :: Python", + "Framework :: Pyramid", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", +] +packages = [{ include = "c2cwsgiutils_app" }] + +[tool.poetry.scripts] +get-hello = "c2cwsgiutils_app.get_hello:main" + +[tool.poetry.plugins."paste.app_factory"] +main = "c2cwsgiutils_app:main" + +[tool.poetry.dependencies] +python = ">=3.10,<4.0" + +[tool.poetry.dev-dependencies] +# pylint = { version = "2.15.6" } +prospector = { extras = ["with_bandit", "with_mypy"], version = "1.13.0" } +prospector-profile-duplicated = "1.6.0" +prospector-profile-utils = "1.9.1" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/acceptance_tests/waitress_app/requirements.txt b/acceptance_tests/waitress_app/requirements.txt new file mode 100644 index 000000000..b13e8a06d --- /dev/null +++ b/acceptance_tests/waitress_app/requirements.txt @@ -0,0 +1,2 @@ +poetry==1.8.4 +pip==24.3.1 diff --git a/acceptance_tests/waitress_app/run-alembic b/acceptance_tests/waitress_app/run-alembic new file mode 100755 index 000000000..d17f1d8be --- /dev/null +++ b/acceptance_tests/waitress_app/run-alembic @@ -0,0 +1,24 @@ +#!/bin/bash +# Upgrade the DB +set -eu + +# wait for the DB to be UP +while ! echo "import sqlalchemy; sqlalchemy.create_engine('${SQLALCHEMY_URL}').connect()" | python3 2> /dev/null; do + echo "Waiting for the DB to be reachable" + sleep 1 +done + +for ini in *alembic*.ini; do + if [[ -f "${ini}" ]]; then + echo "${ini} ===========================" + echo History + alembic -c "${ini}" history + echo Head + alembic -c "${ini}" head + echo Upgrade + alembic -c "${ini}" upgrade head + echo Current + alembic -c "${ini}" current + echo "===========================" + fi +done diff --git a/acceptance_tests/waitress_app/scripts/wait-db b/acceptance_tests/waitress_app/scripts/wait-db new file mode 100755 index 000000000..5d7f29076 --- /dev/null +++ b/acceptance_tests/waitress_app/scripts/wait-db @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023-2024, Camptocamp SA +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + + +import sys +import time + +import psycopg2 + +sleep_time = 1 +max_sleep = 60 + +if __name__ == "__main__": + # wait for the WSGI to be UP + nb_success = 0 + for db in ("db", "db_slave"): + while sleep_time < max_sleep: + print(f"Waiting for the DataBase server '{db}' to be reachable") + try: + connection = psycopg2.connect(host=db, password="www-data", user="www-data", database="test") + cursor = connection.cursor() + cursor.execute("SELECT 1") + nb_success += 1 + break + except Exception as e: + print(str(e)) + time.sleep(sleep_time) + sleep_time *= 2 + sys.exit(0 if nb_success == 2 else 1) diff --git a/c2cwsgiutils/prometheus.py b/c2cwsgiutils/prometheus.py index 5b654eade..6e849524a 100644 --- a/c2cwsgiutils/prometheus.py +++ b/c2cwsgiutils/prometheus.py @@ -2,6 +2,7 @@ import os import re +import resource from collections.abc import Generator, Iterable from typing import Any, Optional, TypedDict, cast @@ -15,6 +16,12 @@ from c2cwsgiutils import broadcast, redis_utils from c2cwsgiutils.debug.utils import dump_memory_maps +PSUTILS = True +try: + import psutil +except ImportError: + PSUTILS = False + _NUMBER_RE = re.compile(r"^[0-9]+$") MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [ "c2cwsgiutils_prometheus_collector_gc", @@ -22,12 +29,21 @@ ] +def start_single_process() -> None: + """Start separate HTTP server to provide the Prometheus metrics.""" + if os.environ.get("C2C_PROMETHEUS_PORT") is not None: + prometheus_client.REGISTRY.register(ResourceCollector()) + prometheus_client.REGISTRY.register(MemoryInfoCollector()) + prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"])) + + def start(registry: Optional[prometheus_client.CollectorRegistry] = None) -> None: """Start separate HTTP server to provide the Prometheus metrics.""" if os.environ.get("C2C_PROMETHEUS_PORT") is not None: broadcast.includeme() registry = prometheus_client.CollectorRegistry() if registry is None else registry + registry.register(ResourceCollector()) registry.register(MemoryMapCollector()) registry.register(prometheus_client.PLATFORM_COLLECTOR) registry.register(MultiProcessCustomCollector()) @@ -179,3 +195,43 @@ def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, N for e in dump_memory_maps(pid): gauge.add_metric([pid, e["name"]], e[self.memory_type + "_kb"] * 1024) yield gauge + + +class ResourceCollector(prometheus_client.registry.Collector): + """Collect the resources used by Python.""" + + def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]: + """Get the gauge from smap file.""" + gauge = prometheus_client.core.GaugeMetricFamily( + build_metric_name("python_resource"), + "Python resources", + labels=["name"], + ) + r = resource.getrusage(resource.RUSAGE_SELF) + for field in dir(r): + if field.startswith("ru_"): + gauge.add_metric([field[3:]], getattr(r, field)) + yield gauge + + +class MemoryInfoCollector(prometheus_client.registry.Collector): + """Collect the resources used by Python.""" + + process = psutil.Process(os.getpid()) + + def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]: + """Get the gauge from smap file.""" + gauge = prometheus_client.core.GaugeMetricFamily( + build_metric_name("python_memory_info"), + "Python memory info", + labels=["name"], + ) + memory_info = self.process.memory_info() + gauge.add_metric(["rss"], memory_info.rss) + gauge.add_metric(["vms"], memory_info.vms) + gauge.add_metric(["shared"], memory_info.shared) + gauge.add_metric(["text"], memory_info.text) + gauge.add_metric(["lib"], memory_info.lib) + gauge.add_metric(["data"], memory_info.data) + gauge.add_metric(["dirty"], memory_info.dirty) + yield gauge diff --git a/poetry.lock b/poetry.lock index fe6e43b75..911d9192c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1338,6 +1338,24 @@ files = [ {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "paste" +version = "3.10.1" +description = "Tools for using a Web Server Gateway Interface stack" +optional = true +python-versions = ">=3" +files = [ + {file = "Paste-3.10.1-py3-none-any.whl", hash = "sha256:995e9994b6a94a2bdd8bd9654fb70ca3946ffab75442468bacf31b4d06481c3d"}, + {file = "paste-3.10.1.tar.gz", hash = "sha256:1c3d12065a5e8a7a18c0c7be1653a97cf38cc3e9a5a0c8334a9dd992d3a05e4a"}, +] + +[package.dependencies] +setuptools = "*" + +[package.extras] +flup = ["flup"] +openid = ["python-openid"] + [[package]] name = "pastedeploy" version = "3.1.0" @@ -2993,18 +3011,19 @@ test = ["zope.testing"] [extras] alembic = ["alembic"] -all = ["SQLAlchemy", "SQLAlchemy-Utils", "alembic", "boltons", "cornice", "gunicorn", "lxml", "objgraph", "prometheus-client", "psutil", "psycopg2", "pyjwt", "pyramid", "pyramid-tm", "pyramid_mako", "redis", "requests-oauthlib", "sentry-sdk", "waitress", "zope.interface", "zope.sqlalchemy"] +all = ["Paste", "SQLAlchemy", "SQLAlchemy-Utils", "alembic", "boltons", "cornice", "gunicorn", "lxml", "objgraph", "prometheus-client", "psutil", "psycopg2", "pyjwt", "pyramid", "pyramid-tm", "pyramid_mako", "redis", "requests-oauthlib", "sentry-sdk", "waitress", "zope.interface", "zope.sqlalchemy"] broadcast = ["redis"] debug = ["objgraph", "psutil"] dev = ["waitress"] oauth2 = ["pyjwt", "requests-oauthlib"] sentry = ["sentry-sdk"] -standard = ["SQLAlchemy", "SQLAlchemy-Utils", "alembic", "cornice", "gunicorn", "prometheus-client", "psycopg2", "pyjwt", "pyramid", "pyramid-tm", "pyramid_mako", "redis", "requests-oauthlib", "sentry-sdk", "zope.interface", "zope.sqlalchemy"] +standard = ["Paste", "SQLAlchemy", "SQLAlchemy-Utils", "alembic", "cornice", "gunicorn", "prometheus-client", "psycopg2", "pyjwt", "pyramid", "pyramid-tm", "pyramid_mako", "redis", "requests-oauthlib", "sentry-sdk", "waitress", "zope.interface", "zope.sqlalchemy"] test-images = ["scikit-image"] tests = ["boltons", "lxml"] +waitress = ["Paste", "SQLAlchemy", "SQLAlchemy-Utils", "cornice", "prometheus-client", "psycopg2", "pyramid", "pyramid-tm", "pyramid_mako", "waitress", "zope.interface", "zope.sqlalchemy"] webserver = ["SQLAlchemy", "SQLAlchemy-Utils", "cornice", "gunicorn", "prometheus-client", "psycopg2", "pyramid", "pyramid-tm", "pyramid_mako", "zope.interface", "zope.sqlalchemy"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "a7a7c5a95650883dd8ab72624f370b3b8c6dafae7227e0b88389879b0fb46851" +content-hash = "7b3d63162c2e3b5ca5212808d10352cdd09d187061142810c1760b6b058ae843" diff --git a/pyproject.toml b/pyproject.toml index 7160bcfb8..622030fe6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ scikit-image = { version = "0.24.0", optional = true } prometheus-client = { version = "0.21.0", optional = true} pyramid_mako = { version = "1.1.0", optional = true} psutil = { version = "6.1.0", optional = true} +Paste = { version = "3.10.1", optional = true} [tool.poetry.extras] standard = [ @@ -117,6 +118,9 @@ standard = [ "zope.sqlalchemy", "prometheus_client", "pyramid_mako", + # waitress + "waitress", + "Paste", ] alembic = ["alembic"] debug = ["objgraph", "psutil"] @@ -136,6 +140,19 @@ webserver = [ "zope.sqlalchemy", "prometheus_client", "pyramid_mako"] +waitress = [ + "waitress", + "Paste", + "cornice", + "psycopg2", + "pyramid", + "pyramid-tm", + "SQLAlchemy", + "SQLAlchemy-Utils", + "zope.interface", + "zope.sqlalchemy", + "prometheus_client", + "pyramid_mako"] tests = ["lxml", "boltons"] all = [ # alembic @@ -148,8 +165,6 @@ all = [ "requests-oauthlib", # sentry "sentry-sdk", - # dev - "waitress", # broadcast "redis", # webserver @@ -167,6 +182,9 @@ all = [ # tests "lxml", "boltons", + # waitress + "waitress", + "Paste", ] test_images = ["scikit-image"]