Add Waitress support for production

Add resource metrics
Add psutil memory info metrics

.pre-commit-config.yaml

@@ -22,9 +22,6 @@ repos:
     rev: 1.1.2
     hooks:
       - id: copyright
-      - id: poetry-check
-        additional_dependencies:
-          - poetry==1.8.4 # pypi
       - id: poetry-lock
         additional_dependencies:
           - poetry==1.8.4 # pypi

Makefile

@@ -60,8 +60,15 @@ build_docker_test:
 	docker build --tag=$(DOCKER_BASE):tests --target=tests .
 
 .PHONY: build_test_app
-build_test_app: build_docker
-	docker build --tag=$(DOCKER_BASE)_test_app --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/app
+build_test_app: build_test_app_gunicorn build_test_app_waitress
+
+.PHONY: build_test_app_gunicorn
+build_test_app_gunicorn: build_docker
+	docker build --tag=$(DOCKER_BASE)_test_app --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/app_gunicorn
+
+.PHONY: build_test_app_waitress
+build_test_app_waitress: build_docker
+	docker build --tag=$(DOCKER_BASE)_test_app_waitress --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/app_waitress
 
 .PHONY: checks
 checks: prospector ## Run the checks

acceptance_tests/gunicorn_app/Dockerfile

@@ -0,0 +1,69 @@
+FROM camptocamp/c2cwsgiutils as base-all
+LABEL maintainer Camptocamp "info@camptocamp.com"
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+
+# Used to convert the locked packages by poetry to pip requirements format
+# We don't directly use `poetry install` because it force to use a virtual environment.
+FROM base-all as poetry
+
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+  --mount=type=cache,target=/var/cache,sharing=locked \
+  apt-get update \
+  && apt-get install --assume-yes --no-install-recommends python-is-python3
+
+# Install Poetry
+WORKDIR /tmp
+COPY requirements.txt ./
+RUN python3 -m pip install --disable-pip-version-check --requirement=requirements.txt
+
+# Do the conversion
+COPY poetry.lock pyproject.toml ./
+RUN poetry export --output=requirements.txt \
+  && poetry export --with=dev --output=requirements-dev.txt
+
+# Base, the biggest thing is to install the Python packages
+FROM base-all as base
+
+WORKDIR /app
+
+EXPOSE 8080
+RUN --mount=type=cache,target=/root/.cache \
+  --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
+  python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt
+
+COPY . /app
+
+ARG GIT_HASH
+
+RUN --mount=type=cache,target=/root/.cache \
+  python3 -m pip install --disable-pip-version-check --no-deps --editable=. \
+  && python3 -m pip freeze > /requirements.txt
+RUN ./models_graph.py > models.dot \
+  && ./models_graph.py Hello > models-hello.dot \
+  && c2cwsgiutils-genversion $GIT_HASH \
+  && python3 -m compileall -q .
+
+ENV \
+  DOCKER_RUN=1 \
+  DEVELOPMENT=0 \
+  SQLALCHEMY_POOL_RECYCLE=30 \
+  SQLALCHEMY_POOL_SIZE=5 \
+  SQLALCHEMY_MAX_OVERFLOW=25 \
+  SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
+  SQLALCHEMY_SLAVE_POOL_SIZE=5 \
+  SQLALCHEMY_SLAVE_MAX_OVERFLOW=25 \
+  LOG_TYPE=console \
+  OTHER_LOG_LEVEL=WARNING \
+  GUNICORN_LOG_LEVEL=WARNING \
+  SQL_LOG_LEVEL=WARNING \
+  C2CWSGIUTILS_LOG_LEVEL=WARNING \
+  LOG_LEVEL=INFO
+
+# www-data
+USER 33
+
+RUN mkdir -p /prometheus-metrics \
+    && chmod a+rwx /prometheus-metrics
+ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+
+CMD ["/venv/bin/gunicorn", "--paste=/app/application.ini"]

acceptance_tests/gunicorn_app/application.ini

@@ -0,0 +1,105 @@
+###
+# app configuration
+# http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/environment.html
+# this file should be used by gunicorn.
+###
+
+[app:app]
+use = egg:c2cwsgiutils_app
+filter-with = proxy-prefix
+
+pyramid.reload_templates = %(DEVELOPMENT)s
+pyramid.debug_authorization = %(DEVELOPMENT)s
+pyramid.debug_notfound = %(DEVELOPMENT)s
+pyramid.debug_routematch = %(DEVELOPMENT)s
+
+pyramid.default_locale_name = en
+
+sqlalchemy.url = %(SQLALCHEMY_URL)s
+sqlalchemy.pool_recycle = %(SQLALCHEMY_POOL_RECYCLE)s
+sqlalchemy.pool_size = %(SQLALCHEMY_POOL_SIZE)s
+sqlalchemy.max_overflow = %(SQLALCHEMY_MAX_OVERFLOW)s
+
+sqlalchemy_slave.url = %(SQLALCHEMY_SLAVE_URL)s
+sqlalchemy_slave.pool_recycle = %(SQLALCHEMY_SLAVE_POOL_RECYCLE)s
+sqlalchemy_slave.pool_size = %(SQLALCHEMY_SLAVE_POOL_SIZE)s
+sqlalchemy_slave.max_overflow = %(SQLALCHEMY_SLAVE_MAX_OVERFLOW)s
+
+c2c.sql_request_id = True
+c2c.requests_default_timeout = 2
+
+[filter:proxy-prefix]
+use = egg:PasteDeploy#prefix
+prefix = %(VISIBLE_ENTRY_POINT)s
+
+[filter:translogger]
+use = egg:Paste#translogger
+setup_console_handler = False
+
+[pipeline:main]
+pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry translogger app
+
+[server:main]
+use = egg:waitress#main
+listen = *:8080
+threads = %(WAITRESS_THREADS)s
+trusted_proxy = True
+clear_untrusted_proxy_headers = False
+
+###
+# Logging configuration
+# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, guwaitressicorn, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
+
+[handlers]
+keys = console, json
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = %(OTHER_LOG_LEVEL)s
+handlers = %(LOG_TYPE)s
+
+[logger_waitress]
+level = %(WAITRESS_LOG_LEVEL)s
+handlers =
+qualname = waitress
+
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither.  (Recommended for production systems.)
+[logger_sqlalchemy]
+level = %(SQL_LOG_LEVEL)s
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_c2cwsgiutils]
+level = %(C2CWSGIUTILS_LOG_LEVEL)s
+handlers =
+qualname = c2cwsgiutils
+
+[logger_c2cwsgiutils_app]
+level = %(LOG_LEVEL)s
+handlers =
+qualname = c2cwsgiutils_app
+
+[handler_console]
+class = logging.StreamHandler
+kwargs = {'stream': 'ext://sys.stdout'}
+level = NOTSET
+formatter = generic
+
+[handler_json]
+class = c2cwsgiutils.pyramid_logging.JsonLogHandler
+kwargs = {'stream': 'ext://sys.stdout'}
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s %(name)s %(message)s
+datefmt = [%Y-%m-%d %H:%M:%S %z]
+class = logging.Formatter

acceptance_tests/app/c2cwsgiutils_app/__init__.py → acceptance_tests/gunicorn_app/c2cwsgiutils_app/__init__.py

@@ -1,11 +1,12 @@
-from c2cwsgiutils_app import models
 from pyramid.config import Configurator
 from pyramid.httpexceptions import HTTPInternalServerError
 
 import c2cwsgiutils.pyramid
 from c2cwsgiutils import broadcast, db
 from c2cwsgiutils.health_check import HealthCheck, JsonCheckException
 
+from c2cwsgiutils_app import models
+
 
 def _failure(_request):
     raise HTTPInternalServerError("failing check")

acceptance_tests/app/c2cwsgiutils_app/get_hello.py → acceptance_tests/gunicorn_app/c2cwsgiutils_app/get_hello.py

@@ -2,11 +2,12 @@
 
 import psycopg2
 import transaction
-from c2cwsgiutils_app import models
 
 import c2cwsgiutils.db
 import c2cwsgiutils.setup_process
 
+from c2cwsgiutils_app import models
+
 
 def _fill_db():
     for db, value in (("db", "master"), ("db_slave", "slave")):

acceptance_tests/app/c2cwsgiutils_app/services.py → acceptance_tests/gunicorn_app/c2cwsgiutils_app/services.py

@@ -2,7 +2,6 @@
 
 import prometheus_client
 import requests
-from c2cwsgiutils_app import models
 from pyramid.httpexceptions import (
     HTTPBadRequest,
     HTTPForbidden,
@@ -13,6 +12,8 @@
 
 from c2cwsgiutils import sentry, services
 
+from c2cwsgiutils_app import models
+
 _PROMETHEUS_TEST_COUNTER = prometheus_client.Counter("test_counter", "Test counter")
 _PROMETHEUS_TEST_GAUGE = prometheus_client.Gauge("test_gauge", "Test gauge", ["value", "toto"])
 _PROMETHEUS_TEST_SUMMARY = prometheus_client.Summary("test_summary", "Test summary")

acceptance_tests/app/models_graph.py → acceptance_tests/gunicorn_app/models_graph.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
-from c2cwsgiutils_app import models
-
 from c2cwsgiutils.models_graph import generate_model_graph
 
+from c2cwsgiutils_app import models
+
 
 def main():
     generate_model_graph(models)

acceptance_tests/app/scripts/wait-db → acceptance_tests/gunicorn_app/scripts/wait-db

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright (c) 2023, Camptocamp SA
+# Copyright (c) 2023-2024, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without

acceptance_tests/tests/docker-compose.override.sample.yaml

@@ -4,20 +4,28 @@ services:
   app:
     # Uncomment to use pserve
     # command:
-    #   - pserve
+    #   - /venv/bin/pserve
     #   - --reload
-    #   - c2c:///app/production.ini
+    #   - c2c:///app/application.ini
     volumes:
       # This mounts the local filesystem inside the container so that
       # the views are automatically reloaded when a file change
       - ../app/c2cwsgiutils_app/:/app/c2cwsgiutils_app/:ro
       - ../../c2cwsgiutils/:/opt/c2cwsgiutils/c2cwsgiutils/:ro
+    environment:
+      - DEVELOPMENT=TRUE
     ports:
       - 9090:9090
 
   app2:
+    # command:
+    #   - /venv/bin/pserve
+    #   - --reload
+    #   - c2c:///app/application.ini
     ports:
       - 9092:9090
+    environment:
+      - DEVELOPMENT=TRUE
 
   db:
     ports:

acceptance_tests/tests/docker-compose.yaml

@@ -34,7 +34,6 @@ services:
       # Test problematic environment variable (values contains % and duplicated with different cass)
       - TEST='%1'
       - test='%2'
-      - PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
       - C2C_PROMETHEUS_PORT=9090
       - C2C_PROMETHEUS_APPLICATION_PACKAGE=c2cwsgiutils_app
     links:
@@ -46,6 +45,7 @@ services:
 
   app2:
     <<: *app
+    image: camptocamp/c2cwsgiutils_test_app_waitress
     # Same as app but with 2 workers (and different Redis DB 2, broadcast_prefix, ports, and JSON log format)
     environment:
       - SQLALCHEMY_URL
@@ -61,7 +61,6 @@ services:
       - C2C_PROFILER_PATH=/api_profiler
       - C2C_PROFILER_MODULES=c2cwsgiutils c2cwsgiutils_app sqlalchemy request
       - C2C_ENABLE_EXCEPTION_HANDLING=1
-      - GUNICORN_CMD_ARGS="--reload" # don't use this in production
       - C2CWSGIUTILS_LOG_LEVEL=DEBUG
       - SQL_LOG_LEVEL=DEBUG
       - OTHER_LOG_LEVEL=INFO
@@ -75,12 +74,9 @@ services:
       - C2C_BROADCAST_PREFIX=app2
       - PYTHONMALLOC=debug
       - DEBUG_LOGCONFIG
-      - GUNICORN_WORKERS=2
-      - GUNICORN_THREADS=10
       # Test problematic environment variable (values contains % and duplicated with different cass)
       - TEST='%1'
       - test='%2'
-      - PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
       - C2C_PROMETHEUS_PORT=9090
       - C2C_PROMETHEUS_APPLICATION_PACKAGES=c2cwsgiutils_app
     ports:

acceptance_tests/tests/tests/conftest.py

@@ -8,7 +8,8 @@
 
 _BASE_URL = "http://app:8080/api/"
 _BASE_URL_APP2 = "http://app2:8080/api/"
-_PROMETHEUS_URL = "http://app2:9090/metrics"
+_PROMETHEUS_URL_1 = "http://app:9090/metrics"
+_PROMETHEUS_URL_2 = "http://app2:9090/metrics"
 _PROMETHEUS_TEST_URL = "http://run_test:9090/metrics"
 _PROMETHEUS_STATS_DB_URL = "http://stats_db:9090/metrics"
 _LOG = logging.getLogger(__name__)
@@ -44,13 +45,23 @@ def app2_connection(composition):
 
 
 @pytest.fixture
-def prometheus_connection(composition):
+def prometheus_1_connection(composition):
     """
     Fixture that returns a connection to a running batch container.
     """
 
     del composition
-    return Connection(base_url=_PROMETHEUS_URL, origin="http://example.com/")
+    return Connection(base_url=_PROMETHEUS_URL_1, origin="http://example.com/")
+
+
+@pytest.fixture
+def prometheus_2_connection(composition):
+    """
+    Fixture that returns a connection to a running batch container.
+    """
+
+    del composition
+    return Connection(base_url=_PROMETHEUS_URL_2, origin="http://example.com/")
 
 
 @pytest.fixture

acceptance_tests/tests/tests/test_prometheus_client.py

@@ -3,8 +3,17 @@
 _PID_RE = re.compile(r',pid="([0-9]+)"')
 
 
-def test_prometheus(prometheus_connection):
+def test_prometheus_1(prometheus_1_connection):
     # One for the root process, one for each workers
     assert (
-        len(set(_PID_RE.findall(prometheus_connection.get("metrics", cache_expected=False, cors=False)))) == 3
+        len(set(_PID_RE.findall(prometheus_1_connection.get("metrics", cache_expected=False, cors=False))))
+        == 3
+    )
+
+
+def test_prometheus_2(prometheus_2_connection):
+    # One for the root process, one for each workers
+    assert (
+        len(set(_PID_RE.findall(prometheus_2_connection.get("metrics", cache_expected=False, cors=False))))
+        == 3
     )