Merge pull request #2610 from camptocamp/backport/2609-to-1.22

[Backport 1.22] Add objectgraph logging

tilecloud_chain/generate.py

@@ -1,5 +1,6 @@
 """Generate the tiles, generate the queue, ..."""
 
+import gc
 import logging
 import os
 import random
@@ -13,6 +14,7 @@
 from typing import IO, cast
 
 import boto3
+import objgraph
 import prometheus_client
 import tilecloud.filter.error
 from tilecloud import Tile, TileCoord, TileStore
@@ -44,6 +46,21 @@
 _LOGGER = logging.getLogger(__name__)
 
 
+def _objgraph(tile: Tile) -> Tile:
+    """Log the objgraph."""
+    for generation in range(3):
+        gc.collect(generation)
+    values = [
+        f"{name}: {number} {diff}"
+        for name, number, diff in objgraph.growth(
+            limit=int(os.environ.get("TILECLOUD_CHAIN_OBJGRAPH_LIMIT", "10"))
+        )
+    ]
+    if values:
+        _LOGGER.debug("Objgraph growth in queue:\n%s", "\n".join(values))
+    return tile
+
+
 class LogTilesContext:
     """Logging tile context."""
 
@@ -111,6 +128,9 @@ def gene(self, layer_name: str | None = None) -> None:
         if self._options.role != "slave" or self._options.tiles:
             self._generate_queue(layer_name)
 
+        if os.environ.get("TILECLOUD_CHAIN_OBJGRAPH_GENE", "0").lower() in ("1", "true", "on"):
+            self._gene.imap(_objgraph)
+
         self.generate_consume()
         self.generate_resume(layer_name)
 

tilecloud_chain/store/postgresql.py

@@ -26,6 +26,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import gc
 import io
 import logging
 import multiprocessing
@@ -38,6 +39,7 @@
 from datetime import datetime, timedelta
 from typing import Any, cast
 
+import objgraph
 import sqlalchemy
 import sqlalchemy.sql.functions
 from prometheus_client import Counter, Gauge, Summary
@@ -525,6 +527,22 @@ def list(self) -> Iterator[Tile]:
                     continue
                 job_id = self.jobs.pop(config_filename)
                 try:
+                    if os.environ.get("TILECLOUD_CHAIN_OBJGRAPH_POSTGRESQL", "0").lower() in (
+                        "1",
+                        "true",
+                        "on",
+                    ):
+                        for generation in range(3):
+                            gc.collect(generation)
+                        values = [
+                            f"{name}: {number} {diff}"
+                            for name, number, diff in objgraph.growth(
+                                limit=int(os.environ.get("TILECLOUD_CHAIN_OBJGRAPH_LIMIT", "10"))
+                            )
+                        ]
+                        if values:
+                            _LOGGER.debug("Objgraph growth in postgresql:\n%s", "\n".join(values))
+
                     with self.SessionMaker() as session:
                         sqlalchemy_tile = (
                             session.query(Queue)