Merge pull request #25 from kozalosev/issue/11-metrics

Add Prometheus counter for chosen inline result
[regex] support of crypto assets with ticker of 4+ chars
Simplify import paths
[tests] fix time zone in mock data

app/bot.py

@@ -3,13 +3,13 @@
 import os
 import asyncio
 from aiohttp import web
-from aiotg import Bot, Chat, InlineQuery, CallbackQuery
+from aiotg import Bot, Chat, InlineQuery, CallbackQuery, ChosenInlineResult
 from klocmod import LocalizationsContainer
 
 import msgdb
 import strconv
 from strconv.currates import update_rates_async_loop
-from txtproc import TextProcessorsLoader, TextProcessor
+from txtproc import TextProcessorsLoader, TextProcessor, metrics
 from txtprocutil import resolve_text_processor_name
 from data.config import *
 from data.currates_conf import EXCHANGE_RATE_SOURCES
@@ -21,7 +21,9 @@
 logging.basicConfig(level=logging.DEBUG if DEBUG else logging.INFO)
 bot = Bot(api_token=TOKEN, default_in_groups=True)
 localizations = LocalizationsContainer.from_file("app/localizations.ini")
+
 text_processors = TextProcessorsLoader(strconv)
+metrics.register(*text_processors.all_processors)
 
 async_tasks = [
     update_rates_async_loop(EXCHANGE_RATE_SOURCES)
@@ -59,7 +61,8 @@ def transform_query(transformer: TextProcessor, **kwargs):
         description = transformer.get_description(request.query, lang_code)
         parse_mode = "HTML" if transformer.use_html else ""
         localized_transformer_name = resolve_text_processor_name(transformer, lang)
-        add_article(localized_transformer_name, processed_str, description, parse_mode, **kwargs)
+        add_article(localized_transformer_name, processed_str, description, parse_mode,
+                    transformer.snake_case_name, **kwargs)
 
     exclusive_processors = text_processors.match_exclusive_processors(request.query)
     if exclusive_processors:
@@ -89,7 +92,14 @@ def decrypt(_, callback_query: CallbackQuery) -> None:
     callback_query.answer(text=message, cache_time=DECRYPT_BUTTON_CACHE_TIME)
 
 
+@bot.chosen_inline_result_callback
+def chosen_inline_result_callback(chosen_result: ChosenInlineResult) -> None:
+    metrics.inc(chosen_result.result_id)
+
+
 if __name__ == '__main__':
+    metrics.serve(METRICS_PORT)
+
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
     for x in async_tasks:

app/queryutil.py

@@ -15,13 +15,14 @@ def __init__(self):
         self._list = []
 
     # noinspection PyShadowingBuiltins
-    def add(self, type: str, **kwargs) -> "InlineQueryResultsBuilder":
+    def add(self, type: str, result_id: str = "", **kwargs) -> "InlineQueryResultsBuilder":
         """Append a new result to the internal list. Can be used in chains of methods.
         :param type: see https://core.telegram.org/bots/api#inlinequeryresult
+        :param result_id: ``id`` of the result will be either an index number or a f"{index_number}:{id}" string
         """
         obj = kwargs
         obj['type'] = type
-        obj['id'] = str(self._id)
+        obj['id'] = f"{self._id}:{result_id}" if result_id else str(self._id)
         self._id += 1
         self._list.append(obj)
         return self
@@ -44,14 +45,16 @@ def get_articles_generator_for(storage: InlineQueryResultsBuilder, max_descripti
     :return: a function `(title: str, text: str, **kwargs) -> None` which you should use to add new articles to the
         storage
     """
-    def add_article(title: str, text: str, description: str = None, parse_mode: str = "", **kwargs) -> None:
+    def add_article(title: str, text: str, description: str = None, parse_mode: str = "",
+                    article_id: str = "", **kwargs) -> None:
         if not description:
             if len(text) > max_description:
                 description = text[:max_description-1].rstrip() + '…'
             else:
                 description = text
         storage.add(
             type='article',
+            result_id=article_id,
             title=title,
             description=description,
             input_message_content={

app/strconv/calc.py

@@ -4,12 +4,12 @@
 import logging
 from io import StringIO
 from txtproc.abc import TextProcessor
-from strconv import currates
+from . import currates
 
 _subst_re = re.compile(r"\{\{(?P<expr>[0-9+\-*/%^., ]+?) *?"
-                       r"((?P<from_curr>[A-Z]{3}|[$€₽£¥]) *?"
+                       r"((?P<from_curr>[A-Z]{3,}|[$€₽£¥]) *?"
                        r"(to|>) *?"
-                       r"(?P<to_curr>[A-Z]{3}|[$€₽£¥]))?? *?}}")
+                       r"(?P<to_curr>[A-Z]{3,}|[$€₽£¥]))?? *?}}")
 _logger = logging.getLogger(__file__)
 _MAX_RECURSION = 100
 

app/strconv/currates/__init__.py

@@ -14,8 +14,8 @@
 from functools import reduce
 from typing import List, Iterable
 
-from strconv.currates.types import *
-from strconv.currates.exceptions import *
+from .types import *
+from .exceptions import *
 
 __all__ = ['update_rates', 'update_rates_async_loop', 'convert']
 

app/txtproc/metrics.py

@@ -0,0 +1,29 @@
+"""Module for management of Prometheus metrics"""
+
+from typing import Dict
+from prometheus_client import start_http_server, Counter
+
+from .abc import TextProcessor
+
+_COUNTER_PREFIX = "used_processor_"
+_counters: Dict[str, Counter] = {}
+
+
+def register(*processors: TextProcessor) -> None:
+    """Registers counter metrics for all processors"""
+    for proc in processors:
+        _counters[proc.snake_case_name] = Counter(_COUNTER_PREFIX + proc.snake_case_name, f"{proc.name} usage")
+
+
+def inc(query_id: str) -> None:
+    """
+    Increments the counter
+    :param query_id: in format ``{index_number}:{proc_name}``
+    """
+    proc_name = "".join(query_id.split(':')[1:])
+    _counters[proc_name].inc()
+
+
+def serve(port: int) -> None:
+    """Runs a WSGI server for metrics"""
+    start_http_server(port)

docker-compose.yml

@@ -7,6 +7,8 @@ services:
     hostname: testUtilsBot
     working_dir: /home/textUtilsBot
     restart: always
+    ports:
+      - 127.0.0.1:8000:8000
     volumes:
     - "./app/data:/home/textUtilsBot/app/data"
     - "/tmp:/tmp"

examples/config.py

@@ -5,6 +5,7 @@
 
 HOST = "bots.example.org"
 SERVER_PORT = 8443                        # A port on a front-end web server.
+METRICS_PORT = 8000
 UNIX_SOCKET = "/tmp/textUtilsBot.sock"    # A Unix domain socket to communicate with that web server.
 
 # Set to 'False' for production use.

examples/nginx-textUtilsBot.conf

@@ -12,6 +12,10 @@ server {
     #access_log /dev/null;
     #error_log /home/username/logs/nginx/textUtilsBot.err.log;
 
+    location = /textUtilsBot/metrics/ {
+        proxy_pass http://127.0.0.1:8000;
+    }
+
     # Ensure the paths are consistent with the NAME and UNIX_SOCKET constants from 'app/data/config.py'.
     location /textUtilsBot/ {
         proxy_pass http://unix:/tmp/textUtilsBot.sock;

requirements.txt

@@ -1,4 +1,5 @@
-aiotg==1.0.0
+git+https://github.com/kozalosev/aiotg@feature/chosen_inline_result#egg=aiotg
 aiohttp==3.8.4
 klocmod==0.3.0
 requests==2.28.2
+prometheus-client==0.16.0

tests/test_currates/test_crypto.py

@@ -3,7 +3,7 @@
 from strconv.currates.extractors import iso_date
 from pathlib import Path
 
-from tests.test_currates import test_fiat
+from . import test_fiat
 
 mock_source = currates.DataSource('mock_source', 'http://localhost/crypto',
                                   status_checker=lambda json: json['status']['error_code'] == 0,
@@ -13,7 +13,7 @@
 mock_eth = 1644.6620025731618
 mock_source_json = f"""{{
     "status": {{
-        "timestamp": "{datetime.date.today().isoformat()}T05:07:57.566Z",
+        "timestamp": "{datetime.datetime.utcnow().date().isoformat()}T05:07:57.566Z",
         "error_code": 0
     }},
     "data": [

tests/test_currates/test_fiat.py

@@ -11,7 +11,7 @@
 mock_source_json = f"""{{
     "success": true,
     "base": "USD",
-    "date": "{datetime.date.today().isoformat()}",
+    "date": "{datetime.datetime.utcnow().date().isoformat()}",
     "rates": {{
         "USD": 1,
         "RUB": {mock_rub},

tests/test_queryutil.py

@@ -7,11 +7,12 @@ def test_answers():
         test=True,
     ).add(
         type='bar',
-        test=0
+        test=0,
+        result_id='foo'
     )
     e = [
         {'type': 'foo', 'id': '0', 'test': True},
-        {'type': 'bar', 'id': '1', 'test': 0}
+        {'type': 'bar', 'id': '1:foo', 'test': 0}
     ]
     assert s.build_list() == e
 
@@ -21,7 +22,7 @@ def test_add_article_to():
     add_article = get_articles_generator_for(a, max_description=10)
     add_article("foo", "test 1")
     add_article("bar", "test long string")
-    add_article("baz", "<b>test 3</b>", description="test 3", parse_mode="HTML")
+    add_article("baz", "<b>test 3</b>", description="test 3", parse_mode="HTML", article_id="foo")
     e = [{
         'type': 'article',
         'id': '0',
@@ -42,7 +43,7 @@ def test_add_article_to():
         }
     }, {
         'type': 'article',
-        'id': '2',
+        'id': '2:foo',
         'title': 'baz',
         'description': 'test 3',
         'input_message_content': {