clickstream_events_spec_and_validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "88e7f59f",
   "metadata": {},
   "source": [
    "### Cerberus Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "96a9f5c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cerberus import Validator\n",
    "from datetime import datetime\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b74eefb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "{}\n",
      "---\n",
      "False\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'letter': ['must be of string type'], 'number': ['must be of integer type']}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_scheme = {\n",
    "    'letter': {'type': 'string'},\n",
    "    'number': {'type': 'integer'}\n",
    "}\n",
    "v = Validator(example_scheme, require_all=True)\n",
    "\n",
    "example_event = {\n",
    "    'letter': 'A',\n",
    "    'number': 1\n",
    "}\n",
    "print(v.validate(example_event))\n",
    "print(v.errors)\n",
    "\n",
    "example_event_errors = {\n",
    "    'letter': 1,\n",
    "    'number': 'B'\n",
    "}\n",
    "print('---')\n",
    "print(v.validate(example_event_errors))\n",
    "v.errors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3154d04e",
   "metadata": {},
   "source": [
    "### Single Scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bcb9eff6",
   "metadata": {},
   "outputs": [],
   "source": [
    "payment_btn_click_scheme = {\n",
    "    'dt': {'type': 'datetime'},\n",
    "    'device_id': {'type': 'string'},\n",
    "    'user_id': {'type': 'integer'},\n",
    "    'name': {'type': 'string', 'allowed': ['payment_btn_click']},\n",
    "    'order_id': {'type': 'integer'}\n",
    "}\n",
    "v = Validator(payment_btn_click_scheme, require_all=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67632002",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: ok\n",
      "1: {'order_id': ['required field']}\n",
      "2: {'name': ['unallowed value payment_btn']}\n",
      "3: {'user_id': ['must be of integer type']}\n"
     ]
    }
   ],
   "source": [
    "payment_btn_click_events = [\n",
    "    {\n",
    "        #ok\n",
    "        'dt': datetime.strptime('2023-01-10 12:23:01','%Y-%m-%d %H:%M:%S'),\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn_click',\n",
    "        'order_id': 101010\n",
    "    },\n",
    "    {\n",
    "        #missing order_id\n",
    "        'dt': datetime.strptime('2023-01-10 12:23:01','%Y-%m-%d %H:%M:%S'),\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn_click'\n",
    "    },\n",
    "    {\n",
    "        #wrong event name\n",
    "        'dt': datetime.strptime('2023-01-10 12:23:01','%Y-%m-%d %H:%M:%S'),\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn',\n",
    "        'order_id': 101010\n",
    "    },\n",
    "    {\n",
    "        #wrong user_id type\n",
    "        'dt': datetime.strptime('2023-01-10 12:23:01','%Y-%m-%d %H:%M:%S'),\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'order_id': 101010\n",
    "    }\n",
    "]\n",
    "\n",
    "for i,e in enumerate(payment_btn_click_events):\n",
    "    if v.validate(e):\n",
    "        print(f'{i}: ok')\n",
    "    else:\n",
    "        print(f'{i}: {v.errors}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "141fbc81",
   "metadata": {},
   "source": [
    "### Multiple Schemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7a25c3e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "known_events = ['payment_btn_click', 'main_banner_click']\n",
    "\n",
    "common_fields_scheme = {\n",
    "    'dt': {'type': 'datetime', 'coerce': lambda x: datetime.strptime(x,'%Y-%m-%d %H:%M:%S')},\n",
    "    'device_id': {'type': 'string'},\n",
    "    'user_id': {'type': 'integer', 'nullable': True},\n",
    "    'name': {'type': 'string', 'allowed': known_events}\n",
    "}\n",
    "\n",
    "\n",
    "payment_btn_click_scheme = {\n",
    "    'name': {'type': 'string', 'allowed': ['payment_btn_click']},\n",
    "    'order_id': {'type': 'integer'}\n",
    "}\n",
    "\n",
    "main_banner_click_scheme = {\n",
    "    'name': {'type': 'string', 'allowed': ['main_banner_click']},\n",
    "    'banner_id': {'type': 'integer'}\n",
    "}\n",
    "\n",
    "\n",
    "events_specs = {\n",
    "    'payment_btn_click': payment_btn_click_scheme,\n",
    "    'main_banner_click': main_banner_click_scheme\n",
    "}\n",
    "#common_fields_scheme['name'].update({'type': 'string', 'allowed': list(events_specs.keys())})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "21d964f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payment_btn: {'name': ['unallowed value payment_btn']}\n",
      "payment_btn_click: {'user_id': ['must be of integer type']}\n",
      "main_banner_click: {'user_id': ['must be of integer type']}\n",
      "main_banner_click: {'user_id': ['must be of integer type']}\n",
      "890: {'name': ['must be of string type'], 'user_id': ['must be of integer type']}\n",
      "payment_btn_click, 0: {'order_id': ['required field']}\n",
      "payment_btn_click, 1: {'order_id': ['must be of integer type']}\n",
      "main_banner_click, 0: {'banner_id': ['required field']}\n",
      "main_banner_click, 1: {'banner_id': ['required field']}\n"
     ]
    }
   ],
   "source": [
    "events = [\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn',\n",
    "        'order_id': 101010\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'order_id': '101010',\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 890\n",
    "    }\n",
    "]\n",
    "\n",
    "v = Validator()\n",
    "v.allow_unknown = True\n",
    "v.require_all = True\n",
    "\n",
    "for i,e in enumerate(events):\n",
    "    if not v.validate(e, common_fields_scheme):\n",
    "        print(f'{e.get(\"name\")}: {v.errors}')\n",
    "        \n",
    "for e_name, s in events_specs.items():\n",
    "    filtered_events = filter(lambda x: x.get('name') == e_name, events)\n",
    "    for i, e in enumerate(filtered_events):\n",
    "        if not v.validate(e, s):\n",
    "            print(f'{e_name}, {i}: {v.errors}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bf12855",
   "metadata": {},
   "source": [
    "### Errors Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01f89801",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>processed_events</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>payment_btn</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>missing_name</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               event  processed_events\n",
       "0  payment_btn_click                 2\n",
       "1        payment_btn                 1\n",
       "2  main_banner_click                 2\n",
       "3                890                 1\n",
       "4       missing_name                 1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>field</th>\n",
       "      <th>msg</th>\n",
       "      <th>err_cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>890</td>\n",
       "      <td>name</td>\n",
       "      <td>unallowed value 890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>890</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>banner_id</td>\n",
       "      <td>required field</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>missing_name</td>\n",
       "      <td>name</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>missing_name</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>payment_btn</td>\n",
       "      <td>name</td>\n",
       "      <td>unallowed value payment_btn</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>order_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>order_id</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               event      field                          msg  err_cnt\n",
       "0                890       name          unallowed value 890        1\n",
       "1                890    user_id      must be of integer type        1\n",
       "2  main_banner_click  banner_id               required field        2\n",
       "3  main_banner_click    user_id      must be of integer type        2\n",
       "4       missing_name       name               required field        1\n",
       "5       missing_name    user_id      must be of integer type        1\n",
       "6        payment_btn       name  unallowed value payment_btn        1\n",
       "7  payment_btn_click   order_id      must be of integer type        1\n",
       "8  payment_btn_click   order_id               required field        1\n",
       "9  payment_btn_click    user_id      must be of integer type        1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>field</th>\n",
       "      <th>msg</th>\n",
       "      <th>err_cnt</th>\n",
       "      <th>processed_events</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>890</td>\n",
       "      <td>name</td>\n",
       "      <td>unallowed value 890</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>890</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>banner_id</td>\n",
       "      <td>required field</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>missing_name</td>\n",
       "      <td>name</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>missing_name</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>payment_btn</td>\n",
       "      <td>name</td>\n",
       "      <td>unallowed value payment_btn</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>order_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>order_id</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               event      field                          msg  err_cnt  \\\n",
       "0                890       name          unallowed value 890        1   \n",
       "1                890    user_id      must be of integer type        1   \n",
       "2  main_banner_click  banner_id               required field        2   \n",
       "3  main_banner_click    user_id      must be of integer type        2   \n",
       "4       missing_name       name               required field        1   \n",
       "5       missing_name    user_id      must be of integer type        1   \n",
       "6        payment_btn       name  unallowed value payment_btn        1   \n",
       "7  payment_btn_click   order_id      must be of integer type        1   \n",
       "8  payment_btn_click   order_id               required field        1   \n",
       "9  payment_btn_click    user_id      must be of integer type        1   \n",
       "\n",
       "   processed_events  \n",
       "0                 1  \n",
       "1                 1  \n",
       "2                 2  \n",
       "3                 2  \n",
       "4                 1  \n",
       "5                 1  \n",
       "6                 1  \n",
       "7                 2  \n",
       "8                 2  \n",
       "9                 2  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events = [\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn',\n",
    "        'order_id': 101010\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'order_id': '101010',\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': '890'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "    }    \n",
    "]\n",
    "\n",
    "v = Validator()\n",
    "v.allow_unknown = True\n",
    "v.require_all = True\n",
    "\n",
    "def gather_errors(errs, event, validate_errors):\n",
    "    #todo: use custom error handler?\n",
    "    for field, msgs in validate_errors.items():\n",
    "        for m in msgs:\n",
    "            errs.append([event, field, m])\n",
    "\n",
    "ev_cnt = {}\n",
    "errs = []\n",
    "for e in events:\n",
    "    e_name = e.get(\"name\", \"missing_name\")\n",
    "    v.validate(e, common_fields_scheme)\n",
    "    gather_errors(errs, e_name, v.errors)\n",
    "    s = events_specs.get(e_name)\n",
    "    if s:\n",
    "        v.validate(e, s)\n",
    "        gather_errors(errs, e_name, v.errors)\n",
    "    ev_cnt[e_name] = ev_cnt.get(e_name, 0) + 1\n",
    "\n",
    "df_evcnt = pd.DataFrame({'event': list(ev_cnt.keys()), 'processed_events': list(ev_cnt.values())})\n",
    "display(df_evcnt)\n",
    "\n",
    "df_errs = pd.DataFrame(errs, columns=['event', 'field', 'msg'])\n",
    "df_errs['err_cnt'] = 1\n",
    "df_errs_cnt = df_errs.groupby(['event', 'field', 'msg'])['err_cnt'].sum().reset_index()\n",
    "display(df_errs_cnt)\n",
    "\n",
    "pd.merge(df_errs_cnt, df_evcnt, on='event', how='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51bd0eb5",
   "metadata": {},
   "source": [
    "### Nested Fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a2575118",
   "metadata": {},
   "outputs": [],
   "source": [
    "known_events = ['payment_btn_click', 'main_banner_click']\n",
    "\n",
    "common_fields_scheme = {\n",
    "    'dt': {'type': 'datetime', 'coerce': lambda x: datetime.strptime(x,'%Y-%m-%d %H:%M:%S')},\n",
    "    'device_id': {'type': 'string'},\n",
    "    'user_id': {'type': 'integer', 'nullable': True},\n",
    "    'name': {'type': 'string', 'allowed': known_events},\n",
    "    'params': {'type': 'dict'}\n",
    "}\n",
    "\n",
    "\n",
    "payment_btn_click_params_scheme = {\n",
    "    'order_id': {'type': 'integer'},\n",
    "    'exp_id': {'type': 'integer'}\n",
    "}\n",
    "\n",
    "main_banner_click_params_scheme = {\n",
    "    'banner_id': {'type': 'integer'},\n",
    "    'exp_id': {'type': 'integer'}\n",
    "}\n",
    "\n",
    "\n",
    "event_params_specs = {\n",
    "    'payment_btn_click': payment_btn_click_params_scheme,\n",
    "    'main_banner_click': main_banner_click_params_scheme\n",
    "}\n",
    "\n",
    "#todo: combine common_fields_scheme and paramters_schemes into single schemes?\n",
    "#common_fields_scheme['name'].update({'type': 'string', 'allowed': list(events_specs.keys())})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "124eca96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>field</th>\n",
       "      <th>msg</th>\n",
       "      <th>err_cnt</th>\n",
       "      <th>processed_events</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>params</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>params</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>params.exp_id</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>params.order_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               event            field                      msg  err_cnt  \\\n",
       "0  main_banner_click           params           required field        1   \n",
       "1  main_banner_click          user_id  must be of integer type        2   \n",
       "2  payment_btn_click           params           required field        1   \n",
       "3  payment_btn_click    params.exp_id           required field        1   \n",
       "4  payment_btn_click  params.order_id  must be of integer type        1   \n",
       "5  payment_btn_click          user_id  must be of integer type        2   \n",
       "\n",
       "   processed_events  \n",
       "0                 2  \n",
       "1                 2  \n",
       "2                 3  \n",
       "3                 3  \n",
       "4                 3  \n",
       "5                 3  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events = [\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'params': {\n",
    "            'order_id': '101010'\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'params': {\n",
    "            'order_id': 101010,\n",
    "            'exp_id': 9990\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click',\n",
    "        'params': {\n",
    "            'banner_id': 1789789,\n",
    "            'exp_id': 9990\n",
    "        }\n",
    "    }   \n",
    "]\n",
    "\n",
    "v = Validator()\n",
    "v.allow_unknown = True\n",
    "v.require_all = True\n",
    "\n",
    "def gather_errors(errs, event, validate_errors, position_prefix=''):\n",
    "    #todo: use custom error handler?\n",
    "    #use document_error_tree for nodes location?\n",
    "    for field, msgs in validate_errors.items():\n",
    "        for m in msgs:\n",
    "            errs.append([event, position_prefix + field, m])\n",
    "\n",
    "ev_cnt = {}\n",
    "errs = []\n",
    "for e in events:\n",
    "    e_name = e.get(\"name\", \"missing_name\")\n",
    "    v.validate(e, common_fields_scheme)\n",
    "    gather_errors(errs, e_name, v.errors)\n",
    "    s = event_params_specs.get(e_name)\n",
    "    pars = e.get('params')\n",
    "    if s and pars:\n",
    "        v.validate(pars, s)\n",
    "        gather_errors(errs, e_name, v.errors, position_prefix='params.')\n",
    "    ev_cnt[e_name] = ev_cnt.get(e_name, 0) + 1\n",
    "\n",
    "\n",
    "df_evcnt = pd.DataFrame({'event': list(ev_cnt.keys()), 'processed_events': list(ev_cnt.values())})\n",
    "#display(df_evcnt)\n",
    "\n",
    "df_errs = pd.DataFrame(errs, columns=['event', 'field', 'msg'])\n",
    "df_errs['err_cnt'] = 1\n",
    "df_errs_cnt = df_errs.groupby(['event', 'field', 'msg'])['err_cnt'].sum().reset_index()\n",
    "#display(df_errs_cnt)\n",
    "\n",
    "pd.merge(df_errs_cnt, df_evcnt, on='event', how='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "192549c0",
   "metadata": {},
   "source": [
    "### Stricter Common Fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ea440766",
   "metadata": {},
   "outputs": [],
   "source": [
    "known_events = ['payment_btn_click', 'main_banner_click']\n",
    "\n",
    "common_fields_scheme = {\n",
    "    'dt': {'type': 'datetime', 'coerce': lambda x: datetime.strptime(x,'%Y-%m-%d %H:%M:%S')},\n",
    "    'device_id': {'type': 'string'},\n",
    "    'user_id': {'type': 'integer', 'nullable': True},\n",
    "    'name': {'type': 'string', 'allowed': known_events},\n",
    "    'params': {'type': 'dict'}\n",
    "}\n",
    "\n",
    "\n",
    "payment_btn_click_scheme = {\n",
    "    'user_id': {'type': 'integer', 'nullable': False},\n",
    "    'params': {'type': 'dict', 'schema': {\n",
    "        'order_id': {'type': 'integer'},\n",
    "        'exp_id': {'type': 'integer'}}\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "main_banner_click_scheme = {\n",
    "    'params': {'type': 'dict', 'schema': {\n",
    "        'banner_id': {'type': 'integer'},\n",
    "        'exp_id': {'type': 'integer'}}\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "event_params_specs = {\n",
    "    'payment_btn_click': payment_btn_click_scheme,\n",
    "    'main_banner_click': main_banner_click_scheme\n",
    "}\n",
    "\n",
    "#todo: combine common_fields_scheme and paramters_schemes into single schemes?\n",
    "#common_fields_scheme['name'].update({'type': 'string', 'allowed': list(events_specs.keys())})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e0557724",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>field</th>\n",
       "      <th>msg</th>\n",
       "      <th>err_cnt</th>\n",
       "      <th>processed_events</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>params</td>\n",
       "      <td>required field</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>params.banner_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>params.exp_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>main_banner_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>params</td>\n",
       "      <td>required field</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>params.exp_id</td>\n",
       "      <td>required field</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>params.order_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>payment_btn_click</td>\n",
       "      <td>user_id</td>\n",
       "      <td>must be of integer type</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               event             field                      msg  err_cnt  \\\n",
       "0  main_banner_click            params           required field        2   \n",
       "1  main_banner_click  params.banner_id  must be of integer type        1   \n",
       "2  main_banner_click     params.exp_id  must be of integer type        1   \n",
       "3  main_banner_click           user_id  must be of integer type        3   \n",
       "4  payment_btn_click            params           required field        2   \n",
       "5  payment_btn_click     params.exp_id           required field        1   \n",
       "6  payment_btn_click   params.order_id  must be of integer type        1   \n",
       "7  payment_btn_click           user_id  must be of integer type        4   \n",
       "\n",
       "   processed_events  \n",
       "0                 3  \n",
       "1                 3  \n",
       "2                 3  \n",
       "3                 3  \n",
       "4                 3  \n",
       "5                 3  \n",
       "6                 3  \n",
       "7                 3  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events = [\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': 1231231,\n",
    "        'name': 'payment_btn_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'params': {\n",
    "            'order_id': '101010'\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'payment_btn_click',\n",
    "        'params': {\n",
    "            'order_id': 101010,\n",
    "            'exp_id': 9990\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click'\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click',\n",
    "        'params': {\n",
    "            'banner_id': 1789789,\n",
    "            'exp_id': 9990\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'dt': '2023-01-10 12:23:01',\n",
    "        'device_id': 'abc123-efg456',\n",
    "        'user_id': '1231231',\n",
    "        'name': 'main_banner_click',\n",
    "        'params': {\n",
    "            'banner_id': '1789789',\n",
    "            'exp_id': '9990'\n",
    "        }\n",
    "    } \n",
    "]\n",
    "\n",
    "v = Validator()\n",
    "v.allow_unknown = True\n",
    "v.require_all = True\n",
    "\n",
    "def gather_errors(errs, event, validate_errors, position_prefix=''):\n",
    "    #todo: use custom error handler?\n",
    "    #use document_error_tree for nodes location?\n",
    "    for field, msgs in validate_errors.items():\n",
    "        for m in msgs:\n",
    "            if type(m) is dict:\n",
    "                gather_errors(errs, event, m, position_prefix=field + '.')\n",
    "            else:\n",
    "                errs.append([event, position_prefix + field, m])\n",
    "\n",
    "ev_cnt = {}\n",
    "errs = []\n",
    "for e in events:\n",
    "    e_name = e.get(\"name\", \"missing_name\")\n",
    "    v.validate(e, common_fields_scheme)\n",
    "    gather_errors(errs, e_name, v.errors)\n",
    "    s = event_params_specs.get(e_name)\n",
    "    if s:\n",
    "        v.validate(e, s)\n",
    "        gather_errors(errs, e_name, v.errors)\n",
    "    ev_cnt[e_name] = ev_cnt.get(e_name, 0) + 1\n",
    "\n",
    "\n",
    "df_evcnt = pd.DataFrame({'event': list(ev_cnt.keys()), 'processed_events': list(ev_cnt.values())})\n",
    "#display(df_evcnt)\n",
    "\n",
    "df_errs = pd.DataFrame(errs, columns=['event', 'field', 'msg'])\n",
    "df_errs['err_cnt'] = 1\n",
    "df_errs_cnt = df_errs.groupby(['event', 'field', 'msg'])['err_cnt'].sum().reset_index()\n",
    "#display(df_errs_cnt)\n",
    "\n",
    "pd.merge(df_errs_cnt, df_evcnt, on='event', how='left')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}