tutorials/notebooks/preprocessing-demo.ipynb

{"cells":[{"cell_type":"markdown","id":"cf393d7e","metadata":{"id":"cf393d7e"},"source":["# Preprocessing demo\n","\n","This is a demo about data preprocessing.\n","We take a raw transactions and convert it to `ptls` format (list with feature dicts).\n","\n","Each paragraph shows a small example."]},{"cell_type":"markdown","metadata":{"id":"spMVAySTSEZ1"},"source":["# Colab setup"],"id":"spMVAySTSEZ1"},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":95591,"status":"ok","timestamp":1723921502396,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"},"user_tz":-180},"id":"WyOYsMF2SEZ3","outputId":"4fa8bce8-03ab-484d-a4b4-edda1f14f891"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting pytorch-lifestream\n","  Downloading pytorch-lifestream-0.6.0.tar.gz (163 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m163.4/163.4 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n","  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n","  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: duckdb in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (0.10.3)\n","Collecting hydra-core>=1.1.2 (from pytorch-lifestream)\n","  Downloading hydra_core-1.3.2-py3-none-any.whl.metadata (5.5 kB)\n","Requirement already satisfied: numpy>=1.21.5 in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (1.26.4)\n","Collecting omegaconf (from pytorch-lifestream)\n","  Downloading omegaconf-2.3.0-py3-none-any.whl.metadata (3.9 kB)\n","Requirement already satisfied: pandas>=1.3.5 in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (2.1.4)\n","Requirement already satisfied: pyarrow>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (14.0.2)\n","Collecting pytorch-lightning>=1.6.0 (from pytorch-lifestream)\n","  Downloading pytorch_lightning-2.4.0-py3-none-any.whl.metadata (21 kB)\n","Requirement already satisfied: scikit-learn>=1.0.2 in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (1.3.2)\n","Requirement already satisfied: torch>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (2.3.1+cu121)\n","Collecting torchmetrics>=0.9.0 (from pytorch-lifestream)\n","  Downloading torchmetrics-1.4.1-py3-none-any.whl.metadata (20 kB)\n","Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from pytorch-lifestream) (4.42.4)\n","Collecting antlr4-python3-runtime==4.9.* (from hydra-core>=1.1.2->pytorch-lifestream)\n","  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from hydra-core>=1.1.2->pytorch-lifestream) (24.1)\n","Requirement already satisfied: PyYAML>=5.1.0 in /usr/local/lib/python3.10/dist-packages (from omegaconf->pytorch-lifestream) (6.0.2)\n","Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.5->pytorch-lifestream) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.5->pytorch-lifestream) (2024.1)\n","Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.5->pytorch-lifestream) (2024.1)\n","Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.10/dist-packages (from pytorch-lightning>=1.6.0->pytorch-lifestream) (4.66.5)\n","Requirement already satisfied: fsspec>=2022.5.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (2024.6.1)\n","Requirement already satisfied: typing-extensions>=4.4.0 in /usr/local/lib/python3.10/dist-packages (from pytorch-lightning>=1.6.0->pytorch-lifestream) (4.12.2)\n","Collecting lightning-utilities>=0.10.0 (from pytorch-lightning>=1.6.0->pytorch-lifestream)\n","  Downloading lightning_utilities-0.11.6-py3-none-any.whl.metadata (5.2 kB)\n","Requirement already satisfied: scipy>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.2->pytorch-lifestream) (1.13.1)\n","Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.2->pytorch-lifestream) (1.4.2)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.2->pytorch-lifestream) (3.5.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch-lifestream) (3.15.4)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch-lifestream) (1.13.1)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch-lifestream) (3.3)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch-lifestream) (3.1.4)\n","Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n","Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n","Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n","Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n","Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n","Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n","Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n","Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n","Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n","Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n","Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n","Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch-lifestream) (2.3.1)\n","Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.12.0->pytorch-lifestream)\n","  Using cached nvidia_nvjitlink_cu12-12.6.20-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers->pytorch-lifestream) (0.23.5)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->pytorch-lifestream) (2024.5.15)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers->pytorch-lifestream) (2.32.3)\n","Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers->pytorch-lifestream) (0.4.4)\n","Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers->pytorch-lifestream) (0.19.1)\n","Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (3.10.2)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.10.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (71.0.4)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas>=1.3.5->pytorch-lifestream) (1.16.0)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.12.0->pytorch-lifestream) (2.1.5)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->pytorch-lifestream) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->pytorch-lifestream) (3.7)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->pytorch-lifestream) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->pytorch-lifestream) (2024.7.4)\n","Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.12.0->pytorch-lifestream) (1.3.0)\n","Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (2.3.5)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (1.3.1)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (24.2.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (1.4.1)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (6.0.5)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (1.9.4)\n","Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>=2022.5.0->pytorch-lightning>=1.6.0->pytorch-lifestream) (4.0.3)\n","Downloading hydra_core-1.3.2-py3-none-any.whl (154 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.5/154.5 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading pytorch_lightning-2.4.0-py3-none-any.whl (815 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m815.2/815.2 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hUsing cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n","Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n","Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n","Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n","Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n","Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n","Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n","Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n","Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n","Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n","Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n","Downloading torchmetrics-1.4.1-py3-none-any.whl (866 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m866.2/866.2 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading lightning_utilities-0.11.6-py3-none-any.whl (26 kB)\n","Using cached nvidia_nvjitlink_cu12-12.6.20-py3-none-manylinux2014_x86_64.whl (19.7 MB)\n","Building wheels for collected packages: pytorch-lifestream, antlr4-python3-runtime\n","  Building wheel for pytorch-lifestream (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for pytorch-lifestream: filename=pytorch_lifestream-0.6.0-py3-none-any.whl size=274603 sha256=3d786c52bc5ca38aee8bd58afedc5f6106f81e9a4d867214eea0bb558765d0f2\n","  Stored in directory: /root/.cache/pip/wheels/90/76/b4/0a944bc7c5a69201e4d757cc54886971117a2a581740e7f11d\n","  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=c7870d556a99d4387da6144199c65889073546943a49470515cdc3ad40b03b84\n","  Stored in directory: /root/.cache/pip/wheels/12/93/dd/1f6a127edc45659556564c5730f6d4e300888f4bca2d4c5a88\n","Successfully built pytorch-lifestream antlr4-python3-runtime\n","Installing collected packages: antlr4-python3-runtime, omegaconf, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, lightning-utilities, nvidia-cusparse-cu12, nvidia-cudnn-cu12, hydra-core, nvidia-cusolver-cu12, torchmetrics, pytorch-lightning, pytorch-lifestream\n","Successfully installed antlr4-python3-runtime-4.9.3 hydra-core-1.3.2 lightning-utilities-0.11.6 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.6.20 nvidia-nvtx-cu12-12.1.105 omegaconf-2.3.0 pytorch-lifestream-0.6.0 pytorch-lightning-2.4.0 torchmetrics-1.4.1\n"]}],"source":["import sys\n","if \"google.colab\" in str(get_ipython()):\n","    ! {sys.executable} -m pip install pytorch-lifestream"],"id":"WyOYsMF2SEZ3"},{"cell_type":"markdown","id":"5ef1a1e1","metadata":{"id":"5ef1a1e1"},"source":["# Common tools for this demo"]},{"cell_type":"code","execution_count":2,"id":"3f8e163b","metadata":{"id":"3f8e163b","executionInfo":{"status":"ok","timestamp":1723921513009,"user_tz":-180,"elapsed":10619,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["import pandas as pd\n","\n","from IPython.display import display\n","from ptls.preprocessing import PandasDataPreprocessor\n","from ptls.data_load.utils import FeatureDict"]},{"cell_type":"code","execution_count":3,"id":"8b16b84b","metadata":{"id":"8b16b84b","executionInfo":{"status":"ok","timestamp":1723921513009,"user_tz":-180,"elapsed":40,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["def load_trx(keep_original_trans_date=False):\n","    \"\"\"\n","    Load trx.\n","\n","    Only for demo purpose we:\n","        - convert trans_date to datetime.\n","          Usually date time field are already hase datetime format.\n","        - load only first 1M records. Demo will be faster\n","    \"\"\"\n","\n","    path = \"https://huggingface.co/datasets/dllllb/age-group-prediction/resolve/main/transactions_train.csv.gz?download=true\"\n","\n","    df = pd.read_csv(path, compression=\"gzip\", nrows=1e6)\n","\n","    if not keep_original_trans_date:\n","        df[\"trans_date\"] = (\n","            pd.to_datetime(\"2000-01-01\") + pd.to_timedelta(df[\"trans_date\"], \"D\")\n","        ).astype(str)\n","\n","    df[\"amount_rur\"] = df[\"amount_rur\"].astype(\"float32\")\n","\n","    return df"]},{"cell_type":"code","execution_count":4,"id":"4cccdfaa","metadata":{"id":"4cccdfaa","executionInfo":{"status":"ok","timestamp":1723921513009,"user_tz":-180,"elapsed":35,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["def load_target():\n","    path = \"https://huggingface.co/datasets/dllllb/age-group-prediction/resolve/main/train_target.csv?download=true\"\n","    df = pd.read_csv(path)\n","\n","    return df"]},{"cell_type":"code","execution_count":5,"id":"53a3f811","metadata":{"id":"53a3f811","executionInfo":{"status":"ok","timestamp":1723921513010,"user_tz":-180,"elapsed":35,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["def print_feature_dict(d):\n","    \"\"\"\n","    Print dict with only first 10 trnasactions\n","    \"\"\"\n","    print(\"\\n\".join([\n","        f\"{k}: {v[:10]} (first 10 values)\"\n","        if FeatureDict.is_seq_feature(k, v) else f\"{k}: {v}\"\n","        for k, v in d.items()\n","    ]))"]},{"cell_type":"markdown","id":"d9c057a3","metadata":{"id":"d9c057a3"},"source":["# Base Example"]},{"cell_type":"code","execution_count":6,"id":"0a8ff052","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"0a8ff052","executionInfo":{"status":"ok","timestamp":1723921517521,"user_tz":-180,"elapsed":4544,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"58202b86-71c4-4372-8c64-842a6b04314a"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        client_id  trans_date  small_group  amount_rur\n","0           33172  2000-01-07            4   71.462997\n","1           33172  2000-01-07           35   45.016998\n","2           33172  2000-01-09           11   13.887000\n","3           33172  2000-01-10           11   15.983000\n","4           33172  2000-01-11           11   21.341000\n","...           ...         ...          ...         ...\n","999995      19189  2001-10-26           36   98.331001\n","999996      19189  2001-10-26           15   17.247000\n","999997      19189  2001-10-26           24  150.645004\n","999998      19189  2001-10-26            1   51.473999\n","999999      19189  2001-10-27           32   86.553001\n","\n","[1000000 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-37f35632-0f62-4003-8a6b-203d93492268\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>trans_date</th>\n","      <th>small_group</th>\n","      <th>amount_rur</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>4</td>\n","      <td>71.462997</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>35</td>\n","      <td>45.016998</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>33172</td>\n","      <td>2000-01-09</td>\n","      <td>11</td>\n","      <td>13.887000</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>33172</td>\n","      <td>2000-01-10</td>\n","      <td>11</td>\n","      <td>15.983000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>33172</td>\n","      <td>2000-01-11</td>\n","      <td>11</td>\n","      <td>21.341000</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>999995</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>36</td>\n","      <td>98.331001</td>\n","    </tr>\n","    <tr>\n","      <th>999996</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>15</td>\n","      <td>17.247000</td>\n","    </tr>\n","    <tr>\n","      <th>999997</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>24</td>\n","      <td>150.645004</td>\n","    </tr>\n","    <tr>\n","      <th>999998</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>1</td>\n","      <td>51.473999</td>\n","    </tr>\n","    <tr>\n","      <th>999999</th>\n","      <td>19189</td>\n","      <td>2001-10-27</td>\n","      <td>32</td>\n","      <td>86.553001</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000000 rows × 4 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-37f35632-0f62-4003-8a6b-203d93492268')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-37f35632-0f62-4003-8a6b-203d93492268 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-37f35632-0f62-4003-8a6b-203d93492268');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-3c52556a-8f30-49f3-ba69-dc492352514f\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3c52556a-8f30-49f3-ba69-dc492352514f')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-3c52556a-8f30-49f3-ba69-dc492352514f button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_trx"}},"metadata":{},"execution_count":6}],"source":["df_trx = load_trx()\n","df_trx"]},{"cell_type":"code","execution_count":7,"id":"84394f5b","metadata":{"id":"84394f5b","executionInfo":{"status":"ok","timestamp":1723921517521,"user_tz":-180,"elapsed":15,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["preprocessor = PandasDataPreprocessor(\n","    col_id=\"client_id\",\n","    col_event_time=\"trans_date\",\n","    cols_category=[\"small_group\"],\n","    cols_numerical=[\"amount_rur\"],\n",")"]},{"cell_type":"code","execution_count":8,"id":"f17a97fb","metadata":{"id":"f17a97fb","executionInfo":{"status":"ok","timestamp":1723921523852,"user_tz":-180,"elapsed":6344,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_data = preprocessor.fit_transform(df_trx)"]},{"cell_type":"code","execution_count":9,"id":"8b4e5d78","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8b4e5d78","executionInfo":{"status":"ok","timestamp":1723921523853,"user_tz":-180,"elapsed":46,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"24ad1dd1-f7d3-403a-faee-16b7a8c53f9e"},"outputs":[{"output_type":"stream","name":"stdout","text":["Unique clients count: 1145\n"]}],"source":["print(f\"Unique clients count: {len(df_data)}\")"]},{"cell_type":"code","execution_count":10,"id":"deebdde0","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"deebdde0","executionInfo":{"status":"ok","timestamp":1723921523853,"user_tz":-180,"elapsed":41,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"69b026d5-46a7-4cc4-a25e-721f94e252ca"},"outputs":[{"output_type":"stream","name":"stdout","text":["client_id: 6\n","event_time: tensor([946684800, 947116800, 947548800, 947635200, 947980800, 947980800,\n","        948067200, 948067200, 948153600, 948240000]) (first 10 values)\n","small_group: tensor([ 4,  3,  1,  3,  4,  1,  4,  3, 22,  2]) (first 10 values)\n","amount_rur: tensor([ 4.0540, 13.7380, 20.7010, 21.5640, 13.4990, 23.7220,  4.3040,  8.6250,\n","        12.9380, 28.1620]) (first 10 values)\n"]}],"source":["print_feature_dict(df_data[0])"]},{"cell_type":"code","execution_count":11,"id":"cd955f8b","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cd955f8b","executionInfo":{"status":"ok","timestamp":1723921523853,"user_tz":-180,"elapsed":36,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"f1b73edf-67b1-4a0f-cdbe-915e04a9273c"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'small_group': 186}"]},"metadata":{},"execution_count":11}],"source":["preprocessor.get_category_dictionary_sizes()"]},{"cell_type":"markdown","id":"ab2ab01a","metadata":{"id":"ab2ab01a"},"source":["pay attention:\n","- `client_id` presented as scalar value\n","- `small_group` converted to embedding indexes with frequency encoding. There are 186 unique values incluging 0 padding token exists.\n","- `amount_rur` don\"t changed\n","- `event_time` converted to timestamp"]},{"cell_type":"markdown","id":"1382d77b","metadata":{"id":"1382d77b"},"source":["# Join target"]},{"cell_type":"markdown","id":"94e49042","metadata":{"id":"94e49042"},"source":["## before preprocessing"]},{"cell_type":"code","execution_count":12,"id":"d08dceee","metadata":{"id":"d08dceee","executionInfo":{"status":"ok","timestamp":1723921527012,"user_tz":-180,"elapsed":3191,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_trx = load_trx()\n","df_target = load_target()"]},{"cell_type":"code","source":["df_trx"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"mFxVSVUZtMRM","executionInfo":{"status":"ok","timestamp":1723921527013,"user_tz":-180,"elapsed":48,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"af6f62c0-8c1f-476b-ddf2-72b4dc5881e2"},"id":"mFxVSVUZtMRM","execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["        client_id  trans_date  small_group  amount_rur\n","0           33172  2000-01-07            4   71.462997\n","1           33172  2000-01-07           35   45.016998\n","2           33172  2000-01-09           11   13.887000\n","3           33172  2000-01-10           11   15.983000\n","4           33172  2000-01-11           11   21.341000\n","...           ...         ...          ...         ...\n","999995      19189  2001-10-26           36   98.331001\n","999996      19189  2001-10-26           15   17.247000\n","999997      19189  2001-10-26           24  150.645004\n","999998      19189  2001-10-26            1   51.473999\n","999999      19189  2001-10-27           32   86.553001\n","\n","[1000000 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-710aa626-4fab-446e-893f-451db4922fba\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>trans_date</th>\n","      <th>small_group</th>\n","      <th>amount_rur</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>4</td>\n","      <td>71.462997</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>35</td>\n","      <td>45.016998</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>33172</td>\n","      <td>2000-01-09</td>\n","      <td>11</td>\n","      <td>13.887000</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>33172</td>\n","      <td>2000-01-10</td>\n","      <td>11</td>\n","      <td>15.983000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>33172</td>\n","      <td>2000-01-11</td>\n","      <td>11</td>\n","      <td>21.341000</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>999995</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>36</td>\n","      <td>98.331001</td>\n","    </tr>\n","    <tr>\n","      <th>999996</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>15</td>\n","      <td>17.247000</td>\n","    </tr>\n","    <tr>\n","      <th>999997</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>24</td>\n","      <td>150.645004</td>\n","    </tr>\n","    <tr>\n","      <th>999998</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>1</td>\n","      <td>51.473999</td>\n","    </tr>\n","    <tr>\n","      <th>999999</th>\n","      <td>19189</td>\n","      <td>2001-10-27</td>\n","      <td>32</td>\n","      <td>86.553001</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000000 rows × 4 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-710aa626-4fab-446e-893f-451db4922fba')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-710aa626-4fab-446e-893f-451db4922fba button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-710aa626-4fab-446e-893f-451db4922fba');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-9c406309-c28f-43fc-ac95-baa882413ee4\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9c406309-c28f-43fc-ac95-baa882413ee4')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-9c406309-c28f-43fc-ac95-baa882413ee4 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_trx"}},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["df_target"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"Fbi2aj2NtIHn","executionInfo":{"status":"ok","timestamp":1723921527013,"user_tz":-180,"elapsed":41,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"e0050b3f-728f-43b5-ec49-304e88e93be8"},"id":"Fbi2aj2NtIHn","execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":["       client_id  bins\n","0          24662     2\n","1           1046     0\n","2          34089     2\n","3          34848     1\n","4          47076     3\n","...          ...   ...\n","29995      14303     1\n","29996      22301     2\n","29997      25731     0\n","29998      16820     3\n","29999       5265     0\n","\n","[30000 rows x 2 columns]"],"text/html":["\n","  <div id=\"df-385411cc-0ee7-4e6a-b18d-689b681d2b33\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>bins</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>24662</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1046</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>34089</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>34848</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>47076</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>29995</th>\n","      <td>14303</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>29996</th>\n","      <td>22301</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>29997</th>\n","      <td>25731</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>29998</th>\n","      <td>16820</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>29999</th>\n","      <td>5265</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>30000 rows × 2 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-385411cc-0ee7-4e6a-b18d-689b681d2b33')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-385411cc-0ee7-4e6a-b18d-689b681d2b33 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-385411cc-0ee7-4e6a-b18d-689b681d2b33');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-f51416e7-ccbf-462e-a8ed-6c7e76820946\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f51416e7-ccbf-462e-a8ed-6c7e76820946')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-f51416e7-ccbf-462e-a8ed-6c7e76820946 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_target","summary":"{\n  \"name\": \"df_target\",\n  \"rows\": 30000,\n  \"fields\": [\n    {\n      \"column\": \"client_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 14448,\n        \"min\": 4,\n        \"max\": 49998,\n        \"num_unique_values\": 30000,\n        \"samples\": [\n          36627,\n          16901,\n          23218\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"bins\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1,\n        \"min\": 0,\n        \"max\": 3,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0,\n          3,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":14}]},{"cell_type":"code","execution_count":15,"id":"a7c40ec0","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"a7c40ec0","executionInfo":{"status":"ok","timestamp":1723921527014,"user_tz":-180,"elapsed":38,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"ea876b2b-1d2b-47f1-a206-8e761f8fc275"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        client_id  trans_date  small_group  amount_rur  bins\n","0           33172  2000-01-07            4   71.462997     0\n","1           33172  2000-01-07           35   45.016998     0\n","2           33172  2000-01-09           11   13.887000     0\n","3           33172  2000-01-10           11   15.983000     0\n","4           33172  2000-01-11           11   21.341000     0\n","...           ...         ...          ...         ...   ...\n","999995      19189  2001-10-26           36   98.331001     0\n","999996      19189  2001-10-26           15   17.247000     0\n","999997      19189  2001-10-26           24  150.645004     0\n","999998      19189  2001-10-26            1   51.473999     0\n","999999      19189  2001-10-27           32   86.553001     0\n","\n","[1000000 rows x 5 columns]"],"text/html":["\n","  <div id=\"df-4c67833a-3e74-4a7d-8bee-6dcf508276c9\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>trans_date</th>\n","      <th>small_group</th>\n","      <th>amount_rur</th>\n","      <th>bins</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>4</td>\n","      <td>71.462997</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>35</td>\n","      <td>45.016998</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>33172</td>\n","      <td>2000-01-09</td>\n","      <td>11</td>\n","      <td>13.887000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>33172</td>\n","      <td>2000-01-10</td>\n","      <td>11</td>\n","      <td>15.983000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>33172</td>\n","      <td>2000-01-11</td>\n","      <td>11</td>\n","      <td>21.341000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>999995</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>36</td>\n","      <td>98.331001</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>999996</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>15</td>\n","      <td>17.247000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>999997</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>24</td>\n","      <td>150.645004</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>999998</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>1</td>\n","      <td>51.473999</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>999999</th>\n","      <td>19189</td>\n","      <td>2001-10-27</td>\n","      <td>32</td>\n","      <td>86.553001</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000000 rows × 5 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c67833a-3e74-4a7d-8bee-6dcf508276c9')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-4c67833a-3e74-4a7d-8bee-6dcf508276c9 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-4c67833a-3e74-4a7d-8bee-6dcf508276c9');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-43b29d3f-d076-43bf-ae25-68f0667810a0\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-43b29d3f-d076-43bf-ae25-68f0667810a0')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-43b29d3f-d076-43bf-ae25-68f0667810a0 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_trx"}},"metadata":{},"execution_count":15}],"source":["df_trx = pd.merge(df_trx, df_target, on=\"client_id\", how=\"inner\")\n","df_trx"]},{"cell_type":"markdown","id":"89b33dc3","metadata":{"id":"89b33dc3"},"source":["> **Note:**\n","> Target are multiplyed. Each row for `client_id` has the same `bins`"]},{"cell_type":"code","execution_count":16,"id":"9fd50f65","metadata":{"id":"9fd50f65","executionInfo":{"status":"ok","timestamp":1723921527014,"user_tz":-180,"elapsed":35,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["preprocessor = PandasDataPreprocessor(\n","    col_id=\"client_id\",\n","    col_event_time=\"trans_date\",\n","    cols_category=[\"small_group\"],\n","    cols_numerical=[\"amount_rur\"],\n","    cols_first_item=[\"bins\"],  # As target is the same for all transactions, lets take it from 1st record\n",")"]},{"cell_type":"code","execution_count":17,"id":"4af02f2b","metadata":{"id":"4af02f2b","executionInfo":{"status":"ok","timestamp":1723921529615,"user_tz":-180,"elapsed":2634,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_data = preprocessor.fit_transform(df_trx)"]},{"cell_type":"code","execution_count":18,"id":"696219ba","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"696219ba","executionInfo":{"status":"ok","timestamp":1723921529616,"user_tz":-180,"elapsed":30,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"4fcd796c-8828-4095-b115-4cdc20a14e38"},"outputs":[{"output_type":"stream","name":"stdout","text":["Unique clients count: 1145\n"]}],"source":["print(f\"Unique clients count: {len(df_data)}\")"]},{"cell_type":"code","execution_count":19,"id":"18ccf73e","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"18ccf73e","executionInfo":{"status":"ok","timestamp":1723921529616,"user_tz":-180,"elapsed":20,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"70d2bc3a-a6d4-447c-8c2e-cadd3c6d63a6"},"outputs":[{"output_type":"stream","name":"stdout","text":["client_id: 6\n","bins: 1\n","event_time: tensor([946684800, 947116800, 947548800, 947635200, 947980800, 947980800,\n","        948067200, 948067200, 948153600, 948240000]) (first 10 values)\n","small_group: tensor([ 4,  3,  1,  3,  4,  1,  4,  3, 22,  2]) (first 10 values)\n","amount_rur: tensor([ 4.0540, 13.7380, 20.7010, 21.5640, 13.4990, 23.7220,  4.3040,  8.6250,\n","        12.9380, 28.1620]) (first 10 values)\n"]}],"source":["print_feature_dict(df_data[0])"]},{"cell_type":"markdown","id":"0e3ce3c2","metadata":{"id":"0e3ce3c2"},"source":["pay attention:\n","- `client_id` presented as scalar value\n","- `small_group` converted to embedding indexes with frequency encoding\n","- `amount_rur` don\"t changed\n","- `event_time` converted to timestamp\n","- target value `bins` is in dict"]},{"cell_type":"markdown","id":"28077688","metadata":{"id":"28077688"},"source":["## after preprocessing"]},{"cell_type":"code","execution_count":20,"id":"4949ac62","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"4949ac62","executionInfo":{"status":"ok","timestamp":1723921533305,"user_tz":-180,"elapsed":3703,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"95fefd7b-eb52-41d3-858d-a0a3e952f954"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        client_id  trans_date  small_group  amount_rur\n","0           33172  2000-01-07            4   71.462997\n","1           33172  2000-01-07           35   45.016998\n","2           33172  2000-01-09           11   13.887000\n","3           33172  2000-01-10           11   15.983000\n","4           33172  2000-01-11           11   21.341000\n","...           ...         ...          ...         ...\n","999995      19189  2001-10-26           36   98.331001\n","999996      19189  2001-10-26           15   17.247000\n","999997      19189  2001-10-26           24  150.645004\n","999998      19189  2001-10-26            1   51.473999\n","999999      19189  2001-10-27           32   86.553001\n","\n","[1000000 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-0b08c3ed-f7bc-4d1c-813a-feb785e4369d\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>trans_date</th>\n","      <th>small_group</th>\n","      <th>amount_rur</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>4</td>\n","      <td>71.462997</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>33172</td>\n","      <td>2000-01-07</td>\n","      <td>35</td>\n","      <td>45.016998</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>33172</td>\n","      <td>2000-01-09</td>\n","      <td>11</td>\n","      <td>13.887000</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>33172</td>\n","      <td>2000-01-10</td>\n","      <td>11</td>\n","      <td>15.983000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>33172</td>\n","      <td>2000-01-11</td>\n","      <td>11</td>\n","      <td>21.341000</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>999995</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>36</td>\n","      <td>98.331001</td>\n","    </tr>\n","    <tr>\n","      <th>999996</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>15</td>\n","      <td>17.247000</td>\n","    </tr>\n","    <tr>\n","      <th>999997</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>24</td>\n","      <td>150.645004</td>\n","    </tr>\n","    <tr>\n","      <th>999998</th>\n","      <td>19189</td>\n","      <td>2001-10-26</td>\n","      <td>1</td>\n","      <td>51.473999</td>\n","    </tr>\n","    <tr>\n","      <th>999999</th>\n","      <td>19189</td>\n","      <td>2001-10-27</td>\n","      <td>32</td>\n","      <td>86.553001</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000000 rows × 4 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0b08c3ed-f7bc-4d1c-813a-feb785e4369d')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-0b08c3ed-f7bc-4d1c-813a-feb785e4369d button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-0b08c3ed-f7bc-4d1c-813a-feb785e4369d');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-82d322e6-5057-4e86-9923-cc43603e1c2d\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-82d322e6-5057-4e86-9923-cc43603e1c2d')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-82d322e6-5057-4e86-9923-cc43603e1c2d button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_trx"}},"metadata":{},"execution_count":20}],"source":["df_trx = load_trx()\n","df_trx"]},{"cell_type":"code","execution_count":21,"id":"7670306b","metadata":{"id":"7670306b","executionInfo":{"status":"ok","timestamp":1723921533305,"user_tz":-180,"elapsed":41,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["preprocessor = PandasDataPreprocessor(\n","    col_id=\"client_id\",\n","    col_event_time=\"trans_date\",\n","    cols_category=[\"small_group\"],\n","    cols_numerical=[\"amount_rur\"],\n","    return_records=False,  # change output type\n",")"]},{"cell_type":"code","execution_count":22,"id":"ef3a1b05","metadata":{"id":"ef3a1b05","executionInfo":{"status":"ok","timestamp":1723921534438,"user_tz":-180,"elapsed":1171,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_data = preprocessor.fit_transform(df_trx)"]},{"cell_type":"code","execution_count":23,"id":"8a88a5e4","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8a88a5e4","executionInfo":{"status":"ok","timestamp":1723921534439,"user_tz":-180,"elapsed":35,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"4d4f9601-bbed-46b9-d5b4-fc7096d673b6"},"outputs":[{"output_type":"stream","name":"stdout","text":["Unique clients count: 1145\n"]}],"source":["print(f\"Unique clients count: {len(df_data)}\")"]},{"cell_type":"markdown","id":"912f93f8","metadata":{"id":"912f93f8"},"source":["> **Note:**\n","> `df_data` is not a `list` this is `DataFrame`"]},{"cell_type":"code","execution_count":24,"id":"9e534f6f","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"9e534f6f","executionInfo":{"status":"ok","timestamp":1723921534837,"user_tz":-180,"elapsed":408,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"8a699537-befe-4e45-8a4b-c13a49602c77"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["       client_id  bins\n","0          24662     2\n","1           1046     0\n","2          34089     2\n","3          34848     1\n","4          47076     3\n","...          ...   ...\n","29995      14303     1\n","29996      22301     2\n","29997      25731     0\n","29998      16820     3\n","29999       5265     0\n","\n","[30000 rows x 2 columns]"],"text/html":["\n","  <div id=\"df-b82c4126-1e5e-472d-a0a7-79240dc6d08d\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>bins</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>24662</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1046</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>34089</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>34848</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>47076</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>29995</th>\n","      <td>14303</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>29996</th>\n","      <td>22301</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>29997</th>\n","      <td>25731</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>29998</th>\n","      <td>16820</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>29999</th>\n","      <td>5265</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>30000 rows × 2 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b82c4126-1e5e-472d-a0a7-79240dc6d08d')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-b82c4126-1e5e-472d-a0a7-79240dc6d08d button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-b82c4126-1e5e-472d-a0a7-79240dc6d08d');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-a7fa2de6-1eef-4e21-bae6-b07c8be1c039\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a7fa2de6-1eef-4e21-bae6-b07c8be1c039')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-a7fa2de6-1eef-4e21-bae6-b07c8be1c039 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_target","summary":"{\n  \"name\": \"df_target\",\n  \"rows\": 30000,\n  \"fields\": [\n    {\n      \"column\": \"client_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 14448,\n        \"min\": 4,\n        \"max\": 49998,\n        \"num_unique_values\": 30000,\n        \"samples\": [\n          36627,\n          16901,\n          23218\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"bins\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1,\n        \"min\": 0,\n        \"max\": 3,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0,\n          3,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":24}],"source":["df_target = load_target()\n","df_target"]},{"cell_type":"code","execution_count":25,"id":"e8e368fc","metadata":{"id":"e8e368fc","executionInfo":{"status":"ok","timestamp":1723921534838,"user_tz":-180,"elapsed":29,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_data = pd.merge(df_data, df_target, on=\"client_id\", how=\"inner\")\n","df_data = df_data.to_dict(orient=\"records\")"]},{"cell_type":"code","execution_count":26,"id":"76193f57","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"76193f57","executionInfo":{"status":"ok","timestamp":1723921534838,"user_tz":-180,"elapsed":29,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"5bdd8dd3-4eae-42b3-8598-8af41461d2cc"},"outputs":[{"output_type":"stream","name":"stdout","text":["Unique clients count: 1145\n"]}],"source":["print(f\"Unique clients count: {len(df_data)}\")"]},{"cell_type":"code","execution_count":27,"id":"a1e0f806","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"a1e0f806","executionInfo":{"status":"ok","timestamp":1723921534839,"user_tz":-180,"elapsed":23,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"3fe5269b-b13c-4077-9bb6-53cf44fb872e"},"outputs":[{"output_type":"stream","name":"stdout","text":["client_id: 6\n","event_time: tensor([946684800, 947116800, 947548800, 947635200, 947980800, 947980800,\n","        948067200, 948067200, 948153600, 948240000]) (first 10 values)\n","small_group: tensor([ 4,  3,  1,  3,  4,  1,  4,  3, 22,  2]) (first 10 values)\n","amount_rur: tensor([ 4.0540, 13.7380, 20.7010, 21.5640, 13.4990, 23.7220,  4.3040,  8.6250,\n","        12.9380, 28.1620]) (first 10 values)\n","bins: 1\n"]}],"source":["print_feature_dict(df_data[0])"]},{"cell_type":"markdown","id":"84340dff","metadata":{"id":"84340dff"},"source":["pay attention:\n","- `client_id` presented as scalar value\n","- `small_group` converted to embedding indexes with frequency encoding\n","- `amount_rur` don\"t changed\n","- `event_time` converted to timestamp\n","- target value `bins` is in dict"]},{"cell_type":"markdown","id":"051e580b","metadata":{"id":"051e580b"},"source":["# Without data preprocessing\n","\n","Original `trans_date` in this dataset is not a datetime. But we can use if for trx ordering"]},{"cell_type":"code","execution_count":28,"id":"d0818fd8","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"d0818fd8","executionInfo":{"status":"ok","timestamp":1723921536251,"user_tz":-180,"elapsed":1431,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"2fa0cf92-a0ce-46a8-dfcb-24b57afd19c0"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        client_id  trans_date  small_group  amount_rur\n","0           33172           6            4   71.462997\n","1           33172           6           35   45.016998\n","2           33172           8           11   13.887000\n","3           33172           9           11   15.983000\n","4           33172          10           11   21.341000\n","...           ...         ...          ...         ...\n","999995      19189         664           36   98.331001\n","999996      19189         664           15   17.247000\n","999997      19189         664           24  150.645004\n","999998      19189         664            1   51.473999\n","999999      19189         665           32   86.553001\n","\n","[1000000 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-d2aee90f-a1fa-4804-8f6c-973a944e65bd\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>trans_date</th>\n","      <th>small_group</th>\n","      <th>amount_rur</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>33172</td>\n","      <td>6</td>\n","      <td>4</td>\n","      <td>71.462997</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>33172</td>\n","      <td>6</td>\n","      <td>35</td>\n","      <td>45.016998</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>33172</td>\n","      <td>8</td>\n","      <td>11</td>\n","      <td>13.887000</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>33172</td>\n","      <td>9</td>\n","      <td>11</td>\n","      <td>15.983000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>33172</td>\n","      <td>10</td>\n","      <td>11</td>\n","      <td>21.341000</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>999995</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>36</td>\n","      <td>98.331001</td>\n","    </tr>\n","    <tr>\n","      <th>999996</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>15</td>\n","      <td>17.247000</td>\n","    </tr>\n","    <tr>\n","      <th>999997</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>24</td>\n","      <td>150.645004</td>\n","    </tr>\n","    <tr>\n","      <th>999998</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>1</td>\n","      <td>51.473999</td>\n","    </tr>\n","    <tr>\n","      <th>999999</th>\n","      <td>19189</td>\n","      <td>665</td>\n","      <td>32</td>\n","      <td>86.553001</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000000 rows × 4 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d2aee90f-a1fa-4804-8f6c-973a944e65bd')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-d2aee90f-a1fa-4804-8f6c-973a944e65bd button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-d2aee90f-a1fa-4804-8f6c-973a944e65bd');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-ed8457b8-2f4a-4c93-97a1-55aa8d97d547\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ed8457b8-2f4a-4c93-97a1-55aa8d97d547')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-ed8457b8-2f4a-4c93-97a1-55aa8d97d547 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_trx"}},"metadata":{},"execution_count":28}],"source":["df_trx = load_trx(keep_original_trans_date=True)\n","df_trx"]},{"cell_type":"code","execution_count":29,"id":"8a0e1344","metadata":{"id":"8a0e1344","executionInfo":{"status":"ok","timestamp":1723921536251,"user_tz":-180,"elapsed":10,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["preprocessor = PandasDataPreprocessor(\n","    col_id=\"client_id\",\n","    col_event_time=\"trans_date\",\n","    event_time_transformation=\"none\",  # none transformation\n","    cols_category=[\"small_group\"],\n","    cols_numerical=[\"amount_rur\"],\n",")"]},{"cell_type":"code","execution_count":30,"id":"ebd75586","metadata":{"id":"ebd75586","executionInfo":{"status":"ok","timestamp":1723921539399,"user_tz":-180,"elapsed":3156,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_data = preprocessor.fit_transform(df_trx)"]},{"cell_type":"code","execution_count":31,"id":"f564ad76","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"f564ad76","executionInfo":{"status":"ok","timestamp":1723921539400,"user_tz":-180,"elapsed":16,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"3128eeaf-b732-401e-b6db-ec916f3b01b9"},"outputs":[{"output_type":"stream","name":"stdout","text":["Unique clients count: 1145\n"]}],"source":["print(f\"Unique clients count: {len(df_data)}\")"]},{"cell_type":"code","execution_count":32,"id":"cdb8262a","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cdb8262a","executionInfo":{"status":"ok","timestamp":1723921539400,"user_tz":-180,"elapsed":11,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"d2553f33-aaf8-4230-acd4-8f395af40d01"},"outputs":[{"output_type":"stream","name":"stdout","text":["client_id: 6\n","trans_date: tensor([ 0,  5, 10, 11, 15, 15, 16, 16, 17, 18]) (first 10 values)\n","event_time: tensor([ 0,  5, 10, 11, 15, 15, 16, 16, 17, 18]) (first 10 values)\n","small_group: tensor([ 4,  3,  1,  3,  4,  1,  4,  3, 22,  2]) (first 10 values)\n","amount_rur: tensor([ 4.0540, 13.7380, 20.7010, 21.5640, 13.4990, 23.7220,  4.3040,  8.6250,\n","        12.9380, 28.1620]) (first 10 values)\n"]}],"source":["print_feature_dict(df_data[0])"]},{"cell_type":"markdown","id":"7d5bfc1b","metadata":{"id":"7d5bfc1b"},"source":["pay attention:\n","- `client_id` presented as scalar value\n","- `small_group` converted to embedding indexes with frequency encoding\n","- `amount_rur` don\"t changed\n","- `event_time` keep original value. Trx are ordered"]},{"cell_type":"markdown","id":"f48ca0d8","metadata":{"id":"f48ca0d8"},"source":["# Category identity encoding\n","\n","`small_group` in out dataset is encoded mcc code. It seems we don\"t need freauency encoding."]},{"cell_type":"code","execution_count":33,"id":"3259c4e3","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"id":"3259c4e3","executionInfo":{"status":"ok","timestamp":1723921540886,"user_tz":-180,"elapsed":1493,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"b80f7a50-0331-4f4b-f190-ee63c11923b9"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        client_id  trans_date  small_group  amount_rur\n","0           33172           6            4   71.462997\n","1           33172           6           35   45.016998\n","2           33172           8           11   13.887000\n","3           33172           9           11   15.983000\n","4           33172          10           11   21.341000\n","...           ...         ...          ...         ...\n","999995      19189         664           36   98.331001\n","999996      19189         664           15   17.247000\n","999997      19189         664           24  150.645004\n","999998      19189         664            1   51.473999\n","999999      19189         665           32   86.553001\n","\n","[1000000 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-e8be5c09-bca8-41e8-94c6-f278ee8f9d22\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>client_id</th>\n","      <th>trans_date</th>\n","      <th>small_group</th>\n","      <th>amount_rur</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>33172</td>\n","      <td>6</td>\n","      <td>4</td>\n","      <td>71.462997</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>33172</td>\n","      <td>6</td>\n","      <td>35</td>\n","      <td>45.016998</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>33172</td>\n","      <td>8</td>\n","      <td>11</td>\n","      <td>13.887000</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>33172</td>\n","      <td>9</td>\n","      <td>11</td>\n","      <td>15.983000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>33172</td>\n","      <td>10</td>\n","      <td>11</td>\n","      <td>21.341000</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>999995</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>36</td>\n","      <td>98.331001</td>\n","    </tr>\n","    <tr>\n","      <th>999996</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>15</td>\n","      <td>17.247000</td>\n","    </tr>\n","    <tr>\n","      <th>999997</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>24</td>\n","      <td>150.645004</td>\n","    </tr>\n","    <tr>\n","      <th>999998</th>\n","      <td>19189</td>\n","      <td>664</td>\n","      <td>1</td>\n","      <td>51.473999</td>\n","    </tr>\n","    <tr>\n","      <th>999999</th>\n","      <td>19189</td>\n","      <td>665</td>\n","      <td>32</td>\n","      <td>86.553001</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1000000 rows × 4 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e8be5c09-bca8-41e8-94c6-f278ee8f9d22')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-e8be5c09-bca8-41e8-94c6-f278ee8f9d22 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-e8be5c09-bca8-41e8-94c6-f278ee8f9d22');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-ce4c5705-f706-424b-a1a4-87e7a0ff139d\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ce4c5705-f706-424b-a1a4-87e7a0ff139d')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-ce4c5705-f706-424b-a1a4-87e7a0ff139d button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_trx"}},"metadata":{},"execution_count":33}],"source":["df_trx = load_trx(keep_original_trans_date=True)\n","df_trx"]},{"cell_type":"code","execution_count":34,"id":"2a7e9c85","metadata":{"id":"2a7e9c85","executionInfo":{"status":"ok","timestamp":1723921540888,"user_tz":-180,"elapsed":18,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["preprocessor = PandasDataPreprocessor(\n","    col_id=\"client_id\",\n","    col_event_time=\"trans_date\",\n","    event_time_transformation=\"none\",  # none transformation\n","    cols_category=[\"small_group\"],\n","    category_transformation=\"none\",\n","    cols_numerical=[\"amount_rur\"],\n",")"]},{"cell_type":"code","execution_count":35,"id":"88fb2c8c","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"88fb2c8c","executionInfo":{"status":"ok","timestamp":1723921542170,"user_tz":-180,"elapsed":1299,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"ce9ede81-6878-4537-ecaf-fbcba6a814e5"},"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:67: FutureWarning: The provided callable <built-in function min> is currently using Series.min. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"min\" instead.\n","  self.min_fit_index, self.max_fit_index = pd_col.agg([min, max])\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:67: FutureWarning: The provided callable <built-in function max> is currently using Series.max. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"max\" instead.\n","  self.min_fit_index, self.max_fit_index = pd_col.agg([min, max])\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:71: UserWarning: 0 values fount in small_group. 0 is a padding index\n","  warnings.warn(f'0 values fount in {self.col_name_original}. 0 is a padding index', UserWarning)\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:82: FutureWarning: The provided callable <built-in function min> is currently using Series.min. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"min\" instead.\n","  min_index, max_index = pd_col.agg([min, max])\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:82: FutureWarning: The provided callable <built-in function max> is currently using Series.max. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"max\" instead.\n","  min_index, max_index = pd_col.agg([min, max])\n"]}],"source":["df_data = preprocessor.fit_transform(df_trx)"]},{"cell_type":"markdown","id":"edd0ddd3","metadata":{"id":"edd0ddd3"},"source":["> Oops. 0 is padding token. Let\"s shift values to 1"]},{"cell_type":"code","execution_count":36,"id":"712fef48","metadata":{"id":"712fef48","executionInfo":{"status":"ok","timestamp":1723921542171,"user_tz":-180,"elapsed":13,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}}},"outputs":[],"source":["df_trx[\"small_group\"] = df_trx[\"small_group\"] + 1"]},{"cell_type":"code","execution_count":37,"id":"50d7aa9c","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"50d7aa9c","executionInfo":{"status":"ok","timestamp":1723921543525,"user_tz":-180,"elapsed":1365,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"e50be794-b3bc-4c7d-ceac-0d1c2c7048d2"},"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:67: FutureWarning: The provided callable <built-in function min> is currently using Series.min. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"min\" instead.\n","  self.min_fit_index, self.max_fit_index = pd_col.agg([min, max])\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:67: FutureWarning: The provided callable <built-in function max> is currently using Series.max. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"max\" instead.\n","  self.min_fit_index, self.max_fit_index = pd_col.agg([min, max])\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:82: FutureWarning: The provided callable <built-in function min> is currently using Series.min. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"min\" instead.\n","  min_index, max_index = pd_col.agg([min, max])\n","/usr/local/lib/python3.10/dist-packages/ptls/preprocessing/pandas/category_identity_encoder.py:82: FutureWarning: The provided callable <built-in function max> is currently using Series.max. In a future version of pandas, the provided callable will be used directly. To keep current behavior pass the string \"max\" instead.\n","  min_index, max_index = pd_col.agg([min, max])\n"]}],"source":["df_data = preprocessor.fit_transform(df_trx)"]},{"cell_type":"code","execution_count":38,"id":"50ffd05e","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"50ffd05e","executionInfo":{"status":"ok","timestamp":1723921543525,"user_tz":-180,"elapsed":25,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"b7574161-09ed-4385-9ead-ad7cdf77b802"},"outputs":[{"output_type":"stream","name":"stdout","text":["Unique clients count: 1145\n"]}],"source":["print(f\"Unique clients count: {len(df_data)}\")"]},{"cell_type":"code","execution_count":39,"id":"f3f2fc42","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"f3f2fc42","executionInfo":{"status":"ok","timestamp":1723921543526,"user_tz":-180,"elapsed":19,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"0c0b873a-1b35-4260-cd34-a15da0a33e18"},"outputs":[{"output_type":"stream","name":"stdout","text":["client_id: 6\n","trans_date: tensor([ 0,  5, 10, 11, 15, 15, 16, 16, 17, 18]) (first 10 values)\n","event_time: tensor([ 0,  5, 10, 11, 15, 15, 16, 16, 17, 18]) (first 10 values)\n","small_group: tensor([16,  4,  2,  4, 16,  2, 16,  4, 38, 12]) (first 10 values)\n","amount_rur: tensor([ 4.0540, 13.7380, 20.7010, 21.5640, 13.4990, 23.7220,  4.3040,  8.6250,\n","        12.9380, 28.1620]) (first 10 values)\n"]}],"source":["print_feature_dict(df_data[0])"]},{"cell_type":"code","execution_count":40,"id":"e84dda7c","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"e84dda7c","executionInfo":{"status":"ok","timestamp":1723921543526,"user_tz":-180,"elapsed":15,"user":{"displayName":"Amurkhan Dzagkoev","userId":"04975642892911186905"}},"outputId":"7b9f4f46-9eb7-47b8-fb5e-3c1feee9dc80"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'small_group': 189}"]},"metadata":{},"execution_count":40}],"source":["preprocessor.get_category_dictionary_sizes()"]},{"cell_type":"markdown","id":"4af87345","metadata":{"id":"4af87345"},"source":["pay attention:\n","- `client_id` presented as scalar value\n","- `small_group` is embedding indexes but orignal values is used. 189 indexes including 0 padding token are used. There was 186 with frequency encoding. This means that some indexes aren\"t presented in dataset and aren\"t used.\n","- `amount_rur` don\"t changed\n","- `event_time` keep original value. Trx are ordered"]}],"metadata":{"kernelspec":{"display_name":"ptls","language":"python","name":"ptls"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.10"},"colab":{"provenance":[]}},"nbformat":4,"nbformat_minor":5}