tutorials/reproduce_multivariate_bocpd.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/phamquiluan/baro/blob/main/tutorials/reproduce_multivariate_bocpd.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IBknvwsqns3l"
      },
      "source": [
        "# Reproduction of Multivariate BOCPD\n",
        "\n",
        "In this notebook, we reproduce the anomaly detection output from Multivariate BOCPD module on the Online Boutique dataset."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install BARO and import packages\n",
        "\n"
      ],
      "metadata": {
        "id": "Os-9oVHroSYh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q fse-baro"
      ],
      "metadata": {
        "id": "zMXnLlawn50X"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "bV13VFwNns3m"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import glob\n",
        "import json\n",
        "import warnings\n",
        "from os.path import join, dirname, basename\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from baro.anomaly_detection import bocpd\n",
        "from baro._bocpd import online_changepoint_detection, partial, constant_hazard, MultivariateT\n",
        "from baro.utility import load_json, drop_constant, download_data"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download Online Boutique dataset from Zenodo"
      ],
      "metadata": {
        "id": "7oaAmaVloGov"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "download_data(\"https://zenodo.org/records/11046533/files/fse-ob.zip?download=1\", \"fse-ob.zip\")\n",
        "import zipfile\n",
        "os.makedirs(\"data\")\n",
        "with zipfile.ZipFile(\"fse-ob.zip\", 'r') as file:\n",
        "    file.extractall(\"data\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GNbn6MVrn3AZ",
        "outputId": "28a306bc-bba1-4768-d183-6c5e7ce261a0"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading fse-ob.zip..: 100%|██████████| 151M/151M [00:03<00:00, 41.0MiB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run"
      ],
      "metadata": {
        "id": "Er1-dnjroW9p"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmq_kictns3n",
        "outputId": "c2b4bef6-4390-479d-b2ee-63f8dde89155"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "./data/fse-ob/checkoutservice_delay/5/naive_bocpd.json\n",
            "Saved Anomalies: [340, 600]\n",
            "Anomalies Detected: [340, 600]\n",
            "./data/fse-ob/checkoutservice_delay/4/naive_bocpd.json\n",
            "Saved Anomalies: [347, 600]\n",
            "Anomalies Detected: [347, 600]\n",
            "./data/fse-ob/checkoutservice_delay/1/naive_bocpd.json\n",
            "Saved Anomalies: [343, 600]\n",
            "Anomalies Detected: [343, 600]\n",
            "./data/fse-ob/checkoutservice_delay/3/naive_bocpd.json\n",
            "Saved Anomalies: [231, 237, 349]\n",
            "Anomalies Detected: [231, 237, 349, 600]\n",
            "./data/fse-ob/checkoutservice_delay/2/naive_bocpd.json\n",
            "Saved Anomalies: [351, 554, 558]\n",
            "Anomalies Detected: [351, 554, 558, 600]\n",
            "./data/fse-ob/checkoutservice_loss/5/naive_bocpd.json\n",
            "Saved Anomalies: [281, 332, 348]\n",
            "Anomalies Detected: [281, 332, 348, 354, 359, 457, 549, 577, 597, 600]\n",
            "./data/fse-ob/checkoutservice_loss/4/naive_bocpd.json\n",
            "Saved Anomalies: [326, 329, 512]\n",
            "Anomalies Detected: [326, 329, 512, 569, 600]\n",
            "./data/fse-ob/checkoutservice_loss/1/naive_bocpd.json\n",
            "Saved Anomalies: [243, 260, 332]\n",
            "Anomalies Detected: [243, 260, 332, 333, 572, 600]\n",
            "./data/fse-ob/checkoutservice_loss/3/naive_bocpd.json\n",
            "Saved Anomalies: [338, 343, 344]\n",
            "Anomalies Detected: [338, 343, 344, 365, 366, 435, 437, 511, 600]\n",
            "./data/fse-ob/checkoutservice_loss/2/naive_bocpd.json\n",
            "Saved Anomalies: [299, 326, 327]\n",
            "Anomalies Detected: [299, 326, 327, 344, 493, 595, 600]\n",
            "./data/fse-ob/productcatalogservice_loss/5/naive_bocpd.json\n",
            "Saved Anomalies: [357, 376, 563]\n",
            "Anomalies Detected: [357, 376, 563, 600]\n",
            "./data/fse-ob/productcatalogservice_loss/4/naive_bocpd.json\n",
            "Saved Anomalies: [340, 353, 356]\n",
            "Anomalies Detected: [340, 353, 356, 357, 376, 458, 498, 501, 600]\n",
            "./data/fse-ob/productcatalogservice_loss/1/naive_bocpd.json\n",
            "Saved Anomalies: [368, 372, 373]\n",
            "Anomalies Detected: [368, 372, 373, 380, 388, 402, 485, 488, 524, 525, 544, 583, 590, 593, 595, 597, 600]\n",
            "./data/fse-ob/productcatalogservice_loss/3/naive_bocpd.json\n",
            "Saved Anomalies: [363, 364, 371]\n",
            "Anomalies Detected: [363, 364, 371, 374, 388, 410, 486, 492, 514, 518, 531, 547, 600]\n",
            "./data/fse-ob/productcatalogservice_loss/2/naive_bocpd.json\n",
            "Saved Anomalies: [69, 70, 234]\n",
            "Anomalies Detected: [69, 70, 234, 236, 358, 483, 600]\n",
            "./data/fse-ob/paymentservice_delay/5/naive_bocpd.json\n",
            "Saved Anomalies: [254, 259, 338]\n",
            "Anomalies Detected: [254, 259, 338, 557, 566, 600]\n",
            "./data/fse-ob/paymentservice_delay/4/naive_bocpd.json\n",
            "Saved Anomalies: [104, 106, 209]\n",
            "Anomalies Detected: [104, 106, 209, 333, 600]\n",
            "./data/fse-ob/paymentservice_delay/1/naive_bocpd.json\n",
            "Saved Anomalies: [66, 67, 340]\n",
            "Anomalies Detected: [66, 67, 340, 600]\n",
            "./data/fse-ob/paymentservice_delay/3/naive_bocpd.json\n",
            "Saved Anomalies: [330, 490, 494]\n",
            "Anomalies Detected: [330, 490, 494, 495, 500, 547, 600]\n",
            "./data/fse-ob/paymentservice_delay/2/naive_bocpd.json\n",
            "Saved Anomalies: [337, 486, 488]\n",
            "Anomalies Detected: [337, 486, 488, 506, 510, 581, 584, 600]\n",
            "./data/fse-ob/currencyservice_cpu/5/naive_bocpd.json\n",
            "Saved Anomalies: [340, 344, 351]\n",
            "Anomalies Detected: [340, 344, 351, 371, 456, 489, 600]\n",
            "./data/fse-ob/currencyservice_cpu/4/naive_bocpd.json\n",
            "Saved Anomalies: [310, 329, 340]\n",
            "Anomalies Detected: [310, 329, 340, 509, 594, 595, 600]\n",
            "./data/fse-ob/currencyservice_cpu/1/naive_bocpd.json\n",
            "Saved Anomalies: [238, 242, 268]\n",
            "Anomalies Detected: [238, 242, 268, 270, 336, 549, 593, 600]\n",
            "./data/fse-ob/currencyservice_cpu/3/naive_bocpd.json\n",
            "Saved Anomalies: [335, 351, 365]\n",
            "Anomalies Detected: [335, 351, 365, 566, 600]\n",
            "./data/fse-ob/currencyservice_cpu/2/naive_bocpd.json\n",
            "Saved Anomalies: [340, 587, 600]\n",
            "Anomalies Detected: [340, 587, 600]\n",
            "./data/fse-ob/currencyservice_delay/5/naive_bocpd.json\n",
            "Saved Anomalies: [180, 184, 337]\n",
            "Anomalies Detected: [180, 184, 337, 480, 508, 585, 600]\n",
            "./data/fse-ob/currencyservice_delay/4/naive_bocpd.json\n",
            "Saved Anomalies: [339, 493, 590]\n",
            "Anomalies Detected: [339, 493, 590, 597, 600]\n",
            "./data/fse-ob/currencyservice_delay/1/naive_bocpd.json\n",
            "Saved Anomalies: [343, 484, 600]\n",
            "Anomalies Detected: [343, 484, 600]\n",
            "./data/fse-ob/currencyservice_delay/3/naive_bocpd.json\n",
            "Saved Anomalies: [340, 344, 439]\n",
            "Anomalies Detected: [340, 344, 439, 479, 561, 576, 600]\n",
            "./data/fse-ob/currencyservice_delay/2/naive_bocpd.json\n",
            "Saved Anomalies: [339, 500, 505]\n",
            "Anomalies Detected: [339, 500, 505, 506, 542, 559, 600]\n",
            "./data/fse-ob/currencyservice_mem/5/naive_bocpd.json\n",
            "Saved Anomalies: [345, 351, 492]\n",
            "Anomalies Detected: [345, 351, 492, 534, 600]\n",
            "./data/fse-ob/currencyservice_mem/4/naive_bocpd.json\n",
            "Saved Anomalies: [346, 600]\n",
            "Anomalies Detected: [346, 600]\n",
            "./data/fse-ob/currencyservice_mem/1/naive_bocpd.json\n",
            "Saved Anomalies: [345, 600]\n",
            "Anomalies Detected: [345, 600]\n",
            "./data/fse-ob/currencyservice_mem/3/naive_bocpd.json\n",
            "Saved Anomalies: [348, 367, 531]\n",
            "Anomalies Detected: [348, 367, 531, 533, 600]\n",
            "./data/fse-ob/currencyservice_mem/2/naive_bocpd.json\n",
            "Saved Anomalies: [344, 550, 600]\n",
            "Anomalies Detected: [344, 550, 600]\n",
            "./data/fse-ob/cartservice_cpu/5/naive_bocpd.json\n",
            "Saved Anomalies: [300, 315, 328]\n",
            "Anomalies Detected: [300, 315, 328, 357, 511, 539, 600]\n",
            "./data/fse-ob/cartservice_cpu/4/naive_bocpd.json\n",
            "Saved Anomalies: [257, 261, 336]\n",
            "Anomalies Detected: [257, 261, 336, 582, 597, 600]\n",
            "./data/fse-ob/cartservice_cpu/1/naive_bocpd.json\n",
            "Saved Anomalies: [341, 344, 387]\n",
            "Anomalies Detected: [341, 344, 387, 390, 418, 527, 546, 600]\n",
            "./data/fse-ob/cartservice_cpu/3/naive_bocpd.json\n",
            "Saved Anomalies: [339, 340, 351]\n",
            "Anomalies Detected: [339, 340, 351, 378, 379, 520, 600]\n",
            "./data/fse-ob/cartservice_cpu/2/naive_bocpd.json\n",
            "Saved Anomalies: [205, 230, 332]\n",
            "Anomalies Detected: [205, 230, 332, 333, 337, 338, 481, 513, 600]\n",
            "./data/fse-ob/cartservice_loss/5/naive_bocpd.json\n",
            "Saved Anomalies: [334, 349, 414]\n",
            "Anomalies Detected: [334, 349, 414, 576, 600]\n",
            "./data/fse-ob/cartservice_loss/4/naive_bocpd.json\n",
            "Saved Anomalies: [367, 368, 369]\n",
            "Anomalies Detected: [367, 368, 369, 378, 391, 407, 445, 502, 600]\n",
            "./data/fse-ob/cartservice_loss/1/naive_bocpd.json\n",
            "Saved Anomalies: [298, 310, 319]\n",
            "Anomalies Detected: [298, 310, 319, 438, 518, 521, 600]\n",
            "./data/fse-ob/cartservice_loss/3/naive_bocpd.json\n",
            "Saved Anomalies: [341, 600]\n",
            "Anomalies Detected: [341, 600]\n",
            "./data/fse-ob/cartservice_loss/2/naive_bocpd.json\n",
            "Saved Anomalies: [180, 201, 257]\n",
            "Anomalies Detected: [180, 201, 257, 314, 315, 385, 406, 410, 453, 454, 506, 600]\n",
            "./data/fse-ob/productcatalogservice_delay/5/naive_bocpd.json\n",
            "Saved Anomalies: [339, 444, 600]\n",
            "Anomalies Detected: [339, 444, 600]\n",
            "./data/fse-ob/productcatalogservice_delay/4/naive_bocpd.json\n",
            "Saved Anomalies: [339, 451, 460]\n",
            "Anomalies Detected: [339, 451, 460, 499, 600]\n",
            "./data/fse-ob/productcatalogservice_delay/1/naive_bocpd.json\n",
            "Saved Anomalies: [108, 110, 133]\n",
            "Anomalies Detected: [108, 110, 133, 147, 340, 487, 600]\n",
            "./data/fse-ob/productcatalogservice_delay/3/naive_bocpd.json\n",
            "Saved Anomalies: [340, 439, 600]\n",
            "Anomalies Detected: [340, 439, 600]\n",
            "./data/fse-ob/productcatalogservice_delay/2/naive_bocpd.json\n",
            "Saved Anomalies: [63, 64, 340]\n",
            "Anomalies Detected: [63, 64, 340, 454, 600]\n",
            "./data/fse-ob/paymentservice_mem/5/naive_bocpd.json\n",
            "Saved Anomalies: [339, 342, 347]\n",
            "Anomalies Detected: [339, 342, 347, 531, 532, 600]\n",
            "./data/fse-ob/paymentservice_mem/4/naive_bocpd.json\n",
            "Saved Anomalies: [157, 159, 160]\n",
            "Anomalies Detected: [157, 159, 160, 177, 320, 321, 352, 354, 600]\n",
            "./data/fse-ob/paymentservice_mem/1/naive_bocpd.json\n",
            "Saved Anomalies: [307, 308, 318]\n",
            "Anomalies Detected: [307, 308, 318, 319, 321, 322, 348, 357, 600]\n",
            "./data/fse-ob/paymentservice_mem/3/naive_bocpd.json\n",
            "Saved Anomalies: [334, 336, 341]\n",
            "Anomalies Detected: [334, 336, 341, 347, 435, 600]\n",
            "./data/fse-ob/paymentservice_mem/2/naive_bocpd.json\n",
            "Saved Anomalies: [345, 346, 472]\n",
            "Anomalies Detected: [345, 346, 472, 474, 475, 480, 520, 521, 600]\n",
            "./data/fse-ob/currencyservice_loss/5/naive_bocpd.json\n",
            "Saved Anomalies: [306, 600]\n",
            "Anomalies Detected: [306, 600]\n",
            "./data/fse-ob/currencyservice_loss/4/naive_bocpd.json\n",
            "Saved Anomalies: [331, 340, 396]\n",
            "Anomalies Detected: [331, 340, 396, 398, 435, 516, 580, 600]\n",
            "./data/fse-ob/currencyservice_loss/1/naive_bocpd.json\n",
            "Saved Anomalies: [347, 349, 372]\n",
            "Anomalies Detected: [347, 349, 372, 397, 398, 552, 600]\n",
            "./data/fse-ob/currencyservice_loss/3/naive_bocpd.json\n",
            "Saved Anomalies: [157, 298, 306]\n",
            "Anomalies Detected: [157, 298, 306, 333, 600]\n",
            "./data/fse-ob/currencyservice_loss/2/naive_bocpd.json\n",
            "Saved Anomalies: [194, 201, 355]\n",
            "Anomalies Detected: [194, 201, 355, 371, 410, 502, 505, 600]\n",
            "./data/fse-ob/checkoutservice_cpu/5/naive_bocpd.json\n",
            "Saved Anomalies: [355, 358, 371]\n",
            "Anomalies Detected: [355, 358, 371, 459, 579, 600]\n",
            "./data/fse-ob/checkoutservice_cpu/4/naive_bocpd.json\n",
            "Saved Anomalies: [170, 173, 324]\n",
            "Anomalies Detected: [170, 173, 324, 406, 442, 600]\n",
            "./data/fse-ob/checkoutservice_cpu/1/naive_bocpd.json\n",
            "Saved Anomalies: [248, 320, 353]\n",
            "Anomalies Detected: [248, 320, 353, 369, 371, 469, 474, 479, 600]\n",
            "./data/fse-ob/checkoutservice_cpu/3/naive_bocpd.json\n",
            "Saved Anomalies: [264, 354, 355]\n",
            "Anomalies Detected: [264, 354, 355, 363, 365, 369, 494, 498, 499, 501, 526, 600]\n",
            "./data/fse-ob/checkoutservice_cpu/2/naive_bocpd.json\n",
            "Saved Anomalies: [115, 347, 494]\n",
            "Anomalies Detected: [115, 347, 494, 499, 600]\n",
            "./data/fse-ob/productcatalogservice_cpu/5/naive_bocpd.json\n",
            "Saved Anomalies: [341, 342, 373]\n",
            "Anomalies Detected: [341, 342, 373, 461, 504, 600]\n",
            "./data/fse-ob/productcatalogservice_cpu/4/naive_bocpd.json\n",
            "Saved Anomalies: [272, 277, 343]\n",
            "Anomalies Detected: [272, 277, 343, 367, 517, 600]\n",
            "./data/fse-ob/productcatalogservice_cpu/1/naive_bocpd.json\n",
            "Saved Anomalies: [339, 353, 478]\n",
            "Anomalies Detected: [339, 353, 478, 480, 531, 600]\n",
            "./data/fse-ob/productcatalogservice_cpu/3/naive_bocpd.json\n",
            "Saved Anomalies: [337, 338, 417]\n",
            "Anomalies Detected: [337, 338, 417, 418, 509, 594, 600]\n",
            "./data/fse-ob/productcatalogservice_cpu/2/naive_bocpd.json\n",
            "Saved Anomalies: [184, 187, 340]\n",
            "Anomalies Detected: [184, 187, 340, 355, 363, 541, 600]\n",
            "./data/fse-ob/cartservice_delay/5/naive_bocpd.json\n",
            "Saved Anomalies: [322, 330, 340]\n",
            "Anomalies Detected: [322, 330, 340, 600]\n",
            "./data/fse-ob/cartservice_delay/4/naive_bocpd.json\n",
            "Saved Anomalies: [311, 317, 333]\n",
            "Anomalies Detected: [311, 317, 333, 451, 517, 521, 522, 600]\n",
            "./data/fse-ob/cartservice_delay/1/naive_bocpd.json\n",
            "Saved Anomalies: [97, 98, 333]\n",
            "Anomalies Detected: [97, 98, 333, 509, 511, 600]\n",
            "./data/fse-ob/cartservice_delay/3/naive_bocpd.json\n",
            "Saved Anomalies: [73, 75, 165]\n",
            "Anomalies Detected: [73, 75, 165, 168, 337, 338, 600]\n",
            "./data/fse-ob/cartservice_delay/2/naive_bocpd.json\n",
            "Saved Anomalies: [335, 600]\n",
            "Anomalies Detected: [335, 600]\n",
            "./data/fse-ob/productcatalogservice_mem/5/naive_bocpd.json\n",
            "Saved Anomalies: [345, 383, 470]\n",
            "Anomalies Detected: [345, 383, 470, 556, 577, 580, 600]\n",
            "./data/fse-ob/productcatalogservice_mem/4/naive_bocpd.json\n",
            "Saved Anomalies: [339, 473, 475]\n",
            "Anomalies Detected: [339, 473, 475, 517, 518, 600]\n",
            "./data/fse-ob/productcatalogservice_mem/1/naive_bocpd.json\n",
            "Saved Anomalies: [325, 340, 341]\n",
            "Anomalies Detected: [325, 340, 341, 509, 600]\n",
            "./data/fse-ob/productcatalogservice_mem/3/naive_bocpd.json\n",
            "Saved Anomalies: [344, 522, 581]\n",
            "Anomalies Detected: [344, 522, 581, 584, 600]\n",
            "./data/fse-ob/productcatalogservice_mem/2/naive_bocpd.json\n",
            "Saved Anomalies: [230, 233, 337]\n",
            "Anomalies Detected: [230, 233, 337, 594, 599]\n",
            "./data/fse-ob/checkoutservice_mem/5/naive_bocpd.json\n",
            "Saved Anomalies: [326, 347, 393]\n",
            "Anomalies Detected: [326, 347, 393, 600]\n",
            "./data/fse-ob/checkoutservice_mem/4/naive_bocpd.json\n",
            "Saved Anomalies: [326, 353, 370]\n",
            "Anomalies Detected: [326, 353, 370, 379, 600]\n",
            "./data/fse-ob/checkoutservice_mem/1/naive_bocpd.json\n",
            "Saved Anomalies: [333, 577, 600]\n",
            "Anomalies Detected: [333, 577, 600]\n",
            "./data/fse-ob/checkoutservice_mem/3/naive_bocpd.json\n",
            "Saved Anomalies: [348, 349, 350]\n",
            "Anomalies Detected: [348, 349, 350, 464, 485, 486, 555, 557, 559, 600]\n",
            "./data/fse-ob/checkoutservice_mem/2/naive_bocpd.json\n",
            "Saved Anomalies: [357, 358, 373]\n",
            "Anomalies Detected: [357, 358, 373, 476, 477, 600]\n",
            "./data/fse-ob/paymentservice_loss/5/naive_bocpd.json\n",
            "Saved Anomalies: [555, 568, 600]\n",
            "Anomalies Detected: [555, 568, 600]\n",
            "./data/fse-ob/paymentservice_loss/4/naive_bocpd.json\n",
            "Saved Anomalies: [133, 227, 250]\n",
            "Anomalies Detected: [133, 227, 250, 271, 341, 384, 397, 473, 547, 600]\n",
            "./data/fse-ob/paymentservice_loss/1/naive_bocpd.json\n",
            "Saved Anomalies: [332, 339, 415]\n",
            "Anomalies Detected: [332, 339, 415, 472, 504, 561, 574, 600]\n",
            "./data/fse-ob/paymentservice_loss/3/naive_bocpd.json\n",
            "Saved Anomalies: [384, 385, 506]\n",
            "Anomalies Detected: [384, 385, 506, 588, 598, 600]\n",
            "./data/fse-ob/paymentservice_loss/2/naive_bocpd.json\n",
            "Saved Anomalies: [227, 231, 316]\n",
            "Anomalies Detected: [227, 231, 316, 317, 341, 342, 351, 352, 357, 358, 456, 457, 560, 591, 600]\n",
            "./data/fse-ob/paymentservice_cpu/5/naive_bocpd.json\n",
            "Saved Anomalies: [242, 319, 453]\n",
            "Anomalies Detected: [242, 319, 453, 455, 524, 525, 529, 530, 534, 535, 600]\n",
            "./data/fse-ob/paymentservice_cpu/4/naive_bocpd.json\n",
            "Saved Anomalies: [357, 375, 378]\n",
            "Anomalies Detected: [357, 375, 378, 518, 519, 520, 534, 545, 549, 550, 571, 575, 579, 582, 600]\n",
            "./data/fse-ob/paymentservice_cpu/1/naive_bocpd.json\n",
            "Saved Anomalies: [324, 425, 433]\n",
            "Anomalies Detected: [324, 425, 433, 488, 600]\n",
            "./data/fse-ob/paymentservice_cpu/3/naive_bocpd.json\n",
            "Saved Anomalies: [395, 404, 600]\n",
            "Anomalies Detected: [395, 404, 600]\n",
            "./data/fse-ob/paymentservice_cpu/2/naive_bocpd.json\n",
            "Saved Anomalies: [367, 390, 404]\n",
            "Anomalies Detected: [367, 390, 404, 515, 536, 551, 553, 600]\n",
            "./data/fse-ob/cartservice_mem/5/naive_bocpd.json\n",
            "Saved Anomalies: [345, 419, 535]\n",
            "Anomalies Detected: [345, 419, 535, 539, 566, 567, 600]\n",
            "./data/fse-ob/cartservice_mem/4/naive_bocpd.json\n",
            "Saved Anomalies: [345, 348, 357]\n",
            "Anomalies Detected: [345, 348, 357, 496, 497, 498, 515, 600]\n",
            "./data/fse-ob/cartservice_mem/1/naive_bocpd.json\n",
            "Saved Anomalies: [339, 343, 430]\n",
            "Anomalies Detected: [339, 343, 430, 600]\n",
            "./data/fse-ob/cartservice_mem/3/naive_bocpd.json\n",
            "Saved Anomalies: [347, 352, 377]\n",
            "Anomalies Detected: [347, 352, 377, 450, 497, 600]\n",
            "./data/fse-ob/cartservice_mem/2/naive_bocpd.json\n",
            "Saved Anomalies: [324, 325, 336]\n",
            "Anomalies Detected: [324, 325, 336, 519, 544, 600]\n"
          ]
        }
      ],
      "source": [
        "def find_cps(maxes):\n",
        "    cps = []\n",
        "    for i in range(1, len(maxes)):\n",
        "        if abs(maxes[i] - maxes[i-1]) > 1:\n",
        "        # m_t = m_t-1 +   1\n",
        "            # print(i)\n",
        "            cps.append((i, abs(maxes[i] - maxes[i-1])))\n",
        "    return cps\n",
        "\n",
        "\n",
        "for data_path in glob.glob(\"./data/fse-ob/**/simple_data.csv\", recursive=True):\n",
        "    service_metric = basename(dirname(dirname(data_path)))\n",
        "    case_idx = basename(dirname(data_path))\n",
        "    data_dir = dirname(data_path)\n",
        "\n",
        "    # PREPARE DATA\n",
        "    data = pd.read_csv(data_path)\n",
        "\n",
        "    # read inject_time, cut data\n",
        "    with open(join(data_dir, \"inject_time.txt\")) as f:\n",
        "        inject_time = int(f.readlines()[0].strip())\n",
        "    normal_df = data[data[\"time\"] < inject_time].tail(300)\n",
        "    anomal_df = data[data[\"time\"] >= inject_time].head(300)\n",
        "    data = pd.concat([normal_df, anomal_df], ignore_index=True)\n",
        "\n",
        "    # drop extra columns\n",
        "    selected_cols = []\n",
        "    for c in data.columns:\n",
        "        if 'queue-master' in c or 'rabbitmq_' in c: continue\n",
        "        if \"latency-50\" in c or \"_error\" in c:\n",
        "            selected_cols.append(c)\n",
        "    data = data[selected_cols]\n",
        "\n",
        "    # handle na\n",
        "    data = drop_constant(data)\n",
        "    data = data.fillna(method=\"ffill\")\n",
        "    data = data.fillna(0)\n",
        "    for c in data.columns:\n",
        "        data[c] = (data[c] - np.min(data[c])) / (np.max(data[c]) - np.min(data[c]))\n",
        "    data = data.fillna(method=\"ffill\")\n",
        "    data = data.fillna(0)\n",
        "    data = data.to_numpy()\n",
        "\n",
        "    # RUN BOCPD\n",
        "    R, maxes = online_changepoint_detection(\n",
        "            data,\n",
        "            partial(constant_hazard, 50),\n",
        "            MultivariateT(dims=data.shape[1])\n",
        "    )\n",
        "    cps = find_cps(maxes)\n",
        "    cps = [p[0] for p in cps]\n",
        "\n",
        "    ############# READ ANOMALY DETECTION OUTPUT ###############\n",
        "    saved_anomalies_path = join(data_dir, \"naive_bocpd.json\")\n",
        "    print(saved_anomalies_path)\n",
        "    saved_anomalies = load_json(saved_anomalies_path)\n",
        "    saved_anomalies = [i[0] for i in saved_anomalies]\n",
        "    print(f\"Saved Anomalies: {saved_anomalies[:3]}\")\n",
        "\n",
        "    ############# RUN ANOMALY DETECTION ###############\n",
        "    print(f\"Anomalies Detected: {cps}\")\n",
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "env",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.2"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}