01_4_generate_data_HMM_arima.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate data from ARIMA - HMM mixture models\n",
    "Generate models with data partially from ARIMA and partially from HMM model. Requires loading HMM models from a file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "from hmmlearn import hmm\n",
    "from model_class import *\n",
    "\n",
    "# Select file with data generated from HMM - it is not important which exactly, it \n",
    "# must only contain any list of hmm.BaseHMM instances\n",
    "FILE_STORING_HMM_MODELS = \"generated_data_set2/HMM_GMM_data_ntrain5_min20_max50_1659090778.pkl\"\n",
    "\n",
    "DATA_DIR = \"./generated_data_set2\"\n",
    "GENERATING_MODEL_NAME = \"ARIMA_HMM_mix\"\n",
    "N_TRAIN_SAMPLES_PER_MODEL = 30\n",
    "N_TEST_SAMPLES_PER_MODEL = 30\n",
    "MIN_SAMPLE_LEN = 50\n",
    "MAX_SAMPLE_LEN = 150\n",
    "\n",
    "FILENAME_SUFFIX = f\"{GENERATING_MODEL_NAME}_data_ntrain{N_TRAIN_SAMPLES_PER_MODEL}_min{MIN_SAMPLE_LEN}_max{MAX_SAMPLE_LEN}\"\n",
    "DESCRIPTION = \"Mixture of ARIMA and HMM models\"\n",
    "\n",
    "time_index = int(time.time())\n",
    "\n",
    "if not os.path.exists(DATA_DIR):\n",
    "    os.makedirs(DATA_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load HMM models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(FILE_STORING_HMM_MODELS, \"rb\") as f:\n",
    "    data_hmm = pickle.load(f)\n",
    "    HMM_models_lst = data_hmm[\"models_lst\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = ARIMA_HMM_model(np.array([0.8]), np.array([0.0]), t = -5, HMM_instance= HMM_models_lst[0])\n",
    "model2 = ARIMA_HMM_model(np.array([0.8]), np.array([2.0]), t = 1, HMM_instance= HMM_models_lst[1], tau = 0.8)\n",
    "model3 = ARIMA_HMM_model(np.array([0.8]), np.array([4.0]), t = 3, HMM_instance= HMM_models_lst[6], tau = 0.8)\n",
    "\n",
    "model4 = ARIMA_HMM_model(np.array([-0.3]), np.array([0.0]), t = 1, mu = 5, sigma = 10, HMM_instance= HMM_models_lst[3])\n",
    "model5 = ARIMA_HMM_model(np.array([-0.3]), np.array([4.0]), t = 1, HMM_instance= HMM_models_lst[6], tau = 0.8)\n",
    "model6 = ARIMA_HMM_model(np.array([-0.3]), np.array([8.0]), t = 0, HMM_instance= HMM_models_lst[5])\n",
    "\n",
    "model7 = ARIMA_HMM_model(np.array([0.1]), np.array([0.0]), t = 4, HMM_instance= HMM_models_lst[6])\n",
    "model8 = ARIMA_HMM_model(np.array([0.4]), np.array([2.0, 3.0, -0.8]), t = 1, HMM_instance= HMM_models_lst[7])\n",
    "model9 = ARIMA_HMM_model(np.array([0.1]), np.array([2.0, -5.0]), t = 3, sigma = 4, HMM_instance= HMM_models_lst[8], tau = 0.6)\n",
    "\n",
    "\n",
    "models_lst = [model1, model2, model3, model4, model5, model6, model7, model8, model9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 100\n",
    "plt.rcParams[\"figure.figsize\"] = (40, 16)\n",
    "plt.rcParams['font.size'] = 25\n",
    "fig, axes = plt.subplots(3,3, sharey=\"all\")\n",
    "plt.suptitle(f\"Samples of length {n} ({GENERATING_MODEL_NAME})\")\n",
    "for i, ax in enumerate(axes.reshape(-1)):\n",
    "    m = models_lst[i]\n",
    "    try:\n",
    "        X = m.sample(n)\n",
    "    except Exception as e:\n",
    "        print(f\"Error in model {i}: {e}\")\n",
    "        continue\n",
    "        \n",
    "    ax.plot(X, color = \"black\")\n",
    "    ax.set_title(f\"Model [{i}]\")\n",
    "    ax.grid()\n",
    "\n",
    "fig.tight_layout()   \n",
    "plt.savefig(f\"./plots/models_{GENERATING_MODEL_NAME}_{time_index}.jpg\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate all data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data(models_lst: list[ARIMA_HMM_model], samples_per_model: int, max_sample_len: int, min_sample_len: int) -> tuple:\n",
    "    \"\"\"Generate data from list of hmm.BaseHMM instances.\n",
    "    Returns:\n",
    "        - labels_df: pd.DataFrame | sample_id | true_label | sample_len\n",
    "        - all_X_samples: a list with generated X samples\n",
    "    \"\"\"\n",
    "    all_X_samples = []\n",
    "    true_labels = []\n",
    "    sample_lengths = []\n",
    "    sample_ids = list(range(samples_per_model*len(models_lst)))\n",
    "    len_step = (max_sample_len-min_sample_len)//10\n",
    "    possible_lenghts = range(min_sample_len, max_sample_len + 1, len_step)\n",
    "    \n",
    "    for i in range(len(models_lst)):\n",
    "        model = models_lst[i]\n",
    "        for j in range(samples_per_model):\n",
    "            sample_len = np.random.choice(possible_lenghts)\n",
    "            X = model.sample(sample_len)\n",
    "            all_X_samples.append(X)\n",
    "            sample_lengths.append(sample_len)\n",
    "            true_labels.append(i)\n",
    "\n",
    "    labels_df = pd.DataFrame({\n",
    "        \"sample_id\": sample_ids,\n",
    "        \"true_label\": true_labels,\n",
    "        \"sample_len\": sample_lengths\n",
    "    })\n",
    "\n",
    "    return labels_df, all_X_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples_per_model = N_TRAIN_SAMPLES_PER_MODEL + N_TEST_SAMPLES_PER_MODEL\n",
    "labels_df, all_X_samples = generate_data(models_lst, samples_per_model, MAX_SAMPLE_LEN, MIN_SAMPLE_LEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_samples_ids = []\n",
    "test_samples_ids = []\n",
    "for label, sub_df in labels_df.groupby(\"true_label\"):\n",
    "    train_ids = sub_df.sample(N_TRAIN_SAMPLES_PER_MODEL).index.values\n",
    "    test_ids = sub_df.drop(train_ids).index.values\n",
    "    if len(test_ids) != N_TEST_SAMPLES_PER_MODEL:\n",
    "        raise Exception(f\"len(test_ids) {len(test_ids)} != N_TEST_SAMPLES_PER_MODEL {N_TEST_SAMPLES_PER_MODEL}\")\n",
    "    train_samples_ids.extend(train_ids)\n",
    "    test_samples_ids.extend(test_ids)\n",
    "    \n",
    "\n",
    "indices_split = {\n",
    "    \"train_samples_ids\": train_samples_ids, \n",
    "    \"test_samples_ids\": test_samples_ids}\n",
    "\n",
    "indices_splits_lst = [indices_split]\n",
    "print(len(indices_split[\"train_samples_ids\"]), len(indices_split[\"test_samples_ids\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_filename = f\"{FILENAME_SUFFIX}_{time_index}.pkl\"\n",
    "output_path = f\"{DATA_DIR}/{data_filename}\"\n",
    "\n",
    "metadata = {\n",
    "    \"N_TRAIN_SAMPLES_PER_MODEL\": N_TRAIN_SAMPLES_PER_MODEL,\n",
    "    \"N_TEST_SAMPLES_PER_MODEL\": N_TEST_SAMPLES_PER_MODEL,\n",
    "    \"MAX_SAMPLE_LEN\": MAX_SAMPLE_LEN,\n",
    "    \"MIN_SAMPLE_LEN\": MIN_SAMPLE_LEN,\n",
    "}\n",
    "\n",
    "data = {\n",
    "    'generating_model': GENERATING_MODEL_NAME,\n",
    "    'data_filename': data_filename,\n",
    "    'time_index': time_index,\n",
    "    'models_lst': models_lst,\n",
    "    'labels_df': labels_df,\n",
    "    'all_X_samples': all_X_samples,\n",
    "    'indices_splits_lst': indices_splits_lst,\n",
    "    'metadata': metadata,\n",
    "    'description': DESCRIPTION\n",
    "}\n",
    "\n",
    "with open(output_path, 'wb') as f:\n",
    "    pickle.dump(data, f)\n",
    "    print(f\"Data saved to {output_path}.\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "824dc94f4a21fb2b9b063d5374c6a5be71ea0704887dd3ffa4e3d703212d1775"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 ('hmm_dtw')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}