TTS_GAN_train_synth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "\n",
    "import torch\n",
    "# from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "from utils.data import get_hsm_dataset, get_solar_energy_dataset, get_fuel_prices_dataset, get_passengers_dataset, split_data, log_returns, DimUniversalStandardScaler\n",
    "from utils.metrics import MAPE, WAPE, MAE\n",
    "from utils.TTS_GAN import TTS_GAN_Generator, TTS_GAN_Discriminator, weights_init, train_TTS_GAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "hsm_dataset_path = Path(\"data/huge_stock_market_dataset/\")\n",
    "solar_energy_dataset_path = Path(\"data/solar_energy/\")\n",
    "fuel_prices_dataset_path = Path(\"data/fuel_prices/\")\n",
    "passengers_dataset_path = Path(\"data/air_passengers/\")\n",
    "models_dir = Path(\"models/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = gpu = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "\n",
    "lr = 2e-4\n",
    "wd = 0\n",
    "ctrl_lr = 3.5e-4\n",
    "beta1 = 0.0\n",
    "beta2 = 0.9\n",
    "max_epoch = 20\n",
    "latent_dim = 128\n",
    "batch_size = gen_batch_size = dis_batch_size = 64\n",
    "ema = 0.995\n",
    "ema_kimg = 500\n",
    "ema_warmup = 0\n",
    "world_size = 0\n",
    "rank = - 1\n",
    "print_freq = 50\n",
    "n_critic = 1\n",
    "phi = 1\n",
    "accumulated_times = g_accumulated_times = 1\n",
    "loss = \"standard\"\n",
    "seq_len = 150\n",
    "\n",
    "n_samples = 80  # number of samples generated by QuantGAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Series #0\n",
      "generator loss: -0.6101 discriminator loss:  1.3817\n",
      "Time Series #1\n",
      "generator loss: -0.4423 discriminator loss:  0.8679\n",
      "Time Series #2\n",
      "generator loss: -0.5821 discriminator loss:  1.5387\n",
      "Time Series #3\n",
      "generator loss: -0.5131 discriminator loss:  1.1995\n",
      "Time Series #4\n",
      "generator loss: -0.5394 discriminator loss:  1.2372\n",
      "Time Series #5\n",
      "generator loss: -0.4120 discriminator loss:  1.1106\n",
      "Time Series #6\n",
      "generator loss: -0.6126 discriminator loss:  1.7059\n",
      "Time Series #7\n",
      "generator loss: -0.3423 discriminator loss:  0.8353\n",
      "Time Series #8\n",
      "generator loss: -0.6023 discriminator loss:  1.3908\n",
      "Time Series #9\n",
      "generator loss: -0.4496 discriminator loss:  1.2067\n",
      "Time Series #10\n",
      "generator loss: -0.5178 discriminator loss:  1.3138\n",
      "Time Series #11\n",
      "generator loss: -0.4435 discriminator loss:  1.0226\n",
      "Time Series #12\n",
      "generator loss: -0.4819 discriminator loss:  1.1233\n",
      "Time Series #13\n",
      "generator loss: -0.4872 discriminator loss:  1.1253\n",
      "Time Series #14\n",
      "generator loss: -0.4918 discriminator loss:  1.2338\n",
      "Time Series #15\n",
      "generator loss: -0.4414 discriminator loss:  1.0923\n",
      "Time Series #16\n",
      "generator loss: -0.5944 discriminator loss:  1.3870\n",
      "Time Series #17\n",
      "generator loss: -0.5868 discriminator loss:  1.3599\n",
      "Time Series #18\n",
      "generator loss: -0.5792 discriminator loss:  1.2749\n",
      "Time Series #19\n",
      "generator loss: -0.4592 discriminator loss:  1.1003\n",
      "Time Series #20\n",
      "generator loss: -0.5840 discriminator loss:  1.4064\n",
      "Time Series #21\n",
      "generator loss: -0.5966 discriminator loss:  1.4071\n",
      "Time Series #22\n",
      "generator loss: -0.4674 discriminator loss:  1.1525\n",
      "Time Series #23\n",
      "generator loss: -0.5281 discriminator loss:  1.2160\n",
      "Time Series #24\n",
      "generator loss: -0.6106 discriminator loss:  1.4121\n",
      "Time Series #25\n",
      "generator loss: -0.5344 discriminator loss:  1.3048\n",
      "Time Series #26\n",
      "generator loss: -0.5916 discriminator loss:  1.3715\n",
      "Time Series #27\n",
      "generator loss: -0.5877 discriminator loss:  1.5596\n",
      "Time Series #28\n",
      "generator loss: -0.5480 discriminator loss:  1.2704\n",
      "Time Series #29\n",
      "generator loss: -0.4861 discriminator loss:  1.1331\n",
      "Time Series #30\n",
      "generator loss: -0.6330 discriminator loss:  1.5410\n",
      "Time Series #31\n",
      "generator loss: -0.4978 discriminator loss:  1.1943\n",
      "Time Series #32\n",
      "generator loss: -0.5284 discriminator loss:  1.2055\n",
      "Time Series #33\n",
      "generator loss: -0.5262 discriminator loss:  1.6694\n",
      "Time Series #34\n",
      "generator loss: -0.5983 discriminator loss:  1.4100\n",
      "Time Series #35\n",
      "generator loss: -0.5187 discriminator loss:  1.2419\n",
      "Time Series #36\n",
      "generator loss: -0.6106 discriminator loss:  1.3606\n",
      "Time Series #37\n",
      "generator loss: -0.5746 discriminator loss:  1.3978\n",
      "Time Series #38\n",
      "generator loss: -0.5894 discriminator loss:  1.3415\n",
      "Time Series #39\n",
      "generator loss: -0.6007 discriminator loss:  1.4033\n",
      "Time Series #40\n",
      "generator loss: -0.5388 discriminator loss:  1.2410\n",
      "Time Series #41\n",
      "generator loss: -0.5946 discriminator loss:  1.3871\n",
      "Time Series #42\n",
      "generator loss: -0.5660 discriminator loss:  1.3570\n",
      "Time Series #43\n",
      "generator loss: -0.5343 discriminator loss:  1.8042\n",
      "Time Series #44\n",
      "generator loss: -0.6114 discriminator loss:  1.4329\n",
      "Time Series #45\n",
      "generator loss: -0.5541 discriminator loss:  1.3134\n",
      "Time Series #46\n",
      "generator loss: -0.5228 discriminator loss:  1.3291\n",
      "Time Series #47\n",
      "generator loss: -0.5177 discriminator loss:  1.2046\n",
      "Time Series #48\n",
      "generator loss: -0.5307 discriminator loss:  1.1970\n",
      "Time Series #49\n",
      "generator loss: -0.4191 discriminator loss:  1.0979\n",
      "Time Series #50\n",
      "generator loss: -0.6074 discriminator loss:  1.4208\n",
      "Time Series #51\n",
      "generator loss: -0.6041 discriminator loss:  1.3723\n",
      "Time Series #52\n",
      "generator loss: -0.4051 discriminator loss:  1.0065\n",
      "Time Series #53\n",
      "generator loss: -0.5704 discriminator loss:  1.3440\n",
      "Time Series #54\n",
      "generator loss: -0.4637 discriminator loss:  1.0878\n",
      "Time Series #55\n",
      "generator loss: -0.6038 discriminator loss:  1.3038\n",
      "Time Series #56\n",
      "generator loss: -0.5368 discriminator loss:  1.2039\n",
      "Time Series #57\n",
      "generator loss: -0.5890 discriminator loss:  1.3294\n",
      "Time Series #58\n",
      "generator loss: -0.6111 discriminator loss:  1.4108\n",
      "Time Series #59\n",
      "generator loss: -0.5918 discriminator loss:  1.5896\n",
      "Time Series #60\n",
      "generator loss: -0.5542 discriminator loss:  1.1102\n",
      "Time Series #61\n",
      "generator loss: -0.5337 discriminator loss:  1.2809\n",
      "Time Series #62\n",
      "generator loss: -0.5106 discriminator loss:  1.3373\n",
      "Time Series #63\n",
      "generator loss: -0.5194 discriminator loss:  1.2290\n",
      "Time Series #64\n",
      "generator loss: -0.5775 discriminator loss:  1.3300\n",
      "Time Series #65\n",
      "generator loss: -0.5623 discriminator loss:  1.3842\n",
      "Time Series #66\n",
      "generator loss: -0.6022 discriminator loss:  1.4404\n",
      "Time Series #67\n",
      "generator loss: -0.6501 discriminator loss:  1.4839\n",
      "Time Series #68\n",
      "generator loss: -0.6220 discriminator loss:  1.4624\n",
      "Time Series #69\n",
      "generator loss: -0.5757 discriminator loss:  1.3806\n",
      "Time Series #70\n",
      "generator loss: -0.4698 discriminator loss:  1.2310\n",
      "Time Series #71\n",
      "generator loss: -0.5894 discriminator loss:  1.3302\n",
      "Time Series #72\n",
      "generator loss: -0.6025 discriminator loss:  1.3625\n",
      "Time Series #73\n",
      "generator loss: -0.5095 discriminator loss:  1.1644\n",
      "Time Series #74\n",
      "generator loss: -0.5347 discriminator loss:  1.3515\n",
      "Time Series #75\n",
      "generator loss: -0.5886 discriminator loss:  1.4302\n",
      "Time Series #76\n",
      "generator loss: -0.5881 discriminator loss:  1.3665\n",
      "Time Series #77\n",
      "generator loss: -0.5896 discriminator loss:  1.3484\n",
      "Time Series #78\n",
      "generator loss: -0.6014 discriminator loss:  1.4159\n",
      "Time Series #79\n",
      "generator loss: -0.6076 discriminator loss:  1.3931\n",
      "Time Series #80\n",
      "generator loss: -0.4791 discriminator loss:  1.1082\n",
      "Time Series #81\n",
      "generator loss: -0.6031 discriminator loss:  1.5720\n",
      "Time Series #82\n",
      "generator loss: -0.5605 discriminator loss:  1.2437\n",
      "Time Series #83\n",
      "generator loss: -0.5552 discriminator loss:  1.2539\n",
      "Time Series #84\n",
      "generator loss: -0.5428 discriminator loss:  1.2751\n",
      "Time Series #85\n",
      "generator loss: -0.4131 discriminator loss:  1.0389\n",
      "Time Series #86\n",
      "generator loss: -0.5601 discriminator loss:  1.3062\n",
      "Time Series #87\n",
      "generator loss: -0.4426 discriminator loss:  1.0320\n",
      "Time Series #88\n",
      "generator loss: -0.4194 discriminator loss:  1.1711\n",
      "Time Series #89\n",
      "generator loss: -0.4867 discriminator loss:  1.2279\n",
      "Time Series #90\n",
      "generator loss: -0.5230 discriminator loss:  1.2434\n",
      "Time Series #91\n",
      "generator loss: -0.5063 discriminator loss:  1.1453\n",
      "Time Series #92\n",
      "generator loss: -0.5237 discriminator loss:  1.2767\n",
      "Time Series #93\n",
      "generator loss: -0.5034 discriminator loss:  1.2094\n",
      "Time Series #94\n",
      "generator loss: -0.4396 discriminator loss:  1.1721\n",
      "Time Series #95\n",
      "generator loss: -0.6328 discriminator loss:  1.4318\n",
      "Time Series #96\n",
      "generator loss: -0.5855 discriminator loss:  1.3544\n",
      "Time Series #97\n",
      "generator loss: -0.6102 discriminator loss:  1.3930\n",
      "Time Series #98\n",
      "generator loss: -0.5479 discriminator loss:  1.2710\n",
      "Time Series #99\n",
      "generator loss: -0.5224 discriminator loss:  1.2259\n"
     ]
    }
   ],
   "source": [
    "ts_iterator = get_hsm_dataset(hsm_dataset_path, selected_files=hsm_dataset_path / \"selected100.csv\")\n",
    "synthetic_path = hsm_dataset_path / \"synthetic/TTS_GAN/\"\n",
    "\n",
    "start_point = 0\n",
    "for _ in range(start_point): next(ts_iterator)\n",
    "\n",
    "for ts_index, time_series in enumerate(ts_iterator, start=start_point):\n",
    "    print(f\"Time Series #{ts_index}\")\n",
    "    \n",
    "    # train_ts = log_returns(time_series)\n",
    "    train_ts = time_series.values\n",
    "\n",
    "    # using sequences of seq_len to train model\n",
    "    train_ts = np.array([train_ts[i: i + seq_len] for i in range(len(train_ts) - seq_len)])\n",
    "\n",
    "    scaler = DimUniversalStandardScaler()\n",
    "    train_ts = scaler.fit_transform(train_ts)\n",
    "\n",
    "    train_dl = torch.utils.data.DataLoader(torch.from_numpy(train_ts.reshape(- 1, 1, 1, seq_len)).to(device), batch_size=batch_size, shuffle=True)\n",
    "\n",
    "    TTS_GAN_gen = TTS_GAN_Generator(seq_len=seq_len, channels=1, latent_dim=latent_dim, ).to(device)\n",
    "    TTS_GAN_dis = TTS_GAN_Discriminator(seq_length=seq_len, in_channels=1).to(device)\n",
    "\n",
    "    gen_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_gen.parameters()), lr)\n",
    "    dis_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_dis.parameters()), lr)\n",
    "    \n",
    "    for epoch in range(max_epoch):\n",
    "        losses = train_TTS_GAN(globals(), TTS_GAN_gen, TTS_GAN_dis, gen_optimizer, dis_optimizer, train_dl, epoch)\n",
    "    tqdm.write(f\"generator loss: {losses[0]: 0.4f} discriminator loss: {losses[1]: 0.4f}\")\n",
    "    del dis_optimizer, gen_optimizer, TTS_GAN_dis, train_dl\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "    synth_data = []\n",
    "    with torch.no_grad():\n",
    "        for _ in range(n_samples):\n",
    "            z = torch.cuda.FloatTensor(np.random.normal(0, 1, (1, latent_dim))).cuda(device, non_blocking=True)\n",
    "            synth_data.append(TTS_GAN_gen(z).cpu().numpy())\n",
    "            del z\n",
    "            torch.cuda.empty_cache()\n",
    "    np.save(synthetic_path / f\"selected{ts_index}.npy\", scaler.inverse_transform(np.row_stack(synth_data)))\n",
    "\n",
    "    del TTS_GAN_gen, synth_data\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Time: ~53 min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Series #0\n",
      "generator loss: -0.6003 discriminator loss:  1.3448\n",
      "Time Series #1\n",
      "generator loss: -0.5580 discriminator loss:  1.2181\n",
      "Time Series #2\n",
      "generator loss: -0.5600 discriminator loss:  1.3332\n",
      "Time Series #3\n",
      "generator loss: -0.5556 discriminator loss:  1.3992\n",
      "Time Series #4\n",
      "generator loss: -0.5871 discriminator loss:  1.4527\n",
      "Time Series #5\n",
      "generator loss: -0.5823 discriminator loss:  1.4358\n",
      "Time Series #6\n",
      "generator loss: -0.5779 discriminator loss:  1.3682\n",
      "Time Series #7\n",
      "generator loss: -0.5697 discriminator loss:  1.3566\n",
      "Time Series #8\n",
      "generator loss: -0.5631 discriminator loss:  1.3004\n",
      "Time Series #9\n",
      "generator loss: -0.5907 discriminator loss:  1.3968\n",
      "Time Series #10\n",
      "generator loss: -0.5541 discriminator loss:  1.3250\n",
      "Time Series #11\n",
      "generator loss: -0.5514 discriminator loss:  1.2524\n",
      "Time Series #12\n",
      "generator loss: -0.6169 discriminator loss:  1.4487\n",
      "Time Series #13\n",
      "generator loss: -0.5068 discriminator loss:  1.1734\n",
      "Time Series #14\n",
      "generator loss: -0.5446 discriminator loss:  1.2267\n",
      "Time Series #15\n",
      "generator loss: -0.5906 discriminator loss:  1.3829\n",
      "Time Series #16\n",
      "generator loss: -0.6103 discriminator loss:  1.3451\n",
      "Time Series #17\n",
      "generator loss: -0.5971 discriminator loss:  1.4656\n",
      "Time Series #18\n",
      "generator loss: -0.5501 discriminator loss:  1.3670\n",
      "Time Series #19\n",
      "generator loss: -0.5878 discriminator loss:  1.4830\n",
      "Time Series #20\n",
      "generator loss: -0.5546 discriminator loss:  1.3157\n",
      "Time Series #21\n",
      "generator loss: -0.5317 discriminator loss:  1.2669\n",
      "Time Series #22\n",
      "generator loss: -0.5364 discriminator loss:  1.3087\n",
      "Time Series #23\n",
      "generator loss: -0.6274 discriminator loss:  1.5473\n",
      "Time Series #24\n",
      "generator loss: -0.5541 discriminator loss:  1.3100\n",
      "Time Series #25\n",
      "generator loss: -0.5545 discriminator loss:  1.3436\n",
      "Time Series #26\n",
      "generator loss: -0.6090 discriminator loss:  1.5412\n",
      "Time Series #27\n",
      "generator loss: -0.5925 discriminator loss:  1.4360\n",
      "Time Series #28\n",
      "generator loss: -0.5552 discriminator loss:  1.4690\n",
      "Time Series #29\n",
      "generator loss: -0.5365 discriminator loss:  1.2239\n",
      "Time Series #30\n",
      "generator loss: -0.5792 discriminator loss:  1.3923\n",
      "Time Series #31\n",
      "generator loss: -0.5602 discriminator loss:  1.2748\n",
      "Time Series #32\n",
      "generator loss: -0.5175 discriminator loss:  1.2950\n",
      "Time Series #33\n",
      "generator loss: -0.6126 discriminator loss:  1.4645\n",
      "Time Series #34\n",
      "generator loss: -0.5794 discriminator loss:  1.3658\n",
      "Time Series #35\n",
      "generator loss: -0.5599 discriminator loss:  1.3243\n",
      "Time Series #36\n",
      "generator loss: -0.5785 discriminator loss:  1.3927\n",
      "Time Series #37\n",
      "generator loss: -0.6025 discriminator loss:  1.4644\n",
      "Time Series #38\n",
      "generator loss: -0.5821 discriminator loss:  1.4690\n",
      "Time Series #39\n",
      "generator loss: -0.5496 discriminator loss:  1.2227\n",
      "Time Series #40\n",
      "generator loss: -0.5723 discriminator loss:  1.4295\n",
      "Time Series #41\n",
      "generator loss: -0.5907 discriminator loss:  1.3846\n",
      "Time Series #42\n",
      "generator loss: -0.5765 discriminator loss:  1.3567\n",
      "Time Series #43\n",
      "generator loss: -0.5769 discriminator loss:  1.2900\n",
      "Time Series #44\n",
      "generator loss: -0.5864 discriminator loss:  1.4039\n",
      "Time Series #45\n",
      "generator loss: -0.5671 discriminator loss:  1.3976\n",
      "Time Series #46\n",
      "generator loss: -0.5609 discriminator loss:  1.2172\n",
      "Time Series #47\n",
      "generator loss: -0.5804 discriminator loss:  1.3190\n",
      "Time Series #48\n",
      "generator loss: -0.5931 discriminator loss:  1.4219\n",
      "Time Series #49\n",
      "generator loss: -0.5624 discriminator loss:  1.3644\n"
     ]
    }
   ],
   "source": [
    "ts_iterator = get_solar_energy_dataset(solar_energy_dataset_path)\n",
    "synthetic_path = solar_energy_dataset_path / \"synthetic/TTS_GAN/\"\n",
    "max_epoch = 4\n",
    "\n",
    "start_point = 0\n",
    "for _ in range(start_point): next(ts_iterator)\n",
    "\n",
    "for ts_index, time_series in enumerate(ts_iterator, start=start_point):\n",
    "    print(f\"Time Series #{ts_index}\")\n",
    "    \n",
    "    # train_ts = log_returns(time_series + 1e-9)\n",
    "    train_ts = time_series.values\n",
    "\n",
    "    # using sequences of seq_len to train model\n",
    "    train_ts = np.array([train_ts[i: i + seq_len] for i in range(len(train_ts) - seq_len)])\n",
    "\n",
    "    scaler = DimUniversalStandardScaler()\n",
    "    train_ts = scaler.fit_transform(train_ts)\n",
    "    train_dl = torch.utils.data.DataLoader(torch.from_numpy(train_ts.reshape(- 1, 1, 1, seq_len)).to(device), batch_size=batch_size, shuffle=True)\n",
    "\n",
    "    TTS_GAN_gen = TTS_GAN_Generator(seq_len=seq_len, channels=1, latent_dim=latent_dim, ).to(device)\n",
    "    TTS_GAN_dis = TTS_GAN_Discriminator(seq_length=seq_len, in_channels=1).to(device)\n",
    "\n",
    "    gen_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_gen.parameters()), lr)\n",
    "    dis_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_dis.parameters()), lr)\n",
    "    \n",
    "    for epoch in range(max_epoch):\n",
    "        losses = train_TTS_GAN(globals(), TTS_GAN_gen, TTS_GAN_dis, gen_optimizer, dis_optimizer, train_dl, epoch)\n",
    "    tqdm.write(f\"generator loss: {losses[0]: 0.4f} discriminator loss: {losses[1]: 0.4f}\")\n",
    "    del dis_optimizer, gen_optimizer, TTS_GAN_dis, train_dl\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "    samples_to_gen = n_samples\n",
    "    synth_data = []\n",
    "    with torch.no_grad():\n",
    "        for _ in range(samples_to_gen):\n",
    "            z = torch.cuda.FloatTensor(np.random.normal(0, 1, (1, latent_dim))).cuda(device, non_blocking=True)\n",
    "            synth_data.append(TTS_GAN_gen(z).cpu().numpy())\n",
    "            del z\n",
    "            torch.cuda.empty_cache()\n",
    "    np.save(synthetic_path / f\"selected{ts_index}.npy\", scaler.inverse_transform(np.row_stack(synth_data)))\n",
    "\n",
    "    del TTS_GAN_gen, synth_data\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Time: 160 min - 10 ts 10k points; 8 min - 30 ts 3k points; 3min 50 ts 1k points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Series #0\n",
      "generator loss: -0.5187 discriminator loss:  1.2032\n",
      "Time Series #1\n",
      "generator loss: -0.4393 discriminator loss:  1.0658\n",
      "Time Series #2\n",
      "generator loss: -0.4639 discriminator loss:  1.0994\n",
      "Time Series #3\n",
      "generator loss: -0.5546 discriminator loss:  1.4700\n",
      "Time Series #4\n",
      "generator loss: -0.3544 discriminator loss:  0.8858\n",
      "Time Series #5\n",
      "generator loss: -0.4610 discriminator loss:  1.1516\n",
      "Time Series #6\n",
      "generator loss: -0.4471 discriminator loss:  0.9922\n",
      "Time Series #7\n",
      "generator loss: -0.4453 discriminator loss:  0.9839\n"
     ]
    }
   ],
   "source": [
    "ts_iterator = get_fuel_prices_dataset(fuel_prices_dataset_path)\n",
    "synthetic_path = fuel_prices_dataset_path / \"synthetic/TTS_GAN/\"\n",
    "max_epoch = 10\n",
    "\n",
    "start_point = 0\n",
    "for _ in range(start_point): next(ts_iterator)\n",
    "\n",
    "for ts_index, time_series in enumerate(ts_iterator, start=start_point):\n",
    "    print(f\"Time Series #{ts_index}\")\n",
    "    \n",
    "    # train_ts = log_returns(time_series + 1e-9)\n",
    "    train_ts = time_series.values\n",
    "\n",
    "    # using sequences of seq_len to train model\n",
    "    train_ts = np.array([train_ts[i: i + seq_len] for i in range(len(train_ts) - seq_len)])\n",
    "\n",
    "    scaler = DimUniversalStandardScaler()\n",
    "    train_ts = scaler.fit_transform(train_ts)\n",
    "    train_dl = torch.utils.data.DataLoader(torch.from_numpy(train_ts.reshape(- 1, 1, 1, seq_len)).to(device), batch_size=batch_size, shuffle=True)\n",
    "\n",
    "    TTS_GAN_gen = TTS_GAN_Generator(seq_len=seq_len, channels=1, latent_dim=latent_dim, ).to(device)\n",
    "    TTS_GAN_dis = TTS_GAN_Discriminator(seq_length=seq_len, in_channels=1).to(device)\n",
    "\n",
    "    gen_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_gen.parameters()), lr)\n",
    "    dis_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_dis.parameters()), lr)\n",
    "    \n",
    "    for epoch in range(max_epoch):\n",
    "        losses = train_TTS_GAN(globals(), TTS_GAN_gen, TTS_GAN_dis, gen_optimizer, dis_optimizer, train_dl, epoch)\n",
    "    tqdm.write(f\"generator loss: {losses[0]: 0.4f} discriminator loss: {losses[1]: 0.4f}\")\n",
    "    del dis_optimizer, gen_optimizer, TTS_GAN_dis, train_dl\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "    samples_to_gen = n_samples\n",
    "    synth_data = []\n",
    "    with torch.no_grad():\n",
    "        for _ in range(samples_to_gen):\n",
    "            z = torch.cuda.FloatTensor(np.random.normal(0, 1, (1, latent_dim))).cuda(device, non_blocking=True)\n",
    "            synth_data.append(TTS_GAN_gen(z).cpu().numpy())\n",
    "            del z\n",
    "            torch.cuda.empty_cache()\n",
    "    np.save(synthetic_path / f\"selected{ts_index}.npy\", scaler.inverse_transform(np.row_stack(synth_data)))\n",
    "\n",
    "    del TTS_GAN_gen, synth_data\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Time: 72 sec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Series #0\n",
      "generator loss: -0.5580 discriminator loss:  1.4303\n",
      "Time Series #1\n",
      "generator loss: -0.4787 discriminator loss:  1.1723\n",
      "Time Series #2\n",
      "generator loss: -0.5233 discriminator loss:  1.3867\n",
      "Time Series #3\n",
      "generator loss: -0.5166 discriminator loss:  1.3812\n",
      "Time Series #4\n",
      "generator loss: -0.5367 discriminator loss:  1.3690\n",
      "Time Series #5\n",
      "generator loss: -0.4585 discriminator loss:  1.3159\n",
      "Time Series #6\n",
      "generator loss: -0.4619 discriminator loss:  1.3334\n",
      "Time Series #7\n",
      "generator loss: -0.4988 discriminator loss:  1.2971\n",
      "Time Series #8\n",
      "generator loss: -0.5702 discriminator loss:  1.4272\n",
      "Time Series #9\n",
      "generator loss: -0.5297 discriminator loss:  1.3865\n",
      "Time Series #10\n",
      "generator loss: -0.5642 discriminator loss:  1.3835\n",
      "Time Series #11\n",
      "generator loss: -0.4673 discriminator loss:  1.1893\n",
      "Time Series #12\n",
      "generator loss: -0.5828 discriminator loss:  1.3618\n",
      "Time Series #13\n",
      "generator loss: -0.5890 discriminator loss:  1.4060\n",
      "Time Series #14\n",
      "generator loss: -0.4848 discriminator loss:  1.2006\n",
      "Time Series #15\n",
      "generator loss: -0.5294 discriminator loss:  1.3942\n",
      "Time Series #16\n",
      "generator loss: -0.4621 discriminator loss:  1.1737\n",
      "Time Series #17\n",
      "generator loss: -0.5342 discriminator loss:  1.4370\n",
      "Time Series #18\n",
      "generator loss: -0.5317 discriminator loss:  1.4492\n",
      "Time Series #19\n",
      "generator loss: -0.5606 discriminator loss:  1.3094\n",
      "Time Series #20\n",
      "generator loss: -0.5536 discriminator loss:  1.4764\n",
      "Time Series #21\n",
      "generator loss: -0.5914 discriminator loss:  1.3571\n",
      "Time Series #22\n",
      "generator loss: -0.5158 discriminator loss:  1.3642\n",
      "Time Series #23\n",
      "generator loss: -0.5887 discriminator loss:  1.6504\n",
      "Time Series #24\n",
      "generator loss: -0.5161 discriminator loss:  1.3541\n",
      "Time Series #25\n",
      "generator loss: -0.5499 discriminator loss:  1.1908\n",
      "Time Series #26\n",
      "generator loss: -0.5404 discriminator loss:  1.4714\n",
      "Time Series #27\n",
      "generator loss: -0.5247 discriminator loss:  1.3058\n",
      "Time Series #28\n",
      "generator loss: -0.5285 discriminator loss:  1.3742\n",
      "Time Series #29\n",
      "generator loss: -0.5473 discriminator loss:  1.4091\n",
      "Time Series #30\n",
      "generator loss: -0.4640 discriminator loss:  1.2329\n",
      "Time Series #31\n",
      "generator loss: -0.5208 discriminator loss:  1.3857\n",
      "Time Series #32\n",
      "generator loss: -0.5421 discriminator loss:  1.3440\n",
      "Time Series #33\n",
      "generator loss: -0.5359 discriminator loss:  1.4815\n",
      "Time Series #34\n",
      "generator loss: -0.5371 discriminator loss:  1.4281\n",
      "Time Series #35\n",
      "generator loss: -0.5175 discriminator loss:  1.3298\n",
      "Time Series #36\n",
      "generator loss: -0.5949 discriminator loss:  1.4015\n",
      "Time Series #37\n",
      "generator loss: -0.4717 discriminator loss:  1.3337\n",
      "Time Series #38\n",
      "generator loss: -0.4048 discriminator loss:  1.1774\n",
      "Time Series #39\n",
      "generator loss: -0.4043 discriminator loss:  1.1239\n",
      "Time Series #40\n",
      "generator loss: -0.5230 discriminator loss:  1.3270\n",
      "Time Series #41\n",
      "generator loss: -0.5612 discriminator loss:  1.4011\n",
      "Time Series #42\n",
      "generator loss: -0.4716 discriminator loss:  1.1877\n",
      "Time Series #43\n",
      "generator loss: -0.5056 discriminator loss:  1.4255\n",
      "Time Series #44\n",
      "generator loss: -0.5486 discriminator loss:  1.3777\n",
      "Time Series #45\n",
      "generator loss: -0.5444 discriminator loss:  1.3543\n",
      "Time Series #46\n",
      "generator loss: -0.4750 discriminator loss:  1.2032\n",
      "Time Series #47\n",
      "generator loss: -0.5095 discriminator loss:  1.3335\n",
      "Time Series #48\n",
      "generator loss: -0.5479 discriminator loss:  1.4645\n",
      "Time Series #49\n",
      "generator loss: -0.4858 discriminator loss:  1.3500\n"
     ]
    }
   ],
   "source": [
    "ts_iterator = get_passengers_dataset(passengers_dataset_path)\n",
    "synthetic_path = passengers_dataset_path / \"synthetic/TTS_GAN/\"\n",
    "max_epoch = 10\n",
    "\n",
    "start_point = 0\n",
    "for _ in range(start_point): next(ts_iterator)\n",
    "\n",
    "for ts_index, time_series in enumerate(ts_iterator, start=start_point):\n",
    "    print(f\"Time Series #{ts_index}\")\n",
    "    \n",
    "    # train_ts = log_returns(time_series + 1e-9)\n",
    "    train_ts = time_series.values\n",
    "\n",
    "    # using sequences of seq_len to train model\n",
    "    train_ts = np.array([train_ts[i: i + seq_len] for i in range(len(train_ts) - seq_len)])\n",
    "\n",
    "    scaler = DimUniversalStandardScaler()\n",
    "    train_ts = scaler.fit_transform(train_ts)\n",
    "    train_dl = torch.utils.data.DataLoader(torch.from_numpy(train_ts.reshape(- 1, 1, 1, seq_len)).to(device), batch_size=batch_size, shuffle=True)\n",
    "\n",
    "    TTS_GAN_gen = TTS_GAN_Generator(seq_len=seq_len, channels=1, latent_dim=latent_dim, ).to(device)\n",
    "    TTS_GAN_dis = TTS_GAN_Discriminator(seq_length=seq_len, in_channels=1).to(device)\n",
    "\n",
    "    gen_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_gen.parameters()), lr)\n",
    "    dis_optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, TTS_GAN_dis.parameters()), lr)\n",
    "    \n",
    "    for epoch in range(max_epoch):\n",
    "        losses = train_TTS_GAN(globals(), TTS_GAN_gen, TTS_GAN_dis, gen_optimizer, dis_optimizer, train_dl, epoch)\n",
    "    tqdm.write(f\"generator loss: {losses[0]: 0.4f} discriminator loss: {losses[1]: 0.4f}\")\n",
    "    del dis_optimizer, gen_optimizer, TTS_GAN_dis, train_dl\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "    samples_to_gen = n_samples\n",
    "    synth_data = []\n",
    "    with torch.no_grad():\n",
    "        for _ in range(samples_to_gen):\n",
    "            z = torch.cuda.FloatTensor(np.random.normal(0, 1, (1, latent_dim))).cuda(device, non_blocking=True)\n",
    "            synth_data.append(TTS_GAN_gen(z).cpu().numpy())\n",
    "            del z\n",
    "            torch.cuda.empty_cache()\n",
    "    np.save(synthetic_path / f\"selected{ts_index}.npy\", scaler.inverse_transform(np.row_stack(synth_data)))\n",
    "\n",
    "    del TTS_GAN_gen, synth_data\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from utils.data import get_hsm_dataset, get_solar_energy_dataset, get_fuel_prices_dataset, get_passengers_dataset, split_data, log_returns\n",
    "from utils.synth_eval import eval_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "hsm_dataset_path = Path(\"data/huge_stock_market_dataset/\")\n",
    "solar_energy_dataset_path = Path(\"data/solar_energy/\")\n",
    "fuel_prices_dataset_path = Path(\"data/fuel_prices/\")\n",
    "passengers_dataset_path = Path(\"data/air_passengers/\")\n",
    "results_dir = Path(\"results\")\n",
    "\n",
    "seq_len = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing hsm dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:32,  3.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing se dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "1it [00:02,  2.02s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2it [00:04,  2.27s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "3it [00:06,  2.09s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "4it [00:08,  2.11s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "5it [00:10,  2.09s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "6it [00:12,  2.12s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "7it [00:14,  2.09s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "8it [00:16,  2.01s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "9it [00:18,  1.91s/it]c:\\Users\\Владислав\\Учеба\\Магистратура\\3 семестр\\synthetic data generation\\project\\utils\\synth_eval.py:49: RuntimeWarning: Mean of empty slice.\n",
      "  kl_div_res.append(kl_div(synth_ts, train_ts[i: i + len(synth_ts)]).mean())\n",
      "c:\\Users\\Владислав\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\numpy\\core\\_methods.py:190: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "10it [00:20,  2.01s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing fp dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [00:01,  5.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing ap dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "50it [00:32,  1.55it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "defaultdict(dict,\n",
       "            {'hsm': {'kl_div': 11.69661606438711,\n",
       "              'kstest_pval': 0.0028594020852601536},\n",
       "             'se': {'kl_div': nan, 'kstest_pval': 2.99821208283929e-06},\n",
       "             'fp': {'kl_div': 13.111345020163736,\n",
       "              'kstest_pval': 0.008089773849450373},\n",
       "             'ap': {'kl_div': 2286.7915978449946,\n",
       "              'kstest_pval': 0.02092120631954569}})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_sim((\"hsm\", \"se\", \"fp\", \"ap\"), (hsm_dataset_path, solar_energy_dataset_path, fuel_prices_dataset_path, passengers_dataset_path),\n",
    "     \"TTS_GAN\", save=True, results_dir=results_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "1289e797c8b2364a1b561fc46768e8fcf8446b2e18e77ab0795c8743ff6ac10a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}