5-2-vae_independent_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Here is the extra experiments file we test the VAE, we did not show it in the paper because of the space\n",
    "Generally speaking, we test the EC2VAE and SketchVAE in the disentanglement of rhythm and pitch.\n",
    "We change the rhythm but maintain the pitch/change the pitch but maintain the rhythm, to see the changes of z_pitch and z_rhythm.\n",
    "Please ***RUN THE CODE WITH OPTIONS*** in the cell.\n",
    "'''\n",
    "import numpy as np\n",
    "import os\n",
    "import shutil\n",
    "import random\n",
    "import torch\n",
    "import sklearn.utils\n",
    "from torch.distributions import kl_divergence,Normal\n",
    "from torch.nn import functional as F\n",
    "from torch import optim\n",
    "\n",
    "# These three data is achived by the code in the below cell, \n",
    "# about how we use the oringinal data to shift pitches and rhythms\n",
    "data_path = [\n",
    "    \"data/exp_vae_independent.npy\",\n",
    "    \"data/exp_vae_independent_pitch.npy\",\n",
    "    \"data/exp_vae_independent_rhythm.npy\"\n",
    "]\n",
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DON'T RUN THIS CODE\n",
    "# Because data has already there\n",
    "def check_rest(x):\n",
    "    for i in x:\n",
    "        if i == 129:\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "def shift_pitch(x, p):\n",
    "    new_x = []\n",
    "    for i in range(len(x)):\n",
    "        if x[i] < 100:\n",
    "            new_x.append(x[i] + p)\n",
    "        else:\n",
    "            new_x.append(x[i])\n",
    "    return new_x\n",
    "\n",
    "def shift_rhythm(x):\n",
    "    x_p = []\n",
    "    x_new = [] \n",
    "    j = 0\n",
    "    for i in range(len(x)):\n",
    "        if x[i] < 128:\n",
    "            x_p.append(x[i])\n",
    "    pos = 0\n",
    "    if x[0] < 128:\n",
    "        x_new.append(x_p[pos])\n",
    "        pos += 1\n",
    "    for j in x[::-1]:\n",
    "        if j < 128:\n",
    "            x_new.append(x_p[pos])\n",
    "            pos += 1\n",
    "        else:\n",
    "            x_new.append(128)\n",
    "        if pos >= len(x_p):\n",
    "            break\n",
    "    while pos < len(x_p):\n",
    "        x_new.append(x_p[pos])\n",
    "        pos += 1\n",
    "    while len(x_new) < len(x):\n",
    "        x_new.append(128)\n",
    "    return x_new\n",
    "\n",
    "train_x = np.load(ori_data_path,allow_pickle=True)\n",
    "print(train_x.shape)\n",
    "print(train_x[10].shape)\n",
    "new_x  = []\n",
    "new_rx = []\n",
    "new_px = []\n",
    "for i in train_x:\n",
    "    for j in i:\n",
    "        if check_rest(j):\n",
    "            rj = shift_rhythm(j)\n",
    "            pj = shift_pitch(j,6)\n",
    "            if rhythm_acc(j,rj) < 1:\n",
    "                new_x.append(j)\n",
    "                new_rx.append(rj)\n",
    "                new_px.append(pj)\n",
    "new_x = np.array(new_x)\n",
    "new_rx = np.array(new_rx)\n",
    "new_px = np.array(new_px)\n",
    "new_x = np.split(new_x[:len(new_x) // 16 * 16], len(new_x) // 16)\n",
    "new_rx = np.split(new_rx[:len(new_rx) // 16 * 16], len(new_rx) // 16)\n",
    "new_px = np.split(new_px[:len(new_px) // 16 * 16], len(new_px) // 16)\n",
    "new_x = np.array(new_x)\n",
    "new_rx = np.array(new_rx)\n",
    "new_px = np.array(new_px)\n",
    "np.save(\"vae_data/exp_vae_independent.npy\", new_x)\n",
    "np.save(\"vae_data/exp_vae_independent_pitch.npy\",new_px)\n",
    "np.save(\"vae_data/exp_vae_independent_rhythm.npy\",new_rx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# accuracy and  extratcion\n",
    "def rhythm_acc(a,b):\n",
    "    s = len(a)\n",
    "    num_pitch = np.sum(a >= 128)\n",
    "    acc = np.sum(np.logical_and(a == b,  a >= 128)) / num_pitch\n",
    "    return acc\n",
    "\n",
    "# note extraction\n",
    "hold_state = 128\n",
    "rest_state = 129\n",
    "def extract_note(x, pad_token = 128):\n",
    "    d = []\n",
    "    for i in x:\n",
    "        if i < 128:\n",
    "            d.append(i)\n",
    "    ori_d = len(d)\n",
    "    d.extend([pad_token] * (len(x) - len(d)))\n",
    "    return np.array(d), ori_d\n",
    "\n",
    "def extract_rhythm(x, hold_token = 2, rest_token = 3):\n",
    "    d = []\n",
    "    for i in x:\n",
    "        if i < 128:\n",
    "             d.append(1)\n",
    "        elif i == hold_state:\n",
    "             d.append(hold_token)\n",
    "        else:\n",
    "             d.append(rest_token)\n",
    "    return np.array(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-1 SketchVAE\n",
    "from SketchVAE.sketchvae import SketchVAE\n",
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "###############################\n",
    "# initial parameters\n",
    "\n",
    "save_path = \"model_backup/\"\n",
    "lr = 1e-4\n",
    "decay = 0.9999\n",
    "hidden_dims = 1024\n",
    "zp_dims = 128\n",
    "zr_dims = 128\n",
    "vae_beta = 0.1\n",
    "input_dims = 130\n",
    "pitch_dims = 129\n",
    "rhythm_dims = 3\n",
    "seq_len = 6 * 4\n",
    "beat_num = 4\n",
    "tick_num = 6\n",
    "save_period = 1\n",
    "##############################\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-2 EC2VAE\n",
    "from ec2vae.model_dis import RECVAE\n",
    "###############################\n",
    "# ec_vae\n",
    "hidden_dims = 2048\n",
    "z1_dims = 128\n",
    "z2_dims = 128\n",
    "vae_beta = 0.1\n",
    "input_dims = 130\n",
    "rhythm_dims = 3\n",
    "seq_len = 6 * 4\n",
    "##############################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input data\n",
    "data_x = np.load(data_path[0], allow_pickle = True)\n",
    "data_px = np.load(data_path[1], allow_pickle = True)\n",
    "data_rx = np.load(data_path[2], allow_pickle = True)\n",
    "print(data_x[0][0])\n",
    "print(data_px[0][0])\n",
    "# dl = DataLoader(train = train_x, validate = train_x, test = train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-1 SketchVAE\n",
    "# input data\n",
    "def processed_data_tensor(data):\n",
    "    print(\"processed data:\")\n",
    "    gd = [] \n",
    "    px = []\n",
    "    rx = []\n",
    "    len_x = []\n",
    "    nrx = []\n",
    "    for d in data:\n",
    "        gd.append([list(dd[0]) for dd in d])\n",
    "        px.append([list(dd[1]) for dd in d])\n",
    "        rx.append([list(dd[2]) for dd in d])\n",
    "        len_x.append([dd[3] for dd in d])\n",
    "        if len(gd[-1][-1]) != 24:\n",
    "            gd[-1][-1].extend([128] * (seq_len - len(gd[-1][-1])))\n",
    "            px[-1][-1].extend([128] * (seq_len - len(px[-1][-1])))\n",
    "            rx[-1][-1].extend([2] * (seq_len - len(rx[-1][-1])))\n",
    "    for i,d in enumerate(len_x):\n",
    "        for j,dd in enumerate(d):\n",
    "            if len_x[i][j] == 0:\n",
    "                gd[i][j][0] = 60\n",
    "                px[i][j][0] = 60\n",
    "                rx[i][j][0] = 1\n",
    "                len_x[i][j] = 1\n",
    "                print(\"find all 0\")\n",
    "            \n",
    "    gd = np.array(gd)\n",
    "    px = np.array(px)\n",
    "    rx = np.array(rx)\n",
    "    len_x = np.array(len_x)\n",
    "    for d in rx:\n",
    "        nnrx = []\n",
    "        for dd in d:\n",
    "            temp = np.zeros((seq_len, rhythm_dims))\n",
    "            lins = np.arange(0, len(dd))\n",
    "            temp[lins, dd - 1] = 1\n",
    "            nnrx.append(temp)\n",
    "        nrx.append(nnrx)\n",
    "    nrx = np.array(nrx)\n",
    "    gd = torch.from_numpy(gd).long()\n",
    "    px = torch.from_numpy(px).long()\n",
    "    rx = torch.from_numpy(rx).float()\n",
    "    len_x = torch.from_numpy(len_x).long()\n",
    "    nrx = torch.from_numpy(nrx).float()\n",
    "    print(\"processed finish!\")\n",
    "    print(gd.size(),px.size(),rx.size(),len_x.size(),nrx.size())\n",
    "    return TensorDataset(px, rx, len_x, nrx, gd)\n",
    "\n",
    "split_size = 24\n",
    "new_data = []\n",
    "for i,d in enumerate(data_x):\n",
    "    ds = d\n",
    "    data = []\n",
    "    for sd in ds:\n",
    "        q,k = extract_note(sd)\n",
    "        s = extract_rhythm(sd)\n",
    "        data.append([sd,q,s,k])\n",
    "    new_data.append(data)\n",
    "dis_data_x = np.array(new_data)\n",
    "\n",
    "new_data = []\n",
    "for i,d in enumerate(data_px):\n",
    "    ds = d\n",
    "    data = []\n",
    "    for sd in ds:\n",
    "        q,k = extract_note(sd)\n",
    "        s = extract_rhythm(sd)\n",
    "        data.append([sd,q,s,k])\n",
    "    new_data.append(data)\n",
    "dis_data_px = np.array(new_data)\n",
    "\n",
    "new_data = []\n",
    "for i,d in enumerate(data_rx):\n",
    "    ds = d\n",
    "    data = []\n",
    "    for sd in ds:\n",
    "        q,k = extract_note(sd)\n",
    "        s = extract_rhythm(sd)\n",
    "        data.append([sd,q,s,k])\n",
    "    new_data.append(data)\n",
    "dis_data_rx = np.array(new_data)\n",
    "\n",
    "\n",
    "temp = DataLoader(\n",
    "    dataset = processed_data_tensor(dis_data_x),\n",
    "    batch_size = 1, \n",
    "    shuffle = False, \n",
    "    num_workers = 8, \n",
    "    pin_memory = True, \n",
    "    drop_last = True\n",
    ")\n",
    "dis_data_x = []\n",
    "for i,d in enumerate(temp):\n",
    "    dis_data_x.append(d)\n",
    "print(len(dis_data_x))\n",
    "\n",
    "temp = DataLoader(\n",
    "    dataset = processed_data_tensor(dis_data_px),\n",
    "    batch_size = 1, \n",
    "    shuffle = False, \n",
    "    num_workers = 8, \n",
    "    pin_memory = True, \n",
    "    drop_last = True\n",
    ")\n",
    "dis_data_px = []\n",
    "for i,d in enumerate(temp):\n",
    "    dis_data_px.append(d)\n",
    "print(len(dis_data_px))\n",
    "\n",
    "temp = DataLoader(\n",
    "    dataset = processed_data_tensor(dis_data_rx),\n",
    "    batch_size = 1, \n",
    "    shuffle = False, \n",
    "    num_workers = 8, \n",
    "    pin_memory = True, \n",
    "    drop_last = True\n",
    ")\n",
    "dis_data_rx = []\n",
    "for i,d in enumerate(temp):\n",
    "    dis_data_rx.append(d)\n",
    "print(len(dis_data_rx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-1 SketchVAE\n",
    "# import model\n",
    "model = SketchVAE(input_dims, pitch_dims, rhythm_dims, hidden_dims, zp_dims, zr_dims, seq_len, beat_num, tick_num, 4000)\n",
    "dic = torch.load(\"model_backup/sketchvae-param.pt\")\n",
    "\n",
    "for name in list(dic.keys()):\n",
    "    dic[name.replace('module.', '')] = dic.pop(name)\n",
    "model.load_state_dict(dic)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('Using: ', torch.cuda.get_device_name(torch.cuda.current_device()))\n",
    "    model.cuda()\n",
    "else:\n",
    "    print('Using: CPU')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-1 SketchVAE\n",
    "zs = []\n",
    "zps = []\n",
    "zrs = []\n",
    "# p(z)\n",
    "\n",
    "model.eval()\n",
    "device = torch.device(torch.cuda.current_device())\n",
    "\n",
    "for i in range(len(dis_data_x)):\n",
    "    v_px, v_rx, v_len_x, v_nrx, v_gd = dis_data_x[i]\n",
    "    v_px = v_px[0]\n",
    "    v_rx = v_rx[0]\n",
    "    v_len_x = v_len_x[0]\n",
    "    v_nrx = v_nrx[0]\n",
    "    v_gd = v_gd[0]\n",
    "    v_px = v_px.to(device = device,non_blocking = True)\n",
    "    v_len_x = v_len_x.to(device = device,non_blocking = True)\n",
    "    v_nrx = v_nrx.to(device = device,non_blocking = True)\n",
    "    v_gd = v_gd.to(device = device,non_blocking = True)\n",
    "    _, v_p_dis, v_r_dis, _ = model(v_px, v_nrx, v_len_x, v_gd)\n",
    "    zp = v_p_dis.mean\n",
    "    zr = v_r_dis.mean\n",
    "    z = torch.cat((zp,zr), -1)\n",
    "    z = z.cpu().detach().numpy()\n",
    "    zs.append(z)\n",
    "    if i % 100 == 0:\n",
    "        print(\"processed : %d\" %i)\n",
    "zs = np.array(zs)\n",
    "\n",
    "for i in range(len(dis_data_px)):\n",
    "    v_px, v_rx, v_len_x, v_nrx, v_gd = dis_data_px[i]\n",
    "    v_px = v_px[0]\n",
    "    v_rx = v_rx[0]\n",
    "    v_len_x = v_len_x[0]\n",
    "    v_nrx = v_nrx[0]\n",
    "    v_gd = v_gd[0]\n",
    "    v_px = v_px.to(device = device,non_blocking = True)\n",
    "    v_len_x = v_len_x.to(device = device,non_blocking = True)\n",
    "    v_nrx = v_nrx.to(device = device,non_blocking = True)\n",
    "    v_gd = v_gd.to(device = device,non_blocking = True)\n",
    "    _, v_p_dis, v_r_dis, _ = model(v_px, v_nrx, v_len_x, v_gd)\n",
    "    zp = v_p_dis.mean\n",
    "    zr = v_r_dis.mean\n",
    "    z = torch.cat((zp,zr), -1)\n",
    "    z = z.cpu().detach().numpy()\n",
    "    zps.append(z)\n",
    "    if i % 100 == 0:\n",
    "        print(\"processed : %d\" %i)\n",
    "zps = np.array(zps)\n",
    "\n",
    "for i in range(len(dis_data_rx)):\n",
    "    v_px, v_rx, v_len_x, v_nrx, v_gd = dis_data_rx[i]\n",
    "    v_px = v_px[0]\n",
    "    v_rx = v_rx[0]\n",
    "    v_len_x = v_len_x[0]\n",
    "    v_nrx = v_nrx[0]\n",
    "    v_gd = v_gd[0]\n",
    "    v_px = v_px.to(device = device,non_blocking = True)\n",
    "    v_len_x = v_len_x.to(device = device,non_blocking = True)\n",
    "    v_nrx = v_nrx.to(device = device,non_blocking = True)\n",
    "    v_gd = v_gd.to(device = device,non_blocking = True)\n",
    "    _, v_p_dis, v_r_dis, _ = model(v_px, v_nrx, v_len_x, v_gd)\n",
    "    zp = v_p_dis.mean\n",
    "    zr = v_r_dis.mean\n",
    "    z = torch.cat((zp,zr), -1)\n",
    "    z = z.cpu().detach().numpy()\n",
    "    zrs.append(z)\n",
    "    if i % 100 == 0:\n",
    "        print(\"processed : %d\" %i)\n",
    "zrs = np.array(zrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-2 EC2VAE\n",
    "# load EC_VAE model\n",
    "\n",
    "model = RECVAE(input_dims, hidden_dims, rhythm_dims, z1_dims,z2_dims,seq_len, 3000)\n",
    "\n",
    "dic = torch.load(\"model_backup/ec2vae-param.pt\")\n",
    "for name in list(dic.keys()):\n",
    "    dic[name.replace('module.', '')] = dic.pop(name)\n",
    "model.load_state_dict(dic)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('Using: ', torch.cuda.get_device_name(torch.cuda.current_device()))\n",
    "    model.cuda()\n",
    "else:\n",
    "    print('Using: CPU')\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Option-2 EC2VAE\n",
    "# EC_VAE decode\n",
    "def convert_onehot(batch_data):\n",
    "    vecs_out = []\n",
    "    for d in batch_data:\n",
    "        midi_vec = np.zeros((len(d),130))\n",
    "        k = np.arange(len(d))\n",
    "        midi_vec[k,d] = 1\n",
    "        vecs_out.append(midi_vec)\n",
    "    return np.array(vecs_out)\n",
    "\n",
    "i = 0\n",
    "for d in data_x:\n",
    "    raw_x = d\n",
    "    if len(raw_x[-1]) != seq_len:\n",
    "        raw_x = raw_x[:-1]\n",
    "    raw_x = np.array(raw_x)\n",
    "    x = convert_onehot(raw_x)\n",
    "    x = torch.from_numpy(x).float()\n",
    "    if torch.cuda.is_available():\n",
    "        x = x.cuda()\n",
    "    _,_,rdis1,rdis2 = model(x)\n",
    "    z1 = rdis1.mean\n",
    "    z2 = rdis2.mean\n",
    "    z = torch.cat((z1,z2),-1)\n",
    "    z = z.cpu().detach().numpy()\n",
    "    zs.append(z)\n",
    "    i += 1\n",
    "    if i % 100 == 0:\n",
    "        print(\"processed : %d\" %i)\n",
    "zs = np.array(zs)\n",
    "\n",
    "i = 0\n",
    "for d in data_px:\n",
    "    raw_x = d\n",
    "    if len(raw_x[-1]) != seq_len:\n",
    "        raw_x = raw_x[:-1]\n",
    "    raw_x = np.array(raw_x)\n",
    "    x = convert_onehot(raw_x)\n",
    "    x = torch.from_numpy(x).float()\n",
    "    if torch.cuda.is_available():\n",
    "        x = x.cuda()\n",
    "    _,_,rdis1,rdis2 = model(x)\n",
    "    z1 = rdis1.mean\n",
    "    z2 = rdis2.mean\n",
    "    z = torch.cat((z1,z2),-1)\n",
    "    z = z.cpu().detach().numpy()\n",
    "    zps.append(z)\n",
    "    i += 1\n",
    "    if i % 100 == 0:\n",
    "        print(\"processed : %d\" %i)\n",
    "zps = np.array(zps)\n",
    "\n",
    "i = 0\n",
    "for d in data_rx:\n",
    "    raw_x = d\n",
    "    if len(raw_x[-1]) != seq_len:\n",
    "        raw_x = raw_x[:-1]\n",
    "    raw_x = np.array(raw_x)\n",
    "    x = convert_onehot(raw_x)\n",
    "    x = torch.from_numpy(x).float()\n",
    "    if torch.cuda.is_available():\n",
    "        x = x.cuda()\n",
    "    _,_,rdis1,rdis2 = model(x)\n",
    "    z1 = rdis1.mean\n",
    "    z2 = rdis2.mean\n",
    "    z = torch.cat((z1,z2),-1)\n",
    "    z = z.cpu().detach().numpy()\n",
    "    zrs.append(z)\n",
    "    i += 1\n",
    "    if i % 100 == 0:\n",
    "        print(\"processed : %d\" %i)\n",
    "zrs = np.array(zrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-2 EC2VAE\n",
    "domi = zs.shape[0] * zs.shape[1]\n",
    "print(\"change rhythm zp:\", np.sum(np.absolute(zs[:, :, :128] - zrs[:,:, :128])) / domi )\n",
    "print(\"change rhythm zr:\",  np.sum(np.absolute(zs[:, :, 128:] - zrs[:,:, 128:])) / domi)\n",
    "print(\"change pitch zp:\", np.sum(np.absolute(zs[:, :, :128] - zps[:,:, :128])) / domi)\n",
    "print(\"change pitch zr:\", np.sum(np.absolute(zs[:, :, 128:] - zps[:,:, 128:])) / domi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option-1 SketchVAE\n",
    "domi = zs.shape[0] * zs.shape[1]\n",
    "print(\"change rhythm zp:\", np.sum(np.absolute(zs[:, :, :128] - zrs[:,:, :128])) / domi )\n",
    "print(\"change rhythm zr:\",  np.sum(np.absolute(zs[:, :, 128:] - zrs[:,:, 128:])) / domi)\n",
    "print(\"change pitch zp:\", np.sum(np.absolute(zs[:, :, :128] - zps[:,:, :128])) / domi)\n",
    "print(\"change pitch zr:\", np.sum(np.absolute(zs[:, :, 128:] - zps[:,:, 128:])) / domi)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}