2-3-1-SketchVAE-data-process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "In this file \n",
    "We do data processing for SketchVAE training\n",
    "Since it requires rhythm and pitch tokens, we should process the previous data into the rhythm and pitch data first.\n",
    "'''\n",
    "from loader.dataloader import DataLoader\n",
    "import numpy as np\n",
    "\n",
    "data_path = [\n",
    "    \"data/irish_train.npy\", \n",
    "    \"data/irish_validate.npy\", \n",
    "    \"data/irish_test.npy\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import data\n",
    "train_x = np.load(data_path[0],allow_pickle = True)\n",
    "validate_x = np.load(data_path[1],allow_pickle = True)\n",
    "test_x = np.load(data_path[2],allow_pickle = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note extraction\n",
    "hold_state = 128\n",
    "rest_state = 129\n",
    "def extract_note(x, pad_token = 128):\n",
    "    d = []\n",
    "    for i in x:\n",
    "        if i < 128:\n",
    "            d.append(i)\n",
    "    ori_d = len(d)\n",
    "    d.extend([pad_token] * (len(x) - len(d)))\n",
    "    return np.array(d), ori_d\n",
    "\n",
    "def extract_rhythm(x, hold_token = 2, rest_token = 3):\n",
    "    d = []\n",
    "    for i in x:\n",
    "        if i < 128:\n",
    "             d.append(1)\n",
    "        elif i == hold_state:\n",
    "             d.append(hold_token)\n",
    "        else:\n",
    "             d.append(rest_token)\n",
    "    return np.array(d)\n",
    "             "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process rhythm and pitch tokens\n",
    "split_size = 24\n",
    "new_data = []\n",
    "# change here to be train_x/validate_x/test_x\n",
    "for i,d in enumerate(train_x):\n",
    "    d = np.array(d[\"notes\"])\n",
    "    ds = np.split(d, list(range(split_size,len(d),split_size)))\n",
    "    data = []\n",
    "    for sd in ds:\n",
    "        if len(sd) != split_size:\n",
    "            continue\n",
    "        q,k = extract_note(sd)\n",
    "        if k == 0:\n",
    "            continue\n",
    "        s = extract_rhythm(sd)\n",
    "        data.append([sd,q,s,k])\n",
    "    new_data.append(data)\n",
    "    if i % 1000 == 0:\n",
    "        print(\"processed:\", i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now you can see what processed data looks like [gd, pitch, rhythm, pitch_len]\n",
    "print(new_data[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we need to extract each measure in each song\n",
    "final_data = []\n",
    "for d in new_data:\n",
    "    for dd in d:\n",
    "        final_data.append(dd)\n",
    "print(len(final_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the data named: irish_xxx_chord_rhythm.npy\n",
    "np.save(\"data/irish_train_chord_rhythm.npy\",final_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}