main_48k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports and utils\n",
    "\n",
    "import tensorflow.compat.v2 as tf\n",
    "import ddsp.training\n",
    "_AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
    "from IPython.display import Audio, display\n",
    "from livelossplot import PlotLosses\n",
    "import numpy as np\n",
    "import random \n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow_datasets as tfds\n",
    "import time\n",
    "import data\n",
    "import random\n",
    "import copy\n",
    "import pydash\n",
    "import tqdm\n",
    "import soundfile\n",
    "import os\n",
    "import model\n",
    "\n",
    "# define constants\n",
    "CLIP_S=4\n",
    "SAMPLE_RATE=48000\n",
    "N_SAMPLES=SAMPLE_RATE*CLIP_S\n",
    "SEED=1\n",
    "FT_FRAME_RATE=250\n",
    "\n",
    "tf.random.set_seed(\n",
    "    SEED\n",
    ")\n",
    "np.random.seed(SEED)\n",
    "random.seed(SEED)\n",
    "\n",
    "# define some utilis\n",
    "def play(audio):\n",
    "  display(Audio(audio,rate=SAMPLE_RATE))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_NSYNTH=False\n",
    "INSTRUMENT_FAMILY=\"Trombone\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[200,48000] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [Op:Fill]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-a5151af73893>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mN_HARMONICS\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m192\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mae\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSAMPLE_RATE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mCLIP_S\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mFT_FRAME_RATE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mZ_SIZE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mN_INSTRUMENTS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mIR_DURATION\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mBIDIRECTIONAL\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mUSE_F0_CONFIDENCE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mN_HARMONICS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mN_NOISE_MAGNITUDES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m# define loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/nic/model.py\u001b[0m in \u001b[0;36mget_model\u001b[0;34m(SAMPLE_RATE, CLIP_S, FT_FRAME_RATE, Z_SIZE, N_INSTRUMENTS, IR_DURATION, BIDIRECTIONAL, USE_F0_CONFIDENCE, N_HARMONICS, N_NOISE_MAGNITUDES)\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0mprocessor_group\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_group\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0mn_instruments\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mN_INSTRUMENTS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m         \u001b[0minstrument_weight_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minstrument_weight_metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m     )\n\u001b[1;32m    181\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mae\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/nic/model.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, preprocessor, encoder, decoder, processor_group, losses, n_instruments, instrument_weight_metadata, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_instruments\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_instruments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstrument_weight_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minstrument_weight_metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minitialize_instrument_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0minitialize_instrument_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/nic/model.py\u001b[0m in \u001b[0;36minitialize_instrument_weights\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstrument_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mweight_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mweight_metadata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstrument_weight_metadata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstrument_weights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mweight_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight_metadata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"initializer\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_instruments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_shared\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/nic/model.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(batch_size)\u001b[0m\n\u001b[1;32m    154\u001b[0m             \u001b[0;34m\"ir\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m             {\n\u001b[0;32m--> 156\u001b[0;31m                 \u001b[0;34m\"initializer\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mIR_SIZE\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m                 \u001b[0;34m\"processing\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mbatched_feature\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatched_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py\u001b[0m in \u001b[0;36mwrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m   2913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2914\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2915\u001b[0;31m     \u001b[0mtensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2916\u001b[0m     \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_zeros_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2917\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py\u001b[0m in \u001b[0;36mzeros\u001b[0;34m(shape, dtype, name)\u001b[0m\n\u001b[1;32m   2974\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shape_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2975\u001b[0m       \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Ensure it's a vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2976\u001b[0;31m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconstant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzero\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2977\u001b[0m   \u001b[0;32massert\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase_dtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2978\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py\u001b[0m in \u001b[0;36mfill\u001b[0;34m(dims, value, name)\u001b[0m\n\u001b[1;32m    238\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mend_compatibility\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m   \"\"\"\n\u001b[0;32m--> 240\u001b[0;31m   \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdims\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    241\u001b[0m   \u001b[0mtensor_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaybe_set_static_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_array_ops.py\u001b[0m in \u001b[0;36mfill\u001b[0;34m(dims, value, name)\u001b[0m\n\u001b[1;32m   3366\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3367\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3368\u001b[0;31m       \u001b[0m_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3369\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3370\u001b[0m       \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mraise_from_not_ok_status\u001b[0;34m(e, name)\u001b[0m\n\u001b[1;32m   6939\u001b[0m   \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\" name: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6940\u001b[0m   \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6941\u001b[0;31m   \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6942\u001b[0m   \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6943\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[200,48000] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [Op:Fill]"
     ]
    }
   ],
   "source": [
    "IR_DURATION=1\n",
    "Z_SIZE=1024 if INSTRUMENT_FAMILY==\"**\" else 512\n",
    "N_INSTRUMENTS=200\n",
    "BIDIRECTIONAL=True\n",
    "USE_F0_CONFIDENCE=True\n",
    "N_NOISE_MAGNITUDES=192\n",
    "N_HARMONICS=192\n",
    "                 \n",
    "ae=model.get_model(SAMPLE_RATE,CLIP_S,FT_FRAME_RATE,Z_SIZE,N_INSTRUMENTS,IR_DURATION,BIDIRECTIONAL,USE_F0_CONFIDENCE,N_HARMONICS,N_NOISE_MAGNITUDES)\n",
    "\n",
    "# define loss\n",
    "fft_sizes = [64]\n",
    "while fft_sizes[-1]<SAMPLE_RATE//4:\n",
    "    fft_sizes.append(fft_sizes[-1]*2)\n",
    "\n",
    "spectral_loss = ddsp.losses.SpectralLoss(loss_type='L1',\n",
    "                                            fft_sizes=fft_sizes,\n",
    "                                            mag_weight=1.0,\n",
    "                                            logmag_weight=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "if USE_NSYNTH:\n",
    "    tfds.load(\"nsynth/gansynth_subset.f0_and_loudness\",split=\"train\", try_gcs=False,download=True) \n",
    "    trn_data_provider = data.CustomNSynthTfds(data_dir=\"/root/tensorflow_datasets/\",split=\"train\")\n",
    "    tfds.load(\"nsynth/gansynth_subset.f0_and_loudness\",split=\"valid\", try_gcs=False,download=True) \n",
    "    val_data_provider = data.CustomNSynthTfds(data_dir=\"/root/tensorflow_datasets/\",split=\"valid\")\n",
    "    def crepe_is_certain(x):\n",
    "        is_playing = tf.cast(x[\"loudness_db\"]>-100.0,dtype=tf.float32)\n",
    "        average_certainty=tf.reduce_sum(x[\"f0_confidence\"]*is_playing)/tf.reduce_sum(is_playing)\n",
    "        return average_certainty\n",
    "    def preprocess_dataset(dataset):\n",
    "        if INSTRUMENT_FAMILY!=\"all\":\n",
    "            dataset=dataset.filter(lambda x: x[\"instrument_family\"]==INSTRUMENT_FAMILY)\n",
    "        return dataset\n",
    "    trn_dataset = preprocess_dataset(trn_data_provider.get_dataset())\n",
    "    val_dataset = preprocess_dataset(val_data_provider.get_dataset())\n",
    "\n",
    "else:\n",
    "\n",
    "    \n",
    "    trn_data_provider=data.MultiTFRecordProvider(f\"datasets/AIR/tfr48k/dev/{INSTRUMENT_FAMILY}/*\",sample_rate=SAMPLE_RATE)\n",
    "    trn_dataset= trn_data_provider.get_dataset()\n",
    "    \n",
    "    val_data_provider=data.MultiTFRecordProvider(f\"datasets/AIR/tfr48k/tst/{INSTRUMENT_FAMILY}/*\",sample_rate=SAMPLE_RATE)\n",
    "    val_dataset=val_data_provider.get_dataset(shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove some samples if number of recordings greater than model capacity\n",
    "trn_dataset = trn_dataset.filter(lambda x: int(x[\"instrument_idx\"])<N_INSTRUMENTS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path=f\"checkpoints/48k_{'bidir' if BIDIRECTIONAL else 'unidir'}_z{Z_SIZE}_conv_family_{INSTRUMENT_FAMILY}{'_f0c' if USE_F0_CONFIDENCE else ''}\"\n",
    "\n",
    "print(checkpoint_path)\n",
    "try:\n",
    "    print(\"loading checkpoint\")\n",
    "    ae.load_weights(checkpoint_path)\n",
    "except:\n",
    "    print(\"couldn't load checkpoint\")\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotlosses = PlotLosses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "## training loop with adam\n",
    "\n",
    "BATCH_SIZE=1\n",
    "batched_trn_dataset=trn_dataset.shuffle(10000).batch(BATCH_SIZE,drop_remainder=True)\n",
    "\n",
    "# 1e-4 was good for saxophone (got us to 4.7-ish in 20 hours our so)\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)\n",
    "\n",
    "e=0\n",
    "while True:\n",
    "  batch_counter=0\n",
    "  epoch_loss=0   \n",
    "  for batch in batched_trn_dataset:\n",
    "    with tf.GradientTape() as tape:\n",
    "        output=ae(batch,train_shared=True)\n",
    "        loss_value=spectral_loss(batch[\"audio\"],output[\"audio_synth\"])\n",
    "        gradients = tape.gradient(loss_value, ae.trainable_variables)\n",
    "        epoch_loss+=loss_value.numpy()\n",
    "        optimizer.apply_gradients(zip(gradients, ae.trainable_variables))\n",
    "        \n",
    "    if batch_counter % 10==0:\n",
    "        print(f\"batch nr {batch_counter}, loss: {loss_value.numpy()}\")\n",
    "\n",
    "    if batch_counter ==0:    \n",
    "        play(tf.reshape(output[\"audio\"],(-1)))\n",
    "        play(tf.reshape(output['audio_synth'],(-1)))\n",
    "        play(tf.reshape(output['harmonic+fn'][\"signal\"],(-1))) \n",
    "        play(tf.reshape(output['harmonic'][\"signal\"],(-1))) \n",
    "        play(tf.reshape(output[\"fn\"][\"signal\"],(-1)))\n",
    "        play(tf.reshape(output[\"ir\"],(-1)))\n",
    "        \n",
    "    if batch_counter>1000-1 and batch_counter % 1000==0:\n",
    "        ae.save_weights(checkpoint_path)\n",
    "\n",
    "    batch_counter+=1\n",
    "    \n",
    "  plotlosses.update({'loss': epoch_loss/batch_counter,})\n",
    "  plotlosses.send()\n",
    "  ae.save_weights(checkpoint_path)\n",
    "  e+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# define some data utilities\n",
    "\n",
    "N_FIT_SECONDS = 16\n",
    "    \n",
    "TRAIN_SHARED=False\n",
    "\n",
    "USE_FNR=True\n",
    "\n",
    "EARLY_REFLECTION_DURATION=0.2\n",
    "\n",
    "def join_batch(batch):\n",
    "    for key in batch.keys():\n",
    "        assert len(batch[key].shape)<3\n",
    "        if len(batch[key].shape)==2:\n",
    "            batch[key]=tf.reshape(batch[key],(1,-1))\n",
    "    return batch\n",
    "\n",
    "def window_signal(a,window_len,hop_len):\n",
    "     assert(a.shape[0]==1)\n",
    "     windows=[]\n",
    "     start_frame=0\n",
    "     while True:\n",
    "        windows.append(a[:,start_frame:start_frame+window_len,...])\n",
    "        start_frame+=hop_len\n",
    "        if start_frame > a.shape[1]-window_len:\n",
    "            break\n",
    "     return tf.concat(windows,axis=0)\n",
    "\n",
    "def window_sample(instance,win_s,hop_s):\n",
    "    instance[\"audio\"]=window_signal(instance[\"audio\"],win_s*SAMPLE_RATE,hop_s*SAMPLE_RATE)\n",
    "    for key in [\"f0_hz\",\"loudness_db\",\"f0_confidence\"]:\n",
    "        instance[key]=window_signal(instance[key],win_s*FT_FRAME_RATE,hop_s*FT_FRAME_RATE)\n",
    "    instance[\"instrument\"]=tf.repeat(instance[\"instrument\"][0],(instance[\"audio\"].shape[0]))\n",
    "    instance[\"instrument_idx\"]=tf.repeat(instance[\"instrument_idx\"][0],(instance[\"audio\"].shape[0]))\n",
    "    #for key,item in instance.items():\n",
    "    #    assert(len(item.shape)<2 or item.shape[0]>1)\n",
    "    return instance\n",
    "\n",
    "def join_and_window(instance,win_s=4,hop_s=1):\n",
    "    return window_sample(join_batch(instance),win_s,hop_s)\n",
    "\n",
    "def rf2cf(row_form):\n",
    "    return {k:[s[k] for s in row_form] for k in row_form[0].keys()}\n",
    "\n",
    "# few shot voice cloning\n",
    "\n",
    "\n",
    "def regularization(batch):\n",
    "    ir = batch[\"ir\"]\n",
    "    ir = ir/tf.reduce_max(tf.abs(ir)+1e-10)\n",
    "    return tf.reduce_mean((ir**2)*tf.cast(tf.linspace(0,1,ir.shape[-1])[None,:],tf.float32))*0.0\n",
    "\n",
    "# constants\n",
    "BATCH_SIZE=1\n",
    "\n",
    "#5e-4 to 1e-5 has worked well\n",
    "\n",
    "N_DEMO_SAMPLES=int(4*SAMPLE_RATE)\n",
    "\n",
    "n_fit_windows=int(N_FIT_SECONDS/CLIP_S)\n",
    "\n",
    "N_FIT_ITERATIONS= 100 if TRAIN_SHARED else int(100*(16/N_FIT_SECONDS)) \n",
    "\n",
    "VAL_LR=3e-5 if TRAIN_SHARED else 2e-3\n",
    "SAVE = True\n",
    "timestamp=0\n",
    "DEMO_PATH=f\"artefacts/demos/{INSTRUMENT_FAMILY}_{timestamp}_{N_FIT_SECONDS}_{'train_shared' if TRAIN_SHARED else ''}/\"\n",
    "\n",
    "DEMO_IR_DURATION=1.0\n",
    "DEMO_IR_SAMPLES=int(DEMO_IR_DURATION*SAMPLE_RATE)\n",
    "\n",
    "\n",
    "val_dataset=list(val_dataset)\n",
    "\n",
    "# group by instrument id\n",
    "val_dataset_by_instrument=pydash.collections.group_by(list(val_dataset),lambda x: str(x[\"instrument\"].numpy()))\n",
    "val_dataset_by_instrument = {k:v for k,v in val_dataset_by_instrument.items() if len(v)>n_fit_windows*2}\n",
    "\n",
    "# load model\n",
    "ae_test=model.get_model(SAMPLE_RATE,CLIP_S,FT_FRAME_RATE,Z_SIZE,N_INSTRUMENTS,IR_DURATION,BIDIRECTIONAL,USE_F0_CONFIDENCE,N_HARMONICS,N_NOISE_MAGNITUDES)\n",
    "# load model weights       \n",
    "\n",
    "ae_test.set_is_shared_trainable(True)\n",
    "ae_test.load_weights(checkpoint_path)\n",
    "\n",
    "ae_test.instrument_weight_metadata[\"ir\"][\"initializer\"]=lambda batch_size: tf.zeros([batch_size,int(DEMO_IR_DURATION*SAMPLE_RATE)])\n",
    "\n",
    "if USE_FNR:\n",
    "\n",
    "    er_samples=int(EARLY_REFLECTION_DURATION*SAMPLE_RATE)\n",
    "\n",
    "    er_amp=np.ones((er_samples))\n",
    "    er_amp[er_samples//2:er_samples]=np.linspace(1,0,er_samples//2)\n",
    "\n",
    "    frame_rate=1000\n",
    "    n_filter_bands=100\n",
    "    n_frames=int(frame_rate*DEMO_IR_DURATION)\n",
    "\n",
    "    ir_fn=ddsp.synths.FilteredNoise(n_samples=DEMO_IR_SAMPLES,\n",
    "                                       window_size=750,\n",
    "                                       scale_fn=tf.nn.relu,\n",
    "                                       initial_bias=0.0001)\n",
    "\n",
    "    def processing_fn(batched_feature):\n",
    "\n",
    "        batch_size=batched_feature.shape[0]\n",
    "        er_ir = tf.nn.tanh(batched_feature[:,:er_samples])\n",
    "\n",
    "        er_amp=np.ones(DEMO_IR_SAMPLES)\n",
    "        er_amp[er_samples//2:er_samples]=np.linspace(1,0,er_samples//2)\n",
    "        er_amp[er_samples:]=0\n",
    "\n",
    "        er_amp = er_amp[None,:]\n",
    "        fn_amp= 1-er_amp\n",
    "\n",
    "        fn_mags=tf.reshape(batched_feature[:,er_samples:],[batch_size,n_frames,n_filter_bands])\n",
    "        fn_ir=ir_fn(fn_mags)\n",
    "\n",
    "        ir=fn_ir*fn_amp+tf.pad(er_ir,[[0,0],[0,int(DEMO_IR_DURATION*SAMPLE_RATE)-er_samples]])*er_amp\n",
    "\n",
    "        #ir = ddsp.core.fft_convolve( fn_ir,er_ir, padding='same', delay_compensation=0)\n",
    "        return ir\n",
    "\n",
    "    ae_test.instrument_weight_metadata[\"ir\"][\"processing\"]=processing_fn\n",
    "\n",
    "    ae_test.instrument_weight_metadata[\"ir\"][\"initializer\"]=lambda batch_size: tf.zeros([batch_size,er_samples+n_frames*n_filter_bands])\n",
    "\n",
    "    ae_test.instrument_weight_metadata[\"wet_gain\"][\"initializer\"]=lambda batch_size: tf.ones([batch_size,1])*0.5\n",
    "\n",
    "ae_test.initialize_instrument_weights()\n",
    "ae_test.set_is_shared_trainable(True)\n",
    "\n",
    "TMP_CHECKPOINT_PATH=\"artefacts/tmp_checkpoint\"\n",
    "ae_test.save_weights(TMP_CHECKPOINT_PATH)\n",
    "\n",
    "for ii,instrument_set in enumerate(list(val_dataset_by_instrument.values())): \n",
    "\n",
    "    print(f\"instrument nr {ii}\")\n",
    "\n",
    "    ae_test.set_is_shared_trainable(True)\n",
    "    ae_test.load_weights(TMP_CHECKPOINT_PATH)\n",
    "    ae_test.initialize_instrument_weights()\n",
    "\n",
    "    # data\n",
    "    # reshape data\n",
    "    fit_data=instrument_set[:n_fit_windows]\n",
    "\n",
    "    print(f\"{len(instrument_set)-5}>={n_fit_windows}\")\n",
    "\n",
    "    assert len(instrument_set)-5>=n_fit_windows\n",
    "\n",
    "    # Use last 4 windows (16 s) as test data\n",
    "    test_data=instrument_set[-5:-1]\n",
    "\n",
    "    def playback_and_save(x,fn):\n",
    "        print(fn)\n",
    "        play(x)\n",
    "        if SAVE:\n",
    "            os.makedirs(DEMO_PATH,exist_ok=True)\n",
    "            path=DEMO_PATH+f\"recording nr: {ii} \"+fn+\".wav\"\n",
    "            soundfile.write(path,x,SAMPLE_RATE)\n",
    "\n",
    "    # convert to column form\n",
    "    fit_data = rf2cf(fit_data)\n",
    "\n",
    "    # get one batch for fitting\n",
    "    fit_batch= next(iter(tf.data.Dataset.from_tensor_slices(fit_data).batch(len(list(fit_data)))))\n",
    "\n",
    "    playback_and_save(tf.reshape(fit_data[\"audio\"],[-1]),\"training data\")\n",
    "\n",
    "    # transform data so that the clips overlap\n",
    "    fit_batch=join_and_window(fit_batch,4,1)\n",
    "    fit_data=tf.data.Dataset.from_tensor_slices(fit_batch)\n",
    "    fit_batched=fit_data.batch(BATCH_SIZE)\n",
    "\n",
    "    # prepare test data\n",
    "    test_data = rf2cf(test_data)\n",
    "    test_batched= tf.data.Dataset.from_tensor_slices(test_data).batch(BATCH_SIZE)\n",
    "    \n",
    "\n",
    "    fit_losses=[]\n",
    "    tst_losses=[]\n",
    "\n",
    "    # set up optimizer\n",
    "    val_optimizer = tf.keras.optimizers.Adam(learning_rate=VAL_LR)\n",
    "\n",
    "    for i in tqdm.tqdm(range(N_FIT_ITERATIONS)):\n",
    "        fit_batched=fit_batched.shuffle(100)\n",
    "\n",
    "        epoch_loss=0\n",
    "        batch_counter=0\n",
    "        test_epoch_loss=0\n",
    "        test_batch_counter=0\n",
    "\n",
    "        for fit_batch in fit_batched:\n",
    "            with tf.GradientTape() as tape:\n",
    "\n",
    "              output = ae_test(fit_batch,train_shared=TRAIN_SHARED)\n",
    "              loss_value=spectral_loss(fit_batch[\"audio\"],output[\"audio_synth\"])+regularization(output)\n",
    "              epoch_loss+=loss_value.numpy()\n",
    "              batch_counter+=1\n",
    "              gradients = tape.gradient(loss_value, ae_test.trainable_weights)\n",
    "            val_optimizer.apply_gradients(zip(gradients, ae_test.trainable_weights))\n",
    "        fit_losses.append(epoch_loss/batch_counter)\n",
    "\n",
    "\n",
    "        for test_batch in test_batched:\n",
    "            test_output=ae_test(test_batch,train_shared=False)\n",
    "            loss_value=spectral_loss(test_batch[\"audio\"],test_output[\"audio_synth\"])   \n",
    "            test_epoch_loss+=loss_value.numpy()\n",
    "            test_batch_counter+=1\n",
    "        tst_losses.append(test_epoch_loss/test_batch_counter)\n",
    "\n",
    "        if i%50==0:\n",
    "\n",
    "            print(\"target\")        \n",
    "            play(tf.reshape(fit_batch[\"audio\"],(-1)))\n",
    "\n",
    "            print(\"estimate\")     \n",
    "            play(tf.reshape(output['audio_synth'],(-1)))\n",
    "            # loss plot\n",
    "            plt.plot(tst_losses,label=\"tst\")\n",
    "            plt.plot(fit_losses,label=\"trn\")\n",
    "            plt.yscale(\"log\")\n",
    "            plt.legend()\n",
    "            plt.show()\n",
    "\n",
    "            ir=output['ir'][0]\n",
    "\n",
    "            plt.plot(ir)\n",
    "            plt.show()\n",
    "\n",
    "            play(tf.reshape(ir,(-1)))\n",
    "\n",
    "            plt.imshow(ddsp.spectral_ops.compute_mel(ir).numpy().T,aspect=\"auto\",origin=\"lower\")\n",
    "            plt.show()\n",
    "\n",
    "            print(f\"wet gain: {output['wet_gain']['controls']['gain_scaled']}\")\n",
    "            print(f\"dry gain: {output['dry_gain']['controls']['gain_scaled']}\")\n",
    "\n",
    "    plt.plot(tst_losses,label=\"tst\")\n",
    "    plt.plot(fit_losses,label=\"trn\")\n",
    "    plt.yscale(\"log\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "    print(\">> seen data:\")\n",
    "    playback_and_save(tf.reshape(fit_batch[\"audio\"],[-1]),\"training target\")\n",
    "\n",
    "    playback_and_save(tf.reshape(output[\"audio_synth\"],[-1]),\"training estimate\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"f0_hz\"]=transposed_fit_batch[\"f0_hz\"]*0.7\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"transposed down\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"f0_hz\"]=transposed_fit_batch[\"f0_hz\"]*1.3\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"transposed up\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"loudness_db\"]=transposed_fit_batch[\"loudness_db\"]-12\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"loudness -12 db\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"loudness_db\"]=transposed_fit_batch[\"loudness_db\"]-6\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"loudness -6 db\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"loudness_db\"]=transposed_fit_batch[\"loudness_db\"]+6\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"loudness +6 db\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"loudness_db\"]=transposed_fit_batch[\"loudness_db\"]+12\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"loudness +12 db\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"f0_confidence\"]=transposed_fit_batch[\"f0_confidence\"]*0.9\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"low confidence\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"f0_confidence\"]=transposed_fit_batch[\"f0_confidence\"]*0.5\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"very low f0 confidence\")\n",
    "\n",
    "    transposed_fit_batch = copy.deepcopy(fit_batch)\n",
    "    transposed_fit_batch[\"f0_confidence\"]=transposed_fit_batch[\"f0_confidence\"]*0.0\n",
    "    transposed_output=ae_test(transposed_fit_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(transposed_output['audio_synth'],(-1)),\"no f0 confidence\")\n",
    "\n",
    "    print(\">> unseen data:\")\n",
    "    playback_and_save(tf.reshape(test_batch[\"audio\"][:,:N_DEMO_SAMPLES],(-1)),\"unseen target\")\n",
    "\n",
    "    test_batch_output = ae_test(test_batch,train_shared=False)\n",
    "    playback_and_save(tf.reshape(test_batch_output['audio_synth'][:,:N_DEMO_SAMPLES],(-1)),\"unseen estimate\")\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "slow instrument cloning.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}