4-2-reptition-nonrepetition-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "4-1 is deleted because it is the file from the previous version\n",
    "In this file we calculate the similarity between past and future conetext measures to develop two subsets.\n",
    "You don't need to use this because the subsets have already been there in your download material.\n",
    "We just show the code here to let you know our process.\n",
    "'''\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.load(\"data/irish-dis-measure-vae-test-whole.npy\", allow_pickle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity(a, b):\n",
    "    s = len(a)\n",
    "    max_acc = 0.0\n",
    "    num_pitch = np.sum(a < 128)\n",
    "    for i in range(s):\n",
    "        temp = np.concatenate((a[i:],a[:i]))\n",
    "#         acc = np.sum(temp == b) / s\n",
    "        acc = np.sum(np.logical_and(temp == b,  temp < 128)) / num_pitch\n",
    "        max_acc = max(acc, max_acc)\n",
    "    return max_acc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "acc = []\n",
    "for i,d in enumerate(data):\n",
    "    d = d[:,0]\n",
    "    a = np.concatenate((d[:6]))\n",
    "    b = np.concatenate((d[10:]))\n",
    "    acc.append([i,similarity(a,b)])\n",
    "    if i % 100 == 0:\n",
    "        print(\"process:\",i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc.sort(key = lambda x:x[1])\n",
    "acc = np.array(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAADT1JREFUeJzt3W2M7GddxvHr8pw28lAt8ayEtBwXDIKGWFvXilabUlT6FIlJTfChxEazMWpTExOpvpAY39Q3Bgwq2ZSKRKQhpUh5qjaBWgm2sAdPS9sDptYKRzBn6xNQE5tTLl/MHFzWmTP37M7/P/Ob+X6STWd2/mf6u3dmrrn3N/f9XycRAKCOb5p3AQCA6RDcAFAMwQ0AxRDcAFAMwQ0AxRDcAFAMwQ0AxRDcAFAMwQ0AxRzu4k6PHDmS9fX1Lu4aAJbSsWPHnkqy1nJsJ8G9vr6u7e3tLu4aAJaS7X9uPZZWCQAU0xTcts+3faftz9o+YfuHui4MADBaa6vkLZLuSXK97XMlPbfDmgAAZzExuG1/i6TLJf2CJCV5RtIz3ZYFABinpVXyUkk7kv7U9t/bvs328/YeZHvT9rbt7Z2dnZkXCgAYaAnuw5IukfQnSS6W9LSkW/YelGQryUaSjbW1phUtAIB9aAnuk5JOJnlweP1ODYIcADAHE4M7yb9K+oLtlw+/9RpJj3VaFQBgrNZVJTdJetdwRckTkm7sriQAwNk0BXeS45I2Oq4FwIys3/Khr19+8tZr51gJusDOSQAohuAGgGIIbgAohuAGgGIIbgAohuAGgGIIbgAohuAGgGIIbgAohuAGgGIIbgAohuAGgGIIbgAohuAGgGIIbgAopvUPKQBYcLvPwY3lxowbAIohuAGgGIIbAIohuAGgGIIbAIohuAGgGIIbAIohuAGgGIIbAIohuAGgmKYt77aflPQVSc9KOp1ko8uiAADjTXOuklcneaqzSgAATWiVAEAxrcEdSX9t+5jtzVEH2N60vW17e2dnZ3YVAgC+QWtwX5bkEklXS/pV25fvPSDJVpKNJBtra2szLRIA8H+agjvJF4f/PSXpfZIu7bIoAMB4E4Pb9vNsn3fmsqSfkPRI14UBAEZrWVXyQknvs33m+L9Ick+nVQEAxpoY3EmekHRRD7UAABqwHBAAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaAYghsAiiG4AaCYw60H2j4kaVvSvyS5rruSAMzS+i0f+vrlJ2+9do6VYFammXHfLOlEV4UAANo0BbftCyVdK+m2bssBAEzS2ip5s6TflHReh7UAmNLuNghWx8QZt+3rJJ1KcmzCcZu2t21v7+zszKxAAMA3ammVXCbpJ20/KekOSVfa/vO9ByXZSrKRZGNtbW3GZQIAzpgY3El+K8mFSdYlvV7SR5P8fOeVAQBGYh03ABTTvI5bkpLcJ+m+TioBADRhxg0AxRDcAFAMwQ0AxRDcAFAMwQ0AxUy1qgTA/LHNHcy4AaAYZtzACuHc3MuBGTcAFENwA0AxBDcAFENwA0AxBDcAFENwA0AxBDcAFENwA0AxBDcAFENwA0AxBDcAFENwA0AxBDcAFENwA0AxBDcAFMP5uIEVxbm562LGDQDFENwAUMzE4Lb9zbY/afsh24/a/t0+CgMAjNbS4/4fSVcm+artcyR93PZHkjzQcW0AgBEmBneSSPrq8Oo5w690WRQAYLymHrftQ7aPSzol6d4kD3ZbFgBgnKbgTvJsku+TdKGkS22/cu8xtjdtb9ve3tnZmXWdAIChqVaVJPlPSfdJumrEbVtJNpJsrK2tzag8AMBeLatK1myfP7z8HEk/JumzXRcGABitZVXJiyT9me1DGgT9e5J8sNuyAADjtKwqeVjSxT3UAgBowM5JACiG4AaAYghuACiG4AaAYjgfN1DA7nNnd33/nJt78THjBoBiCG4AKIbgBoBiCG4AKIbgBoBiCG4AKIbgBoBiCG4AKGbhNuCwEQAAzo4ZNwAUQ3ADQDEENwAUQ3ADQDEENwAUs3CrSlYRK2kATIPgBjAWk4rFRKsEAIohuAGgGIIbAIohuAGgGIIbAIohuAGgmInBbfvFtj9m+4TtR23f3EdhAIDRWtZxn5b0G0k+bfs8Scds35vksY5rAwCMMHHGneRLST49vPwVSSckXdB1YQCA0abqcdtel3SxpAdH3LZpe9v29s7OzmyqAwD8P81b3m0/X9J7Jf16ki/vvT3JlqQtSdrY2MjMKixu3Jbh3d+f9t8CWG1NwW37HA1C+11J7uq2pOU1bVgDwCgTg9u2Jb1d0okkf9B9SZgGs3Jg9bT0uC+TdIOkK20fH35d03FdAIAxJs64k3xcknuoBQDQgPNxF0FLBMAZbHkHgGIIbgAohlbJDNDGANAnZtwAUAwz7iXCzB9YDcy4AaAYghsAiiG4AaAYghsAiiG4AaAYVpVMgVUbABYBM24AKIbgBoBiCG4AKIYe95KiHw8sL2bcAFAMwQ0AxdAqAdCE9tviYMYNAMUQ3ABQDK2SFcOvu0B9BDfOiqAHFg/BPWO7g25RzKomQhxYDPS4AaAYghsAipnYKrF9u6TrJJ1K8sruS6phEVsiAFZDS4/7HZLeKumd3ZaCVl28adC/xhlMShbfxOBOcr/t9e5LQd/GvUB54QKLjR43ABQzs+WAtjclbUrS0aNHZ3W3KIA2C9CvmQV3ki1JW5K0sbGRWd3vvNE2ALBo2IADLCgmDRinZTnguyVdIemI7ZOS3pTk7V0XBqwiwhotWlaV/EwfhQAA2tAqwUzxQSXQPYIbnTlbiBPwwP4R3ACmxhvvfLEBBwCKIbgBoBhaJQAOhLZJ/5hxA0AxzLgxd8zYgOkQ3COwe20yfkYYhTfhftAqAYBimHFjaTDbw6oguIf41X8x9Bm+BH23+Pl2Z6WDm7BePQd5zAkiLIqVDm7UQWguDx7Lg+PDSQAohhk3Smv5S/V9zuqm/f/SrsN+rFxw80KZj1X/udMemN645ww/vxUMbtTRMpuuqHr9mD963ABQDDNu4ICYQU/Gz2i2CG5gHwii2aD3vz9LG9w8IQAsq6UNbmAWZrXTEstrHpNEPpwEgGKYcQOitbYI+vgNZVkeZ4Ib2IMWx2JblvA9iKbgtn2VpLdIOiTptiS3dlrVCC0PFi84YLWMy4VlD/eJwW37kKQ/kvTjkk5K+pTtu5M81nVxs0KgA8tvlV7nLR9OXirp8SRPJHlG0h2SXtdtWQCAcVpaJRdI+sKu6ycl/WA35Uxvld5lsT88R1bbQc4guagtFyc5+wH2T0t6bZJfGl6/QdKlSW7ac9ympM3h1ZdL+tw+azoi6al9/tuqGPPyW7XxSox5Wt+RZK3lwJYZ90lJL951/UJJX9x7UJItSVtN5Z2F7e0kGwe9n0oY8/JbtfFKjLlLLT3uT0l6me2X2D5X0usl3d1tWQCAcSbOuJOctv1rkv5Kg+WAtyd5tPPKAAAjNa3jTvJhSR/uuJYzDtxuKYgxL79VG6/EmDsz8cNJAMBi4SRTAFDMXILb9lW2P2f7cdu3jLjdtv9wePvDti+ZR52z1DDmnxuO9WHbn7B90TzqnKVJY9513A/Yftb29X3W14WWMdu+wvZx24/a/pu+a5y1huf2t9r+gO2HhmO+cR51zort222fsv3ImNu7z68kvX5p8AHnP0p6qaRzJT0k6Xv2HHONpI9IsqRXSXqw7zrnMOYflvSC4eWrV2HMu477qAafoVw/77p7eJzPl/SYpKPD698+77p7GPNvS/r94eU1Sf8u6dx5136AMV8u6RJJj4y5vfP8mseMu2UL/eskvTMDD0g63/aL+i50hiaOOcknkvzH8OoDGqyXr6z1VAk3SXqvpFN9FteRljH/rKS7knxekpJUH3fLmCPpPNuW9HwNgvt0v2XOTpL7NRjDOJ3n1zyCe9QW+gv2cUwl047nFzV4x65s4phtXyDppyS9rce6utTyOH+XpBfYvs/2Mdtv6K26brSM+a2SvluDjXufkXRzkq/1U95cdJ5f8zgft0d8b+/SlpZjKmkej+1XaxDcP9JpRd1rGfObJb0xybODyVh5LWM+LOn7Jb1G0nMk/Z3tB5L8Q9fFdaRlzK+VdFzSlZK+U9K9tv82yZe7Lm5OOs+veQR3yxb6pm32hTSNx/b3SrpN0tVJ/q2n2rrSMuYNSXcMQ/uIpGtsn07yl/2UOHOtz+2nkjwt6Wnb90u6SFLV4G4Z842Sbs2gAfy47X+S9ApJn+ynxN51nl/zaJW0bKG/W9Ibhp/OvkrSfyX5Ut+FztDEMds+KukuSTcUnn3tNnHMSV6SZD3JuqQ7Jf1K4dCW2p7b75f0o7YP236uBmfaPNFznbPUMubPa/Abhmy/UIOT0D3Ra5X96jy/ep9xZ8wWetu/PLz9bRqsMLhG0uOS/luDd+yyGsf8O5K+TdIfD2egp1P4BD2NY14qLWNOcsL2PZIelvQ1Df6i1MhlZRU0Ps6/J+kdtj+jQRvhjUnKnjXQ9rslXSHpiO2Tkt4k6Rypv/xi5yQAFMPOSQAohuAGgGIIbgAohuAGgGIIbgAohuAGgGIIbgAohuAGgGL+F7j7kP9O/Nh3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc_sim = acc[:,1]\n",
    "acc_sim\n",
    "# similarity distribution\n",
    "num_bins = 100\n",
    "fig, ax = plt.subplots()\n",
    "n, bins, patches = ax.hist(acc_sim, num_bins, density = 1)\n",
    "# ax.plot(bins)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pick another topK if you want to get the repetition dataset\n",
    "new_data = data[acc[:200,0].astype(int),:,:]\n",
    "np.save(\"irish-dis-measure-vae-validate-non-repetition.npy\", new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}