Scrape Spotify Previews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5dced9d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import librosa\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "import skimage.io as skio\n",
    "from skimage.transform import resize\n",
    "import pandas as pd\n",
    "import os\n",
    "import requests\n",
    "from pyarrow import feather\n",
    "import io\n",
    "import concurrent.futures\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1359538f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_audio_to_mel_image(filepath_to_audio, filepath_to_save, image_size=(128,512), n_mels=128, fmax=8000,):\n",
    "    signal, sr = librosa.load(filepath_to_audio)\n",
    "    mels = librosa.power_to_db(librosa.feature.melspectrogram(y=signal, sr=sr, n_mels=n_mels, fmax=fmax), ref=np.max)\n",
    "    image = (((80+mels)/80)*255)\n",
    "    image = np.flip(image, axis=0)\n",
    "    resize_image = resize(image, image_size).astype(np.uint8)\n",
    "    skio.imsave(filepath_to_save, resize_image)\n",
    "    \n",
    "def convert_audio_to_composite_image(filepath_to_audio, filepath_to_save, image_size=(128,512), n_mels=128, fmax=8000,):\n",
    "    \n",
    "    signal, sr = librosa.load(filepath_to_audio)\n",
    "    \n",
    "    mels = librosa.power_to_db(librosa.feature.melspectrogram(y=signal, sr=sr, n_mels=n_mels, fmax=fmax), ref=np.max)\n",
    "    mel_image = (((80+mels)/80)*255)\n",
    "    mel_image = np.flip(mel_image, axis=0)\n",
    "    mel_image = resize(mel_image, (128,512)).astype(np.uint8)\n",
    "    \n",
    "    mfcc = librosa.power_to_db(librosa.feature.mfcc(y=signal, sr=sr, n_mfcc=128, fmax=8000), ref=np.max)\n",
    "    mfcc_image = (((80+mfcc)/80)*255)\n",
    "    mfcc_image = np.flip(mfcc_image, axis=0)\n",
    "    mfcc_image = resize(mfcc_image, (128,512)).astype(np.uint8)\n",
    "    \n",
    "    chromagram = librosa.feature.chroma_cqt(y=signal, sr=sr)\n",
    "    chroma_image = resize(chromagram*255, (128,512)).astype(np.uint8)\n",
    "    \n",
    "    composite = np.dstack((mel_image, mfcc_image, chroma_image))\n",
    "    \n",
    "    skio.imsave(filepath_to_save, composite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "29beb518",
   "metadata": {},
   "outputs": [],
   "source": [
    "tracks_df = feather.read_feather('data/all_tracks.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "72aa474a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tracks_df = tracks_df.dropna(subset=['track_preview_link'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "48ab9e43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>track_name</th>\n",
       "      <th>track_preview_link</th>\n",
       "      <th>track_popularity</th>\n",
       "      <th>track_uri</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>artist_id</th>\n",
       "      <th>artist_genres</th>\n",
       "      <th>artist_popularity</th>\n",
       "      <th>release_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3SdSjZ651RbiG0rDEh8AQm</td>\n",
       "      <td>Electrofunk</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/8b32113c205b135d...</td>\n",
       "      <td>1</td>\n",
       "      <td>spotify:track:3SdSjZ651RbiG0rDEh8AQm</td>\n",
       "      <td>TomBad</td>\n",
       "      <td>4T3DN9PLFsn3I8t65jfDAw</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-10-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0YLZeGKEk0N921qz6wpW1p</td>\n",
       "      <td>Silly Puddy</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/b61e27bbc60440d6...</td>\n",
       "      <td>39</td>\n",
       "      <td>spotify:track:0YLZeGKEk0N921qz6wpW1p</td>\n",
       "      <td>Zion I</td>\n",
       "      <td>3LLs28LJVlXAjWc8UDkcQZ</td>\n",
       "      <td>[abstract hip hop, alternative hip hop, pop ra...</td>\n",
       "      <td>45</td>\n",
       "      <td>2009-09-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1Sej8madYsRgWZaYDlrxqE</td>\n",
       "      <td>Her Worth (Remix)</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/c771a0a178063357...</td>\n",
       "      <td>0</td>\n",
       "      <td>spotify:track:1Sej8madYsRgWZaYDlrxqE</td>\n",
       "      <td>Devon Sanders</td>\n",
       "      <td>5Ob0wqCDOHjYqBPGQCBO9W</td>\n",
       "      <td>[]</td>\n",
       "      <td>12</td>\n",
       "      <td>2020-05-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6RdHnDdeayZWNfN6HFvN6Y</td>\n",
       "      <td>Love Game - Single Version</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/1fedaca4eda2ebaf...</td>\n",
       "      <td>5</td>\n",
       "      <td>spotify:track:6RdHnDdeayZWNfN6HFvN6Y</td>\n",
       "      <td>Facts &amp; Fiction</td>\n",
       "      <td>460oEmwFYbERxguQI2Z0vX</td>\n",
       "      <td>[italian disco, new italo disco]</td>\n",
       "      <td>22</td>\n",
       "      <td>2010-02-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60rIdEPDrzyLiLC0icp3xz</td>\n",
       "      <td>Flower</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/bc5c66092e62968e...</td>\n",
       "      <td>55</td>\n",
       "      <td>spotify:track:60rIdEPDrzyLiLC0icp3xz</td>\n",
       "      <td>Moby</td>\n",
       "      <td>3OsRAKCvk37zwYcnzRf5XF</td>\n",
       "      <td>[downtempo, electronica]</td>\n",
       "      <td>67</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563288</th>\n",
       "      <td>57dJUvTqBEErXvGsViASbV</td>\n",
       "      <td>Wait A While</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/dc685df30cd2f361...</td>\n",
       "      <td>17</td>\n",
       "      <td>spotify:track:57dJUvTqBEErXvGsViASbV</td>\n",
       "      <td>Old Man Luedecke</td>\n",
       "      <td>1L29dr4NAXlj2tGo51txba</td>\n",
       "      <td>[canadian americana, canadian folk, canadian i...</td>\n",
       "      <td>27</td>\n",
       "      <td>2015-07-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563289</th>\n",
       "      <td>3YSr4sgJRsYqQgi6XVbP4d</td>\n",
       "      <td>Bu Aşk</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/105ca7017607da2f...</td>\n",
       "      <td>41</td>\n",
       "      <td>spotify:track:3YSr4sgJRsYqQgi6XVbP4d</td>\n",
       "      <td>Gökhan Kırdar</td>\n",
       "      <td>0iTcX6LMMIt0DeYJ3qBSQX</td>\n",
       "      <td>[turkish pop, turkish rock, turkish singer-son...</td>\n",
       "      <td>41</td>\n",
       "      <td>1999-06-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563290</th>\n",
       "      <td>5WKgWQSpOC7xvMHyW9pW1j</td>\n",
       "      <td>The Giant</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/487edd889baff2e8...</td>\n",
       "      <td>2</td>\n",
       "      <td>spotify:track:5WKgWQSpOC7xvMHyW9pW1j</td>\n",
       "      <td>Chris Thornborrow</td>\n",
       "      <td>1WNCJp42oApfoL0wB29oCB</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563291</th>\n",
       "      <td>4R3uUe9AAVa0ltlCWWS8Io</td>\n",
       "      <td>Better</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/d97b8f37da2b67d9...</td>\n",
       "      <td>35</td>\n",
       "      <td>spotify:track:4R3uUe9AAVa0ltlCWWS8Io</td>\n",
       "      <td>Mike Donehey</td>\n",
       "      <td>2jKlK6gsKyZDItyVP36II2</td>\n",
       "      <td>[ccm, christian pop]</td>\n",
       "      <td>37</td>\n",
       "      <td>2020-12-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563292</th>\n",
       "      <td>2BJXYZYFjLoYDKKBqFXxIL</td>\n",
       "      <td>Yanımda Sen Olmayınca</td>\n",
       "      <td>https://p.scdn.co/mp3-preview/f25a8df4cd8c2943...</td>\n",
       "      <td>45</td>\n",
       "      <td>spotify:track:2BJXYZYFjLoYDKKBqFXxIL</td>\n",
       "      <td>Koray Avcı</td>\n",
       "      <td>3sV7ijrP5xMHgWRlOnPTCi</td>\n",
       "      <td>[karadeniz turkuleri, turkish pop]</td>\n",
       "      <td>51</td>\n",
       "      <td>2016-07-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>563293 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      track_id                  track_name  \\\n",
       "0       3SdSjZ651RbiG0rDEh8AQm                 Electrofunk   \n",
       "1       0YLZeGKEk0N921qz6wpW1p                 Silly Puddy   \n",
       "2       1Sej8madYsRgWZaYDlrxqE           Her Worth (Remix)   \n",
       "3       6RdHnDdeayZWNfN6HFvN6Y  Love Game - Single Version   \n",
       "4       60rIdEPDrzyLiLC0icp3xz                      Flower   \n",
       "...                        ...                         ...   \n",
       "563288  57dJUvTqBEErXvGsViASbV                Wait A While   \n",
       "563289  3YSr4sgJRsYqQgi6XVbP4d                      Bu Aşk   \n",
       "563290  5WKgWQSpOC7xvMHyW9pW1j                   The Giant   \n",
       "563291  4R3uUe9AAVa0ltlCWWS8Io                      Better   \n",
       "563292  2BJXYZYFjLoYDKKBqFXxIL       Yanımda Sen Olmayınca   \n",
       "\n",
       "                                       track_preview_link  track_popularity  \\\n",
       "0       https://p.scdn.co/mp3-preview/8b32113c205b135d...                 1   \n",
       "1       https://p.scdn.co/mp3-preview/b61e27bbc60440d6...                39   \n",
       "2       https://p.scdn.co/mp3-preview/c771a0a178063357...                 0   \n",
       "3       https://p.scdn.co/mp3-preview/1fedaca4eda2ebaf...                 5   \n",
       "4       https://p.scdn.co/mp3-preview/bc5c66092e62968e...                55   \n",
       "...                                                   ...               ...   \n",
       "563288  https://p.scdn.co/mp3-preview/dc685df30cd2f361...                17   \n",
       "563289  https://p.scdn.co/mp3-preview/105ca7017607da2f...                41   \n",
       "563290  https://p.scdn.co/mp3-preview/487edd889baff2e8...                 2   \n",
       "563291  https://p.scdn.co/mp3-preview/d97b8f37da2b67d9...                35   \n",
       "563292  https://p.scdn.co/mp3-preview/f25a8df4cd8c2943...                45   \n",
       "\n",
       "                                   track_uri        artist_name  \\\n",
       "0       spotify:track:3SdSjZ651RbiG0rDEh8AQm             TomBad   \n",
       "1       spotify:track:0YLZeGKEk0N921qz6wpW1p             Zion I   \n",
       "2       spotify:track:1Sej8madYsRgWZaYDlrxqE      Devon Sanders   \n",
       "3       spotify:track:6RdHnDdeayZWNfN6HFvN6Y    Facts & Fiction   \n",
       "4       spotify:track:60rIdEPDrzyLiLC0icp3xz               Moby   \n",
       "...                                      ...                ...   \n",
       "563288  spotify:track:57dJUvTqBEErXvGsViASbV   Old Man Luedecke   \n",
       "563289  spotify:track:3YSr4sgJRsYqQgi6XVbP4d      Gökhan Kırdar   \n",
       "563290  spotify:track:5WKgWQSpOC7xvMHyW9pW1j  Chris Thornborrow   \n",
       "563291  spotify:track:4R3uUe9AAVa0ltlCWWS8Io       Mike Donehey   \n",
       "563292  spotify:track:2BJXYZYFjLoYDKKBqFXxIL         Koray Avcı   \n",
       "\n",
       "                     artist_id  \\\n",
       "0       4T3DN9PLFsn3I8t65jfDAw   \n",
       "1       3LLs28LJVlXAjWc8UDkcQZ   \n",
       "2       5Ob0wqCDOHjYqBPGQCBO9W   \n",
       "3       460oEmwFYbERxguQI2Z0vX   \n",
       "4       3OsRAKCvk37zwYcnzRf5XF   \n",
       "...                        ...   \n",
       "563288  1L29dr4NAXlj2tGo51txba   \n",
       "563289  0iTcX6LMMIt0DeYJ3qBSQX   \n",
       "563290  1WNCJp42oApfoL0wB29oCB   \n",
       "563291  2jKlK6gsKyZDItyVP36II2   \n",
       "563292  3sV7ijrP5xMHgWRlOnPTCi   \n",
       "\n",
       "                                            artist_genres  artist_popularity  \\\n",
       "0                                                      []                  1   \n",
       "1       [abstract hip hop, alternative hip hop, pop ra...                 45   \n",
       "2                                                      []                 12   \n",
       "3                        [italian disco, new italo disco]                 22   \n",
       "4                                [downtempo, electronica]                 67   \n",
       "...                                                   ...                ...   \n",
       "563288  [canadian americana, canadian folk, canadian i...                 27   \n",
       "563289  [turkish pop, turkish rock, turkish singer-son...                 41   \n",
       "563290                                                 []                  1   \n",
       "563291                               [ccm, christian pop]                 37   \n",
       "563292                 [karadeniz turkuleri, turkish pop]                 51   \n",
       "\n",
       "       release_date  \n",
       "0        2004-10-15  \n",
       "1        2009-09-08  \n",
       "2        2020-05-23  \n",
       "3        2010-02-23  \n",
       "4              None  \n",
       "...             ...  \n",
       "563288   2015-07-24  \n",
       "563289   1999-06-02  \n",
       "563290   2016-06-28  \n",
       "563291   2020-12-15  \n",
       "563292   2016-07-01  \n",
       "\n",
       "[563293 rows x 10 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_tracks_df = tracks_df.sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "random_tracks_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6145a610",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_preview_with_index(index):\n",
    "    track_url = random_tracks_df.loc[index, 'track_preview_link']\n",
    "\n",
    "    preview = requests.get(track_url)\n",
    "    \n",
    "    track_id = random_tracks_df.loc[index, 'track_id']\n",
    "\n",
    "    filename = f'data/Spotify/mp3s/{track_id}.mp3'\n",
    "    png_name = f'data/Spotify/comp_pngs/{track_id}.png'\n",
    "\n",
    "    with open(filename, 'wb') as f:\n",
    "        f.write(preview.content)\n",
    "\n",
    "    convert_audio_to_composite_image(filename, png_name)\n",
    "\n",
    "    os.remove(filename)\n",
    "    \n",
    "    print(index, end='\\r')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ff260fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "indices = tracks_df.index\n",
    "MAX_THREADS = 50\n",
    "threads = min(MAX_THREADS, len(indices))\n",
    "count = 0\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:\n",
    "    executor.map(download_preview_with_index, indices)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}