dataset_download.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "from pytube import YouTube\n",
    "import os\n",
    "from tqdm import tqdm  \n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "dataset_root_path = '/mnt/Work/Dataset/EEV/'\n",
    "train = pd.read_csv('{}/eev/train.csv'.format(dataset_root_path))\n",
    "test = pd.read_csv('{}/eev/test.csv'.format(dataset_root_path))\n",
    "val = pd.read_csv('{}/eev/val.csv'.format(dataset_root_path))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "num_videos_train = train['YouTube ID'].unique()\n",
    "num_video_val = val['YouTube ID'].unique()\n",
    "num_video_test = test['Video ID'].unique()\n",
    "print(len(num_videos_train), len(num_video_val), len(num_video_test))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "3061 755 1337\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "def download_youtube_video(video_id, path):\n",
    "    try:\n",
    "        yt = YouTube('https://youtu.be/{}'.format(video_id))\n",
    "        is_availability = yt.check_availability()\n",
    "        filename = yt.streams.filter(file_extension='mp4').first().default_filename\n",
    "        filename_write = '{}.{}'.format(video_id, filename.split('.')[-1])\n",
    "        # print(os.path.join(path, filename_write))\n",
    "        if os.path.isfile(os.path.join(path, filename_write)):\n",
    "            return ''\n",
    "        yt.streams.filter(file_extension='mp4').first().download(path, filename_write)\n",
    "        # return ''\n",
    "    except:\n",
    "        print('Please check ', video_id)\n",
    "        # return video_id\n",
    "    "
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "os.makedirs('{}/dataset'.format(dataset_root_path), exist_ok=True)\n",
    "os.makedirs('{}/dataset/train'.format(dataset_root_path), exist_ok=True)\n",
    "os.makedirs('{}/dataset/val'.format(dataset_root_path), exist_ok=True)\n",
    "os.makedirs('{}/dataset/test'.format(dataset_root_path), exist_ok=True)\n",
    "\n",
    "# download_youtube_video(num_videos_train[0],  '{}/dataset/train'.format(dataset_root_path))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "print('Train')\n",
    "p = Parallel(n_jobs=20)(delayed(download_youtube_video)(num_videos_train[idx], '{}/dataset/train'.format(dataset_root_path)) for idx in tqdm(range(len(num_videos_train))))\n",
    "print('Val')\n",
    "pv = Parallel(n_jobs=20)(delayed(download_youtube_video)(num_video_val[idx], '{}/dataset/val'.format(dataset_root_path)) for idx in tqdm(range(len(num_video_val))))\n",
    "print('Test')\n",
    "pt = Parallel(n_jobs=20)(delayed(download_youtube_video)(num_video_test[idx], '{}/dataset/test'.format(dataset_root_path)) for idx in tqdm(range(len(num_video_test))))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Test\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "list_test = [x.split('.')[0] for x in os.listdir('/mnt/Work/Dataset/EEV/dataset/test/') if x.endswith('.mp4')]\n",
    "print(len(list_test))\n",
    "list_train = [x.split('.')[0] for x in os.listdir('/mnt/Work/Dataset/EEV/dataset/train/') if x.endswith('.mp4')]\n",
    "\n",
    "for x in num_videos_train:\n",
    "    if x not in list_train:\n",
    "        print(x)\n",
    "        download_youtube_video(x, '{}/dataset/train'.format(dataset_root_path))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "1329\n",
      "2Cj3VB34ugs\n",
      "Please check  2Cj3VB34ugs\n",
      "4R27wVIz5mI\n",
      "Please check  4R27wVIz5mI\n",
      "9gn7JVFDi-o\n",
      "Please check  9gn7JVFDi-o\n",
      "AdMpVZ42RLk\n",
      "Please check  AdMpVZ42RLk\n",
      "BblkjAFyaUo\n",
      "Please check  BblkjAFyaUo\n",
      "G3QqV8lLWf8\n",
      "Please check  G3QqV8lLWf8\n",
      "JM1nFVVE9CQ\n",
      "Please check  JM1nFVVE9CQ\n",
      "KA1k9VCW5_4\n",
      "Please check  KA1k9VCW5_4\n",
      "KQRpGNdG3f0\n",
      "Please check  KQRpGNdG3f0\n",
      "MIvXeH6o5po\n",
      "Please check  MIvXeH6o5po\n",
      "PwkIUx-mviQ\n",
      "Please check  PwkIUx-mviQ\n",
      "U3h-WOr2C-g\n",
      "Please check  U3h-WOr2C-g\n",
      "UEP_ivCp-SI\n",
      "Please check  UEP_ivCp-SI\n",
      "_BbuL_OYqns\n",
      "Please check  _BbuL_OYqns\n",
      "dA_Bt8YDtFg\n",
      "Please check  dA_Bt8YDtFg\n",
      "ebybqir7AM0\n",
      "Please check  ebybqir7AM0\n",
      "fB2JYVTQinQ\n",
      "Please check  fB2JYVTQinQ\n",
      "hvOEIKmgHow\n",
      "Please check  hvOEIKmgHow\n",
      "lI-oAW8LR3c\n",
      "Please check  lI-oAW8LR3c\n",
      "lWRLLDVJ0fE\n",
      "Please check  lWRLLDVJ0fE\n",
      "liPqMGBGO40\n",
      "Please check  liPqMGBGO40\n",
      "mNWKNHIt5JA\n",
      "Please check  mNWKNHIt5JA\n",
      "nZHolGhOr18\n",
      "Please check  nZHolGhOr18\n",
      "oveDwY9VEJc\n",
      "Please check  oveDwY9VEJc\n"
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}