small_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from nlp_utils import preprocess_sentence, TextTokenizing\n",
    "from transformer import transformer, CustomSchedule, loss_function\n",
    "from utils import make_checkpoint, accuracy, load_csv_and_processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21801"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = pd.read_csv(\"./super_small_4_dataset.csv\")\n",
    "train_data.head()\n",
    "\n",
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21794, 21794)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions, answers = load_csv_and_processing(\"./super_small_4_dataset.csv\")\n",
    "\n",
    "# questions = questions[15000:]\n",
    "# answers = answers[15000:]\n",
    "\n",
    "len(questions), len(answers)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For Small Model -> Small Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(str,\n",
       " ['응  난 엄청 근처에 있어  .',\n",
       "  '집 근처에 병원이 있으면 편리한 거 같아  .',\n",
       "  '언제 사고가 발생할지 모르잖아  .',\n",
       "  '나는 집 근처에 병원이 있어서 다행이야  .'],\n",
       " '힘들어서 결혼할까봐')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type (questions[0]), answers[:4], questions[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "textTokenizing = TextTokenizing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
    "#    questions+answers, target_vocab_size=2**16\n",
    "#\n",
    "\n",
    "tokenizer = textTokenizing.create_tokenizer(questions, answers, target_vocab_size=2**16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "textTokenizing.save_tokenizer(\"super_ultra_small_vocab\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = textTokenizing.load_tokenizer(\"super_ultra_small_vocab\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54759, [54757], [54758])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VOCAB_SIZE, START_TOKEN, END_TOKEN = textTokenizing.tokens()\n",
    "\n",
    "VOCAB_SIZE, START_TOKEN, END_TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions, answers = textTokenizing.tokenize_and_filter(questions, answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "질문 데이터의 크기:(21794, 50)\n",
      "답변 데이터의 크기:(21794, 50)\n"
     ]
    }
   ],
   "source": [
    "print(f'질문 데이터의 크기:{questions.shape}')\n",
    "print(f'답변 데이터의 크기:{answers.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "BATCH_SIZE = 64\n",
    "BUFFER_SIZE = 20000\n",
    "\n",
    "dataset = textTokenizing.make_dataset(BATCH_SIZE, BUFFER_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[54757   226    29    28    54  2355    26     1 54758     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0]\n",
      "[[54757   226    29    28    54  2355    26     1 54758     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0]]\n",
      "[[  226    29    28    54  2355    26     1 54758     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0]]\n"
     ]
    }
   ],
   "source": [
    "# 임의의 샘플에 대해서 [:, :-1]과 [:, 1:]이 어떤 의미를 가지는지 테스트해본다.\n",
    "print(answers[0]) # 기존 샘플\n",
    "print(answers[:1][:, :-1]) # 마지막 패딩 토큰 제거하면서 길이가 39가 된다.\n",
    "print(answers[:1][:, 1:]) # 맨 처음 토큰이 제거된다. 다시 말해 시작 토큰이 제거된다. 길이는 역시 39가 된다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hyperparameters import NUM_LAYERS, D_MODEL, NUM_HEADS, DFF, DROPOUT, MAX_LENGTH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 54759, 256)\n",
      "(1, 54759, 256)\n"
     ]
    }
   ],
   "source": [
    "model = transformer(\n",
    "    vocab_size=VOCAB_SIZE,\n",
    "    num_layers=NUM_LAYERS,\n",
    "    dff=DFF,\n",
    "    d_model=D_MODEL,\n",
    "    num_heads=NUM_HEADS,\n",
    "    dropout=DROPOUT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"transformer\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " inputs (InputLayer)            [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " dec_inputs (InputLayer)        [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " enc_padding_mask (Lambda)      (None, 1, 1, None)   0           ['inputs[0][0]']                 \n",
      "                                                                                                  \n",
      " encoder (Functional)           (None, None, 256)    15072512    ['inputs[0][0]',                 \n",
      "                                                                  'enc_padding_mask[0][0]']       \n",
      "                                                                                                  \n",
      " look_ahead_mask (Lambda)       (None, 1, None, Non  0           ['dec_inputs[0][0]']             \n",
      "                                e)                                                                \n",
      "                                                                                                  \n",
      " dec_padding_mask (Lambda)      (None, 1, 1, None)   0           ['inputs[0][0]']                 \n",
      "                                                                                                  \n",
      " decoder (Functional)           (None, None, 256)    15599872    ['dec_inputs[0][0]',             \n",
      "                                                                  'encoder[0][0]',                \n",
      "                                                                  'look_ahead_mask[0][0]',        \n",
      "                                                                  'dec_padding_mask[0][0]']       \n",
      "                                                                                                  \n",
      " outputs (Dense)                (None, None, 54759)  14073063    ['decoder[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 44,745,447\n",
      "Trainable params: 44,745,447\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp_callback = make_checkpoint(\"training_super_small/cp-{epoch:04d}.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(D_MODEL)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(\n",
    "    learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)\n",
    "\n",
    "\n",
    "model.compile(optimizer=optimizer, loss=loss_function, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_opts = tf.compat.v1.RunOptions(report_tensor_allocations_upon_oom=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      " 80/341 [======>.......................] - ETA: 3:56 - loss: 1.5509 - accuracy: 0.0096"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_9884/3441346976.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mEPOCHS\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mEPOCHS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcp_callback\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\utils\\traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=broad-except\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1373\u001b[0m               \u001b[0mlogs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtmp_logs\u001b[0m  \u001b[1;31m# No error, now safe to assign to logs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1374\u001b[0m               \u001b[0mend_step\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1375\u001b[1;33m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1376\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1377\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m    436\u001b[0m     \"\"\"\n\u001b[0;32m    437\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 438\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'end'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    439\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    440\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[1;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[0;32m    295\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    296\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'end'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 297\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    298\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    299\u001b[0m       raise ValueError(\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[1;34m(self, mode, batch, logs)\u001b[0m\n\u001b[0;32m    316\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    317\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 318\u001b[1;33m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    319\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[1;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[0;32m    354\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    355\u001b[0m       \u001b[0mhook\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 356\u001b[1;33m       \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    357\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    358\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m   1378\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1379\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_should_save_on_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1380\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_save_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_current_epoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1381\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1382\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mon_epoch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36m_save_model\u001b[1;34m(self, epoch, batch, logs)\u001b[0m\n\u001b[0;32m   1453\u001b[0m                 f'\\nEpoch {epoch + 1}: saving model to {filepath}')\n\u001b[0;32m   1454\u001b[0m           \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_weights_only\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1455\u001b[1;33m             self.model.save_weights(\n\u001b[0m\u001b[0;32m   1456\u001b[0m                 filepath, overwrite=True, options=self._options)\n\u001b[0;32m   1457\u001b[0m           \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\utils\\traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=broad-except\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36msave_weights\u001b[1;34m(self, filepath, overwrite, save_format, options)\u001b[0m\n\u001b[0;32m   2508\u001b[0m       \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2509\u001b[0m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2510\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_trackable_saver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2511\u001b[0m       \u001b[1;31m# Record this checkpoint so it's visible from tf.train.latest_checkpoint.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2512\u001b[0m       tf.__internal__.train.update_checkpoint_state(\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\util.py\u001b[0m in \u001b[0;36msave\u001b[1;34m(self, file_prefix, checkpoint_number, session, options)\u001b[0m\n\u001b[0;32m   1269\u001b[0m       \u001b[0mfile_io\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecursive_create_dir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdirname\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_prefix\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1270\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1271\u001b[1;33m     save_path, new_feed_additions = self._save_cached_when_graph_building(\n\u001b[0m\u001b[0;32m   1272\u001b[0m         file_prefix_tensor, object_graph_tensor, options)\n\u001b[0;32m   1273\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mnew_feed_additions\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\util.py\u001b[0m in \u001b[0;36m_save_cached_when_graph_building\u001b[1;34m(self, file_prefix, object_graph_tensor, options)\u001b[0m\n\u001b[0;32m   1213\u001b[0m       saver = functional_saver.MultiDeviceSaver(named_saveable_objects,\n\u001b[0;32m   1214\u001b[0m                                                 registered_savers)\n\u001b[1;32m-> 1215\u001b[1;33m       \u001b[0msave_op\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msaver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1216\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"/cpu:0\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1217\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrol_dependencies\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msave_op\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\training\\saving\\functional_saver.py\u001b[0m in \u001b[0;36msave\u001b[1;34m(self, file_prefix, options)\u001b[0m\n\u001b[0;32m    369\u001b[0m       \u001b[0mtf_function_save\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 371\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0msave_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    372\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    373\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfile_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\training\\saving\\functional_saver.py\u001b[0m in \u001b[0;36msave_fn\u001b[1;34m()\u001b[0m\n\u001b[0;32m    343\u001b[0m           \u001b[1;31m# initial read operations should be placed on the SaveableObject's\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m           \u001b[1;31m# device.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 345\u001b[1;33m           \u001b[0msharded_saves\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msaver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshard_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    346\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrol_dependencies\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msharded_saves\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\training\\saving\\functional_saver.py\u001b[0m in \u001b[0;36msave\u001b[1;34m(self, file_prefix, options)\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[0msave_device\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_io_device\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;34m\"cpu:0\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msave_device\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 80\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mio_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_v2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_names\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_slices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfile_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\ops\\gen_io_ops.py\u001b[0m in \u001b[0;36msave_v2\u001b[1;34m(prefix, tensor_names, shape_and_slices, tensors, name)\u001b[0m\n\u001b[0;32m   1704\u001b[0m       \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1705\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1706\u001b[1;33m       return save_v2_eager_fallback(\n\u001b[0m\u001b[0;32m   1707\u001b[0m           \u001b[0mprefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_names\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape_and_slices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1708\u001b[0m           ctx=_ctx)\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\ops\\gen_io_ops.py\u001b[0m in \u001b[0;36msave_v2_eager_fallback\u001b[1;34m(prefix, tensor_names, shape_and_slices, tensors, name, ctx)\u001b[0m\n\u001b[0;32m   1725\u001b[0m   \u001b[0m_inputs_flat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mprefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_names\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape_and_slices\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1726\u001b[0m   \u001b[0m_attrs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m\"dtypes\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_attr_dtypes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1727\u001b[1;33m   _result = _execute.execute(b\"SaveV2\", 0, inputs=_inputs_flat, attrs=_attrs,\n\u001b[0m\u001b[0;32m   1728\u001b[0m                              ctx=ctx, name=name)\n\u001b[0;32m   1729\u001b[0m   \u001b[0m_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\YSH\\anaconda3\\envs\\mlenv\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     52\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 54\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     55\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     56\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "EPOCHS = 20\n",
    "model.fit(dataset, epochs=EPOCHS, callbacks=[cp_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "9585e0e58f3ada4c387d89b399b9d9bb88b52954ed4e2235f58d5a052e970ed6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}