Taxi_v3.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Taxi_v3.ipynb","provenance":[{"file_id":"10sIcVq_8RZNeHplzhJgGfsFrhAD9JdNd","timestamp":1657564559362}],"collapsed_sections":[],"authorship_tag":"ABX9TyPTNT9isWrolPuJvvsygMEc"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# **Taxi 3**"],"metadata":{"id":"I97Wy0tujCAC"}},{"cell_type":"code","execution_count":1,"metadata":{"id":"V3I58Hu-icox","executionInfo":{"status":"ok","timestamp":1657566837176,"user_tz":-120,"elapsed":421,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}}},"outputs":[],"source":["# Imports\n","\n","import numpy as np\n","import gym\n","import random"]},{"cell_type":"code","source":["# Create environment: Taxi-v3 from OpenAI Gym\n","env = gym.make(\"Taxi-v3\")\n","\n","# Render gym\n","env.render()"],"metadata":{"id":"m5DujEHQimdd","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1657566839789,"user_tz":-120,"elapsed":15,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}},"outputId":"0722e0a2-97df-4ce6-fbe4-992f36b8e042"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["+---------+\n","|R: | : :G|\n","| : | : : |\n","| : : : : |\n","| | : | : |\n","|\u001b[34;1mY\u001b[0m| :\u001b[43m \u001b[0m|\u001b[35mB\u001b[0m: |\n","+---------+\n","\n"]}]},{"cell_type":"code","source":["# Create and initialize Q-table\n","action_size = env.action_space.n\n","print(\"Action size:\", action_size)\n","\n","state_size = env.observation_space.n\n","print(\"State size:\", state_size)"],"metadata":{"id":"cysPk4s7jNuZ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1657566851078,"user_tz":-120,"elapsed":409,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}},"outputId":"5f147051-6661-422b-ef28-62a9f95122e7"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["Action size: 6\n","State size: 500\n"]}]},{"cell_type":"code","source":["# Fill Q-table with zeros \n","q_table = np.zeros((state_size, action_size))\n","print(q_table)"],"metadata":{"id":"23nsho3Fj4u2","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1657566855530,"user_tz":-120,"elapsed":381,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}},"outputId":"1837f55d-c083-4959-9147-2aa0837b4595"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["[[0. 0. 0. 0. 0. 0.]\n"," [0. 0. 0. 0. 0. 0.]\n"," [0. 0. 0. 0. 0. 0.]\n"," ...\n"," [0. 0. 0. 0. 0. 0.]\n"," [0. 0. 0. 0. 0. 0.]\n"," [0. 0. 0. 0. 0. 0.]]\n"]}]},{"cell_type":"markdown","source":["**3. Hyperparameters**"],"metadata":{"id":"pnMSwCgbj8PA"}},{"cell_type":"code","source":["# Number of episodes\n","num_episodes = 50000 \n","\n","# Max steps per episode\n","num_steps = 99 \n","\n","# Learning rate\n","learning_rate = 0.06\n","\n","# Discount rate\n","gamma = 0.86\n","\n","# Exploration rate & its max starting value\n","epsilon = 0.95\n","epsilon_max = 0.95\n","\n","# Minimum exploration probability \n","epsilon_min = 0.01   \n","\n","# Decay rate\n","decay_rate = 0.0099             "],"metadata":{"id":"Hk-RPFCrkEYQ","executionInfo":{"status":"ok","timestamp":1657568644345,"user_tz":-120,"elapsed":5,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}}},"execution_count":40,"outputs":[]},{"cell_type":"code","source":["# Q-learning\n","\n","for episode in range(num_episodes):\n","    # Reset the environment before each episode\n","    state = env.reset()\n","\n","    step = 0\n","    done = False\n","    \n","    for step in range(num_steps):\n","        # Epsilon-greedy strategy\n","        explore_probability = random.uniform(0,1)\n","        \n","        # In case of exploitation\n","        if explore_probability > epsilon:\n","            action = np.argmax(q_table[state,:])\n","        \n","        # In case of exploration\n","        else:\n","            action = env.action_space.sample()\n","        \n","        # Take the action and get new state and reward\n","        new_state, reward, done, info = env.step(action)\n","\n","        # Update Q-table\n","        q_table[state, action] = q_table[state, action] + learning_rate * (reward + gamma * np.max(q_table[new_state, :]) - q_table[state, action])\n","                \n","        # Update state\n","        state = new_state\n","        \n","        # If done: finish episode\n","        if done: \n","            break\n","    \n","    # Reduce epsilon\n","    epsilon = epsilon_min + (epsilon_max - epsilon_min) * np.exp(-decay_rate * episode) "],"metadata":{"id":"JtPUsowskJDb","executionInfo":{"status":"ok","timestamp":1657568670499,"user_tz":-120,"elapsed":24392,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}}},"execution_count":41,"outputs":[]},{"cell_type":"code","source":["# Test\n","\n","env.reset()\n","rewards = []\n","num_test_episodes = 99\n","\n","\n","for episode in range(num_test_episodes):\n","    state = env.reset()\n","    step = 0\n","    done = False\n","    total_rewards = 0\n","\n","    for step in range(num_steps):\n","        action = np.argmax(q_table[state,:])  \n","        new_state, reward, done, info = env.step(action)\n","        #env.render()\n","        total_rewards += reward\n","        \n","        if done:\n","            rewards.append(total_rewards)\n","            break\n","        state = new_state\n","env.close()\n","print (\"Score: \" +  str(sum(rewards) / num_test_episodes))"],"metadata":{"id":"RzwVyOSYkgTy","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1657568672761,"user_tz":-120,"elapsed":414,"user":{"displayName":"Adrienn Horváth","userId":"02789652139741753412"}},"outputId":"38329dee-dafc-4eda-9673-105b7c1650cc"},"execution_count":42,"outputs":[{"output_type":"stream","name":"stdout","text":["Score: 8.242424242424242\n"]}]}]}