Simulated_American_put_option.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.optimize import minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class American_put_option():\n",
    "    def __init__(self, p0, d, seed=1):\n",
    "        np.random.seed(seed)\n",
    "        self.lower = 60\n",
    "        self.upper = 160\n",
    "        self.p0 = p0\n",
    "        self.A = [0, 1] # a=0 is not exercising, a=1 is exercising\n",
    "        self.K = 100\n",
    "        self.epsilon = 5\n",
    "        self.c = [1.02, 0.98]\n",
    "        self.s0 = np.random.uniform(self.K - self.epsilon, self.K + self.epsilon, 1)\n",
    "        self.d = d\n",
    "        self.Delta = (self.upper - self.lower) / d\n",
    "        self.anchor = [80 + x * self.Delta for x in range(d)]\n",
    "\n",
    "    def reset(self):\n",
    "        self.S = [self.s0]\n",
    "        self.A = [] # save the action history\n",
    "        self.R = [] # save the reward history\n",
    "        self.h = 0 # reset the step to 0\n",
    "        self.feature = [] # save the feature trajectory with respect to a=0\n",
    "        self.current_state = self.s0  # reset the current state to initial state\n",
    "\n",
    "    def phi(self, s):\n",
    "        phi = [max(0, (1 - np.abs(s - si) / self.Delta)[0]) for si in self.anchor]    \n",
    "        return np.array(phi)\n",
    "    \n",
    "    def add_state(self, s):\n",
    "        self.S.append(s)\n",
    "        self.current_state = s\n",
    "\n",
    "    def update_state(self):\n",
    "        idx = np.random.choice([0, 1], p = [self.p0, 1 - self.p0])\n",
    "        sprime = self.current_state * self.c[idx]\n",
    "        return sprime\n",
    "    \n",
    "    def next_state(self, a):\n",
    "        if a == 1:\n",
    "            next_state = np.array([99999])\n",
    "            self.add_state(next_state)\n",
    "            return next_state\n",
    "        else:\n",
    "            next_state = self.update_state()\n",
    "            # next_state = max(self.lower, next_state)\n",
    "            # next_state = min(self.upper, next_state)\n",
    "            self.add_state(next_state)\n",
    "            return next_state\n",
    "    \n",
    "    def generate_reward(self, a):\n",
    "        if a == 0:\n",
    "            reward = 0\n",
    "            self.R.append(reward)\n",
    "            return reward\n",
    "        else:\n",
    "            reward = max(0, self.K - self.current_state)\n",
    "            self.R.append(reward)\n",
    "            return reward\n",
    "    \n",
    "    def step(self, a):\n",
    "        self.A.append(a)\n",
    "        phi = self.phi(self.current_state)\n",
    "        self.feature.append(phi)\n",
    "        self.generate_reward(a)\n",
    "        self.next_state(a)\n",
    "        self.h += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSVI_UCB():\n",
    "    def __init__(self, H, beta, lam, d):\n",
    "        self.lam = lam\n",
    "        self.beta = beta\n",
    "        self.H = H\n",
    "        self.K = 100\n",
    "        self.d = d\n",
    "        self.w = [np.zeros(self.d) for _ in range(self.H)]\n",
    "        self.Lambda = [self.lam * np.diag(np.ones(self.d)) for _ in range(self.H)]\n",
    "\n",
    "    def get_action(self, phi, h, current_state):\n",
    "        Q_exercising = max(0, self.K - current_state)\n",
    "        Q_not_exercising = self.get_Q_func(phi, h)\n",
    "        Q = [Q_not_exercising, Q_exercising]\n",
    "        return np.argmax(Q)\n",
    "\n",
    "    def get_Q_func(self, phi, h):\n",
    "        Lambda_h_inverse = np.linalg.inv(self.Lambda[h])\n",
    "        Q_h = np.min([(self.w[h] @ phi + self.beta * np.sqrt(phi @ Lambda_h_inverse @ phi)), self.H])\n",
    "        return Q_h \n",
    "    \n",
    "    def update_Q(self, history):\n",
    "        # Backward induction\n",
    "        self.w = [None for _ in range(self.H)] # initialize weights w\n",
    "        for h in range(self.H-1, -1, -1):\n",
    "            # update Lambda_h\n",
    "            phi_h = history['phi'][-1][h]\n",
    "            self.Lambda[h] += np.outer(phi_h, phi_h)\n",
    "            # update w_h \n",
    "            Lambda_h_inverse = np.linalg.inv(self.Lambda[h])\n",
    "            w_h = np.zeros(self.d)\n",
    "            if h == self.H - 1:\n",
    "                self.w[h] = w_h\n",
    "            else:\n",
    "                for tau in range(history['k']):\n",
    "                    phi_tau_h = history['phi'][tau][h]\n",
    "                    phi_tau_h_plus_one = history['phi'][tau][h+1]\n",
    "                    s_tau_h_plus_one = history['state'][tau][h+1]\n",
    "                    r_tau_h = history['r'][tau][h]\n",
    "                    Q_tau_h_plus_one = [self.get_Q_func(phi_tau_h_plus_one, h + 1), max(0, self.K - s_tau_h_plus_one)]\n",
    "                    V_tau_h_plus_one = np.max(Q_tau_h_plus_one)\n",
    "                    w_h += Lambda_h_inverse @ (phi_tau_h * (r_tau_h + V_tau_h_plus_one))\n",
    "            self.w[h] = w_h\n",
    "\n",
    "class DR_LSVI_UCB():\n",
    "    def __init__(self, d, beta, H, lam, rho):\n",
    "        self.lam = lam\n",
    "        self.d = d\n",
    "        self.H = H\n",
    "        self.K = 100\n",
    "        self.beta = beta\n",
    "        self.w = [np.zeros(self.d) for _ in range(self.H)]\n",
    "        self.Lambda = [self.lam * np.diag(np.ones(self.d)) for _ in range(self.H)]\n",
    "        self.rho = rho\n",
    "\n",
    "    def get_action(self, phi, h, current_state):\n",
    "        Q_exercising = max(0, self.K - current_state)\n",
    "        Q_not_exercising = self.get_Q_func(phi, h)\n",
    "        Q = [Q_not_exercising, Q_exercising]\n",
    "        return np.argmax(Q)\n",
    "    \n",
    "    def get_Q_func(self, phi, h):\n",
    "        Lambda_h_inverse = np.linalg.inv(self.Lambda[h])\n",
    "        bonus =  self.beta * np.sqrt(phi @ np.diag(np.diagonal(Lambda_h_inverse)) @ phi)\n",
    "        Q_h = np.min([(self.w[h] @ phi + bonus), self.H - h])\n",
    "        return Q_h \n",
    "    \n",
    "    def get_nu_h(self, history, h, rho):\n",
    "        Lambda_h_inverse = np.linalg.inv(self.Lambda[h])\n",
    "        nu_h = np.zeros(self.d)  \n",
    "        Phi_h = np.zeros((0,self.d)) \n",
    "        V_h_plus_one = np.zeros(0)\n",
    "        for tau in range(history['k']): \n",
    "            phi_tau_h = history['phi'][tau][h]\n",
    "            Phi_h = np.vstack((Phi_h, phi_tau_h))\n",
    "            phi_tau_h_plus_one = history['phi'][tau][h+1]\n",
    "            s_tau_h_plus_one = history['state'][tau][h+1]\n",
    "            Q_tau_h_plus_one = [self.get_Q_func(phi_tau_h_plus_one, h + 1), max(0, (self.K - s_tau_h_plus_one))]\n",
    "            V_tau_h_plus_one = np.max(Q_tau_h_plus_one)\n",
    "            V_h_plus_one = np.hstack((V_h_plus_one, V_tau_h_plus_one))\n",
    "        for i in range(self.d):\n",
    "            #print(i)\n",
    "            def z_alpha_i(alpha):\n",
    "                # compact formular for z\n",
    "                z = Lambda_h_inverse @ Phi_h.T @ np.minimum(V_h_plus_one, alpha)\n",
    "                return - z[i] + rho * alpha\n",
    "            result =  minimize(z_alpha_i, self.H/2, method='Nelder-Mead', bounds=[(0,self.H)])\n",
    "            nu_h[i] = - result.fun\n",
    "            #print(result.x)\n",
    "        return nu_h\n",
    "    \n",
    "    def update_Q(self, history): \n",
    "        # Backward induction\n",
    "        self.w = [None for _ in range(self.H)] # initialize weights w\n",
    "        for h in range(self.H-1, -1, -1):\n",
    "            # update Lambda_h\n",
    "            phi_h = history['phi'][-1][h]\n",
    "            self.Lambda[h] += np.outer(phi_h, phi_h)\n",
    "            # update w_h \n",
    "            if h == self.H - 1:\n",
    "                w_h = np.zeros(self.d)\n",
    "            else:\n",
    "                nu_h = self.get_nu_h(history, h, rho=self.rho)\n",
    "                w_h = nu_h\n",
    "            self.w[h] = w_h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_once(epoch, H, beta, lam, d, p0, seed):\n",
    "    history = {'k': 0, 'phi':[], 'r':[], 'state':[]}\n",
    "    env = American_put_option(p0=p0, d=d, seed=seed)\n",
    "    agent = LSVI_UCB(H=H, beta=beta, lam=lam, d=d)\n",
    "    #reward = 0\n",
    "    #Reward = []\n",
    "    for t in range(epoch):\n",
    "        env.reset()\n",
    "        for h in range(H):\n",
    "            current_state = env.current_state\n",
    "            phi = env.phi(current_state) \n",
    "            action = agent.get_action(phi, h, current_state)\n",
    "            env.step(action)\n",
    "        # log the trajectory\n",
    "        history['phi'].append(env.feature)\n",
    "        history['r'].append(env.R)\n",
    "        history['state'].append(env.S)\n",
    "        history['k'] += 1\n",
    "        # update the agent\n",
    "        agent.update_Q(history)\n",
    "        #reward += np.sum(env.R)\n",
    "        #Reward.append(copy.copy(reward))\n",
    "    return agent\n",
    "\n",
    "def train_once_DR(epoch, H, beta, lam, d, p0, rho, seed):\n",
    "    history = {'k': 0, 'phi':[], 'r':[], 'state':[]}\n",
    "    env = American_put_option(p0=p0, d=d, seed=seed)\n",
    "    DR_agent = DR_LSVI_UCB(H=H, beta=beta, lam=lam, d=d, rho=rho)\n",
    "    # reward = 0\n",
    "    # Reward = []\n",
    "    for t in range(epoch):\n",
    "        env.reset()\n",
    "        for h in range(H):\n",
    "            current_state = env.current_state\n",
    "            phi = env.phi(current_state) \n",
    "            action = DR_agent.get_action(phi, h, current_state)\n",
    "            env.step(action)\n",
    "        # log the trajectory\n",
    "        history['phi'].append(env.feature)\n",
    "        history['r'].append(env.R)\n",
    "        history['state'].append(env.S)\n",
    "        history['k'] += 1\n",
    "        # update the agent\n",
    "        DR_agent.update_Q(history)\n",
    "        # reward += np.sum(env.R)\n",
    "        # Reward.append(copy.copy(reward))\n",
    "    return DR_agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T1 = 100\n",
    "H = 10\n",
    "beta = 1\n",
    "lam = 0.1\n",
    "rho = 0.5\n",
    "p0 = 0.5\n",
    "d = 20\n",
    "replication = 10\n",
    "agent_dic = {}\n",
    "DR_agent_dic = {}\n",
    "\n",
    "# Train on the source domain\n",
    "for rep in range(replication):\n",
    "    agent = train_once(epoch=T1, H=H, beta=beta, lam=lam, d=d, p0=p0, seed=rep)\n",
    "    DR_agent = train_once_DR(epoch=T1, H=H, beta=beta, lam=lam, d=d, p0=p0, rho=rho, seed=rep)\n",
    "    agent_dic[str(rep)] = agent\n",
    "    DR_agent_dic[str(rep)] = DR_agent\n",
    "\n",
    "\n",
    "# Test on the target domain\n",
    "PROB = [x / 40 for x in range(6,35)]\n",
    "T2 = 100\n",
    "R_LSVI_UCB = []\n",
    "R_DR_LSVI_UCB = []\n",
    "for p in PROB:\n",
    "    REWARD = 0\n",
    "    REWARD_DR = 0\n",
    "    for rep in range(replication):\n",
    "        reward = 0\n",
    "        reward_DR = 0\n",
    "        env_test = American_put_option(p, d, seed=rep)\n",
    "        env_test_DR = American_put_option(p, d, seed=rep)\n",
    "        # agent = agent_dic[str(rep)]\n",
    "        # DR_agent = DR_agent_dic[str(rep)]\n",
    "        agent = agent_dic['0']\n",
    "        DR_agent = DR_agent_dic['0']\n",
    "        for t in range(T2):\n",
    "            env_test.reset()\n",
    "            env_test_DR.reset()\n",
    "            for h in range(H):\n",
    "                current_state = env_test.current_state\n",
    "                phi = env_test.phi(current_state)\n",
    "                action = agent.get_action(phi, h, current_state)\n",
    "\n",
    "                current_state_DR = env_test_DR.current_state\n",
    "                phi_DR = env_test_DR.phi(current_state_DR)\n",
    "                action_DR = DR_agent.get_action(phi_DR, h, current_state_DR)\n",
    "\n",
    "                env_test.step(action)\n",
    "                env_test_DR.step(action_DR)\n",
    "            reward += np.sum(env_test.R) / T2\n",
    "            reward_DR += np.sum(env_test_DR.R) / T2\n",
    "        REWARD += reward / replication\n",
    "        REWARD_DR += reward_DR / replication\n",
    "    #print(np.log(REWARD))\n",
    "    #print(REWARD_DR)\n",
    "    R_LSVI_UCB.append(REWARD)\n",
    "    R_DR_LSVI_UCB.append(REWARD_DR)\n",
    "plt.plot(PROB, R_LSVI_UCB, label = 'LSVI-UCB')\n",
    "plt.plot(PROB, R_DR_LSVI_UCB, label = 'DR-LSVI-UCB')\n",
    "plt.legend(fontsize=16)\n",
    "plt.xlabel('price-up probability', size=16)\n",
    "plt.ylabel('Average Reward', size=16)\n",
    "plt.savefig(f'APO_{p0}_{d}_{rho}.pdf', dpi=1000, bbox_inches='tight', pad_inches=0.0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}