Transformer的实现.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "#关于embedding，以序列建模为例\n",
    "#构建序列，序列的字符以在词表中索引的形式表示\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#torch.manual_seed(1) # 为CPU设置随机种子\n",
    "#torch.cuda.manual_seed(1) # 为当前GPU设置随机种子\n",
    "#torch.cuda.manual_seed_all(1) # 为所有GPU设置随机种子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class Embeddings(nn.Module):\n",
    "    \"\"\"\n",
    "    d_model:512/256,Embeddings_size\n",
    "    vocab:vocab_size\n",
    "    \"\"\"\n",
    "    def __init__(self, d_model,  vocab):\n",
    "        super(Embeddings, self).__init__()\n",
    "        self.lut = nn.Embedding(vocab, d_model)\n",
    "        self.d_model = d_model\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.lut(x) * math.sqrt(self.d_model)#做了*开方的效果\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "class PositionalEncoding(nn.Module):\n",
    "    \"Implement the PE function.\"\n",
    "    def __init__(self, d_model, dropout, max_len=5000):#d_model的默认值是512，每个词是512维\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)#dropout的几率\n",
    "\n",
    "        # Compute the positional encodings once in log space.\n",
    "        pe = torch.zeros(max_len, d_model)#max_len*d_model维度的0矩阵\n",
    "        position = torch.arange(0, max_len).unsqueeze(1)#0,max_len-1的值按照维度1展开，（0是行，1是列）\n",
    "        #因为刚才的0矩阵的维度要匹配上位置\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2) *#0到d_model中，每隔2个的数值，如0,2,4,6,8...\n",
    "                             -(math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)#维度跳跃，从0开始间隔为2\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)#维度跳跃，从1开始间隔为2\n",
    "        pe = pe.unsqueeze(0)#解压到0维度\n",
    "        self.register_buffer('pe', pe)#加载到内存中，方便读取\n",
    "\n",
    "    def forward(self, x):#前向传播\n",
    "        x = x + Variable(self.pe[:, :x.size(1)],\n",
    "                         requires_grad=False)\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "#注意力机制\n",
    "def attention(query, key, value, mask=None, dropout=None):\n",
    "    #q,k,v，\n",
    "    \"Compute 'Scaled Dot Product Attention'\"\n",
    "    d_k = query.size(-1)\n",
    "    #d_k=d_q,确定q,k他们的维度\n",
    "    # batch size, head size, query len , head dim\n",
    "    # batch size, head size, key len , head dim\n",
    "    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n",
    "    #q*k的转置再/一个根号dk\n",
    "    if mask is not None:#掩码，这里是softmax指数，把0替换成-1e9避免数据出错\n",
    "        scores = scores.masked_fill(mask == 0, -10e9)\n",
    "    p_attn = F.softmax(scores, dim = -1)\n",
    "    #softmax()\n",
    "    if dropout is not None:\n",
    "        p_attn = dropout(p_attn)\n",
    "    #softmax()后再进行矩阵乘法\n",
    "    return torch.matmul(p_attn, value), p_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#为了复制论文中的N个块\n",
    "import copy\n",
    "def clones(module, N):\n",
    "    #加起来，X个块\n",
    "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#多头注意力\n",
    "class MultiHeadedAttention(nn.Module):\n",
    "    def __init__(self, h, d_model, dropout=0.1):\n",
    "        \"\"\"\n",
    "        :param h: 几个头，论文是8个\n",
    "        :param d_model: #输入的维度\n",
    "        :param dropout:\n",
    "        \"\"\"\n",
    "        \"Take in model size and number of heads.\"\n",
    "        super(MultiHeadedAttention, self).__init__()\n",
    "        assert d_model % h == 0\n",
    "        # We assume d_v always equals d_k\n",
    "        self.d_k = d_model // h\n",
    "        self.h = h#几个头\n",
    "        self.linears = clones(nn.Linear(d_model, d_model), 4)\n",
    "        #深复制4份,目的是3个qkv和1个x残差\n",
    "        self.attn = None\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "    def forward(self, X1, X2, X3, mask=None):\n",
    "        \"\"\"\n",
    "        key len = value len\n",
    "        X1 = [batch size, query len, d_model]\n",
    "        X2 = [batch size, key len, d_model]\n",
    "        X3 = [batch size, value len, d_model]\n",
    "        mask = [batch size,1, key len] or [batch size, query len, key len]\n",
    "        \"\"\"\n",
    "        if mask is not None:\n",
    "            # Same mask applied to all h heads.\n",
    "            mask = mask.unsqueeze(1)\n",
    "            #三角矩阵的mask\n",
    "        nbatches = X1.size(0)\n",
    "\n",
    "        # 1) Do all the linear projections in batch from d_model => h x d_k\n",
    "        query, key, value = \\\n",
    "            [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)#view与reshape类似\n",
    "             for l, x in zip(self.linears, (X1, X2, X3))]#这里是计算x与W_Q,W_K,W_V三个的产出QKV\n",
    "\n",
    "        # 2) Apply attention on all the projected vectors in batch.\n",
    "        # x = batch size, h, query len, h_dim\n",
    "        x, self.attn = attention(query, key, value, mask=mask,\n",
    "                                 dropout=self.dropout)\n",
    "\n",
    "        # 3) \"Concat\" using a view and apply a final linear.\n",
    "        x = x.transpose(1, 2).contiguous() \\\n",
    "             .view(nbatches, -1, self.h * self.d_k)\n",
    "        return self.linears[-1](x)#Linear\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from torch.nn import LayerNorm\n",
    "\n",
    "#add&norm\n",
    "class SublayerConnection(nn.Module):\n",
    "    \"\"\"\n",
    "    A residual connection followed by a layer norm.\n",
    "    Note for code simplicity the norm is first as opposed to last.\n",
    "\n",
    "    add&norm\n",
    "    \"\"\"\n",
    "    def __init__(self, size, dropout):\n",
    "        super(SublayerConnection, self).__init__()\n",
    "        self.norm = LayerNorm(size)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x, sublayer):\n",
    "        \"Apply residual connection to any sublayer with the same size.\"\n",
    "        return x + self.dropout(sublayer(self.norm(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class PositionwiseFeedForward(nn.Module):\n",
    "    \"Implements FFN equation.\"\n",
    "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
    "        super(PositionwiseFeedForward, self).__init__()\n",
    "        self.w_1 = nn.Linear(d_model, d_ff)\n",
    "        self.w_2 = nn.Linear(d_ff, d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.w_2(self.dropout(F.relu(self.w_1(x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    \"Encoder is made up of self-attn and feed forward (defined below)\"\n",
    "    def __init__(self, size, self_attn, feed_forward, dropout):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.self_attn = self_attn\n",
    "        self.feed_forward = feed_forward\n",
    "        self.sublayer = clones(SublayerConnection(size, dropout), 2)\n",
    "        self.size = size\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        \"Follow Figure 1 (left) for connections.\"\n",
    "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n",
    "        return self.sublayer[1](x, self.feed_forward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    \"Core encoder is a stack of N layers\"\n",
    "    def __init__(self, layer, N):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.layers = clones(layer, N)\n",
    "        self.norm = LayerNorm(layer.size)\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        \"Pass the input (and mask) through each layer in turn.\"\n",
    "        for layer in self.layers:\n",
    "            x = layer(x, mask)\n",
    "        return self.norm(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def subsequent_mask(size):\n",
    "    \"Mask out subsequent positions.\"\n",
    "    #下三角矩阵\n",
    "    attn_shape = (1, size, size)\n",
    "    subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')\n",
    "    #上三角矩阵，且往上移动一下\n",
    "    return torch.from_numpy(subsequent_mask) == 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    \"Decoder is made of self-attn, src-attn, and feed forward (defined below)\"\n",
    "    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.size = size\n",
    "        self.self_attn = self_attn\n",
    "        self.src_attn = src_attn\n",
    "        self.feed_forward = feed_forward\n",
    "        self.sublayer = clones(SublayerConnection(size, dropout), 3)\n",
    "\n",
    "    def forward(self, x, memory, src_mask, tgt_mask):\n",
    "        \"\"\"\n",
    "        x: decoder input\n",
    "        memory: encoder output\n",
    "        \"\"\"\n",
    "        m = memory\n",
    "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n",
    "        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n",
    "        return self.sublayer[2](x, self.feed_forward)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    \"Generic N layer decoder with masking.\"\n",
    "    def __init__(self, layer, N):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.layers = clones(layer, N)\n",
    "        self.norm = LayerNorm(layer.size)\n",
    "\n",
    "    def forward(self, x, memory, src_mask, tgt_mask):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x, memory, src_mask, tgt_mask)\n",
    "        return self.norm(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Generator(nn.Module):\n",
    "    \"Define standard linear + softmax generation step.\"\n",
    "    #最后一步\n",
    "    def __init__(self, d_model, vocab):\n",
    "        super(Generator, self).__init__()\n",
    "        self.proj = nn.Linear(d_model, vocab)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.log_softmax(self.proj(x), dim=-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "    \"\"\"\n",
    "    A standard Encoder-Decoder architecture. Base for this and many\n",
    "    other models.\n",
    "    \"\"\"\n",
    "    def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.src_embed = src_embed\n",
    "        self.tgt_embed = tgt_embed\n",
    "        self.generator = generator\n",
    "\n",
    "    def forward(self, src, tgt, src_mask, tgt_mask):\n",
    "        \"Take in and process masked src and target sequences.\"\n",
    "        return self.decode(self.encode(src, src_mask), src_mask,\n",
    "                            tgt, tgt_mask)\n",
    "\n",
    "    def encode(self, src, src_mask):\n",
    "        return self.encoder(self.src_embed(src), src_mask)\n",
    "\n",
    "    def decode(self, memory, src_mask, tgt, tgt_mask):\n",
    "        return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上面就是Transformer的定义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def make_model(src_vocab, tgt_vocab, N=6,\n",
    "               d_model=512, d_ff=2048, h=8, dropout=0.1):\n",
    "    #d_ff前馈链接大小\n",
    "    \"Helper: Construct a model from hyperparameters.\"\n",
    "    c = copy.deepcopy\n",
    "    attn = MultiHeadedAttention(h, d_model)#分头行动\n",
    "    ff = PositionwiseFeedForward(d_model, d_ff, dropout)#FFN\n",
    "    position = PositionalEncoding(d_model, dropout)#位置编码\n",
    "    model = Transformer(\n",
    "        Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),\n",
    "        Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),\n",
    "        nn.Sequential(Embeddings(d_model, src_vocab), c(position)),\n",
    "        nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),\n",
    "        Generator(d_model, tgt_vocab))\n",
    "\n",
    "    # This was important from their code.\n",
    "    # Initialize parameters with Glorot / fan_avg.\n",
    "    # xavier初始化model\n",
    "    for p in model.parameters():\n",
    "        if p.dim() > 1:\n",
    "            nn.init.xavier_uniform(p)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "额外的工作,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Batch:\n",
    "    #批次\n",
    "    \"Object for holding a batch of data with mask during training.\"\n",
    "    def __init__(self, src, trg=None, pad=0):\n",
    "        \"\"\"\n",
    "        src = batch size, seq len\n",
    "\n",
    "        \"\"\"\n",
    "        self.src = src\n",
    "        # src_mask = batch size, 1, seq len\n",
    "        self.src_mask = (src != pad).unsqueeze(-2)\n",
    "        \n",
    "        if trg is not None:\n",
    "            self.trg = trg[:, :-1]#因为需要最开始的字符<EOS>开始decoder预测\n",
    "            self.trg_y = trg[:, 1:]#loss的预测中需要排除掉开始字符的概率，因为都有\n",
    "            self.trg_mask = \\\n",
    "                self.make_std_mask(self.trg, pad)\n",
    "            self.ntokens = (self.trg_y != pad).data.sum()\n",
    "\n",
    "    @staticmethod\n",
    "    def make_std_mask(tgt, pad):\n",
    "        \"Create a mask to hide padding and future words.\"\n",
    "        #掩码未来\n",
    "        tgt_mask = (tgt != pad).unsqueeze(-2)#-2的位置加一个维度1\n",
    "        #原来tgt.shape=(batch,seq_len,dim)-->(batch,seq_len,1,dim)\n",
    "        #随机的tensor可能含有tgt==pad，而词向量中不存在\n",
    "        tgt_mask = tgt_mask & Variable(subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data))\n",
    "        #按位运算\n",
    "        return tgt_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***********\n",
      "***********\n",
      "***********\n",
      "trg_mask\n",
      "***********\n",
      "***********\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "\n",
    "batch_s=torch.IntTensor(7,6,10)\n",
    "batch_t=torch.IntTensor(7,5,10)\n",
    "#batch_s=torch.from_numpy(batch_s)\n",
    "#batch_t=torch.from_numpy(batch_t)\n",
    "#print(batch_s)\n",
    "print(\"***********\")\n",
    "B=Batch(batch_s,batch_t)\n",
    "#print(B)\n",
    "print(\"***********\")\n",
    "#print(B.src)\n",
    "print(\"***********\")\n",
    "#print(B.src_mask)\n",
    "print(\"trg_mask\")\n",
    "#print(B.trg_mask)\n",
    "print(\"***********\")\n",
    "#print(batch_s.data)\n",
    "#print(batch_t.data)\n",
    "print(\"***********\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([7, 4, 1, 10])\n",
      "<class 'torch.Tensor'>\n",
      "torch.Size([7, 4, 10, 10])\n"
     ]
    }
   ],
   "source": [
    "a=torch.BoolTensor(size=(7,4,1,10))\n",
    "b=torch.BoolTensor(size=(1,10,10))\n",
    "print(a.shape)\n",
    "print(type(b))\n",
    "c=a & b\n",
    "print(c.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#warmup\n",
    "class NoamOpt:\n",
    "    \"Optim wrapper that implements rate.\"\n",
    "    def __init__(self, model_size, factor, warmup, optimizer):\n",
    "        self.optimizer = optimizer\n",
    "        self._step = 0\n",
    "        self.warmup = warmup\n",
    "        self.factor = factor\n",
    "        self.model_size = model_size\n",
    "        self._rate = 0\n",
    "\n",
    "    def zero_grad(self):\n",
    "        self.optimizer.zero_grad()\n",
    "    def step(self):\n",
    "        \"Update parameters and rate\"\n",
    "        self._step += 1\n",
    "        rate = self.rate()\n",
    "        for p in self.optimizer.param_groups:\n",
    "            p['lr'] = rate\n",
    "        self._rate = rate\n",
    "        self.optimizer.step()\n",
    "\n",
    "    def rate(self, step = None):\n",
    "        \"Implement `lrate` above\"\n",
    "        if step is None:\n",
    "            step = self._step\n",
    "        return self.factor * \\\n",
    "            (self.model_size ** (-0.5) *\n",
    "            min(step ** (-0.5), step * self.warmup ** (-1.5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_std_opt(model):\n",
    "    return NoamOpt(model.src_embed[0].d_model, 2, 4000,\n",
    "            torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD4CAYAAAD2FnFTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABN40lEQVR4nO3dd3hUVfrA8e9Jh/RCeiUJJRBKCL2DIAiKgCCK2FDEsrqWXV13dV1XV1Zd28rqT10F1BVRQVEQEJiAhF5CgEAgIT0hlXRS5/z+mCEGSBmSSSblfJ4nT2buvefed4Yw75x7z32PkFKiKIqiKFczM3UAiqIoSsekEoSiKIrSIJUgFEVRlAapBKEoiqI0SCUIRVEUpUEWpg7AGNzc3GRgYKCpw1AURelUjhw5kiel7NXY+i6RIAIDAzl8+LCpw1AURelUhBApTa1Xp5gURVGUBqkEoSiKojRIJQhFURSlQQZdgxBCzADeBcyBT6SUK65aL/TrbwLKgXullEebaiuEWAC8BPQHRkgpD9fb35+ApUAt8LiUcmsrXqOiKJ1QdXU16enpVFRUmDqUTs/GxgZfX18sLS2vq12zCUIIYQ6sBKYB6cAhIcRGKWVcvc1mAqH6n5HAB8DIZtqeBOYB/3fV8cKARcAAwBvYLoToI6Wsva5XpihKp5aeno69vT2BgYHovoMqLSGlJD8/n/T0dIKCgq6rrSGnmEYACVLK81LKKmAtMOeqbeYAa6TOfsBJCOHVVFsp5WkpZXwDx5sDrJVSVkopk4AE/X4URelGKioqcHV1VcmhlYQQuLq6tqgnZkiC8AHS6j1P1y8zZBtD2rbkeAghlgkhDgshDufm5jazS0VROiOVHIyjpe+jIQmioT1fXSO8sW0MaduS4yGl/EhKGSmljOzVq9H7PJSrlFaV8s3Zb6jR1pg6FEVROjhDEkQ64FfvuS+QaeA2hrRtyfGUFlodt5qX973Mf0/819ShKEqnEBgYSHh4OEOGDCEyMhKAb775hgEDBmBmZnbFTbq//PILw4YNIzw8nGHDhrFz584m9/3mm28ihCAvL69u2WuvvUZISAh9+/Zl69bfxuccOXKE8PBwQkJCePzxx7k8l09lZSW33347ISEhjBw5kuTkZKO9dkMSxCEgVAgRJISwQncBeeNV22wE7hY6o4AiKWWWgW2vthFYJISwFkIEobvwffA6XpPShANZBwD4+MTHpBanmjgaRekcNBoNMTExdclg4MCBrF+/ngkTJlyxnZubGz/++CMnTpxg9erVLFmypNF9pqWl8csvv+Dv71+3LC4ujrVr13Lq1Cm2bNnCI488Qm2tbnzOww8/zEcffcS5c+c4d+4cW7ZsAeC///0vzs7OJCQk8OSTT/Lss88a7XU3myCklDXAY8BW4DSwTkp5SgixXAixXL/ZZuA8ugvKHwOPNNUWQAgxVwiRDowGNgkhturbnALWAXHAFuBRNYLJOPIv5ROTE8P80PlYmlnyt31/q/sWoiiK4fr370/fvn2vWT506FC8vb0BGDBgABUVFVRWVja4jyeffJLXX3/9iusDP/zwA4sWLcLa2pqgoCBCQkI4ePAgWVlZFBcXM3r0aIQQ3H333Xz//fd1be655x4AbrvtNnbs2GG0/9cG3QchpdyMLgnUX/ZhvccSeNTQtvrlG4ANjbR5FXjVkNgUw+1O341Ecnvf2xngNoCX973MhoQNzAudZ+rQFKVJf/vxFHGZxUbdZ5i3A3+9eUCz2wkhmD59OkIIHnroIZYtW2bQ/r/77juGDh2KtbU1AA888ADLly8nMjKSjRs34uPjw+DBg69ok5GRwahRo+qe+/r6kpGRgaWlJb6+vtcsv9zGz093Vt7CwgJHR0fy8/Nxc3MzKM6mdIlifYphdqbtxMvWi34u/ejr0pdN5zfx5uE3Ge8znl491YV+RWlIdHQ03t7e5OTkMG3aNPr163fNqaWrnTp1imeffZZt27bVLfvkk08AKC8v59VXX71i3WUNffMXQjS6vKk2xqASRDdxqeYS+zP3Mzd0LkIIBIKXRr/E/I3zeXn/y7w3+T01pFDpsAz5pt9WLp8ycnd3Z+7cuRw8eLDJBJGens7cuXNZs2YNwcHB16xPTEwkKSmprveQnp5OREQEBw8exNfXl7S0tCv25e3tja+vL+np6dcsB+ra+Pr6UlNTQ1FRES4uLkZ57aoWUzexL3MfFbUVTPabXLcs0DGQxyMeJyotiu8TvjdZbIrSUZWVlVFSUlL3eNu2bQwcOLDR7QsLC5k1axavvfYaY8eObXCb8PBwcnJySE5OJjk5GV9fX44ePYqnpye33HILa9eupbKykqSkJM6dO8eIESPw8vLC3t6e/fv3I6VkzZo1zJmju1/5lltuYfXq1QB8++23TJkyxWhf9lSC6Cai0qKwt7Qn0jPyiuVLwpYwwnMEKw6uIK0kreHGitJNZWdnM27cOAYPHsyIESOYNWsWM2bMYMOGDfj6+rJv3z5mzZrFjTfeCMD7779PQkICf//73xkyZAhDhgwhJycH0F2DaG7emgEDBrBw4ULCwsKYMWMGK1euxNzcHIAPPviABx54gJCQEIKDg5k5cyYAS5cuJT8/n5CQEN566y1WrFjR1CGui+gKo1giIyOlmjCocbXaWqZ8M4WRXiN5fcLr16zPKs1i/sb5hDiH8NmNn2FuZm6CKBXlSqdPn6Z///6mDqPLaOj9FEIckVJGNtJE9SC6g9i8WAoqCq44vVSfl50Xz496nmM5x/js1GftHJ2iKB2VShDdgCZVg4WZBeN8xjW6zaygWdwYeCMrj60kNje2HaNTFKWjUgmiG9CkaRjuMRx7K/tGtxFC8MKoF/Cw9eAPu/5AUWVRO0aoKEpHpBJEF3e+6DzJxclM9m/49FJ9jtaOvDHhDXIu5fCX6L+ou6wVpZtTCaKLi0qLAmj0+sPVwnuF8/Swp4lKi+LzuM/bLC5FUTo+lSC6OE2qhv4u/fG09TS4zeL+i5nqP5W3j7zN8dzjbRidoigdmUoQXVjepTyO5x43uPdwmRCCl8e+jIetB09pniK3XE3IpHRfbVHuOyYmhlGjRtXt8+DB3wpWd6Ry30gpO/3PsGHDpHKt785+JweuGihP559uUfsz+Wfk8C+Gy8WbFsvKmkojR6coTYuLizN1CFJKKQMCAmRubu4Vy+Li4uSZM2fkxIkT5aFDh+qWHz16VGZkZEgppTxx4oT09vZucJ/Tpk2TmzdvllJKuWnTJjlx4kQppZSnTp2SgwYNkhUVFfL8+fOyd+/esqamRkop5fDhw+XevXulVquVM2bMqGu/cuVK+dBDD0kppfzqq6/kwoULGzxmQ+8ncFg28dmqehBdmCZVg7etN32dry1LbIi+Ln15ddyrHM89zqsHXlUXrRVFr7XlvoUQFBfrqtMWFRXVtemU5b6Vzqe8upx9WfuYHzq/VXVZpgVMY9mgZXwU+xF9nftyZ/87jRilohjo5+fgwgnj7tMzHGY2X5aiLcp9v/POO9x4440888wzaLVa9u7dC6hy30o72Z+1n8raSoOGtzbn0SGPcrbgLK8fep0gxyBGe482QoSK0jkYu9w36Ooqvf3228yfP59169axdOlStm/frsp9K+1Dk6bB3tKeYR7DWr0vM2HGa+NfY8nPS3gq6ilWz1xNH+c+RohSUQxkwDf9tmLsct8Aq1ev5t133wVgwYIFPPDAAwCq3LfS9mq1texO380433FYmlkaZZ92VnZ8cMMH9LToySPbHyG7LNso+1WUjqwtyn2DLuns2rULgJ07dxIaGgrQ4cp9m3wEkjF+1CimKx25cEQOXDVQ/nz+Z6Pv+0z+GTnyy5Fy3g/zZEllidH3ryiXdYRRTImJiXLQoEFy0KBBMiwsTL7yyitSSinXr18vfXx8pJWVlXR3d5fTp0+XUkr597//Xfbs2VMOHjy47ic7O1tKKeXSpUvrRjz9+uuvMiIiQg4aNEiOGDFCHj58uO6Yr7zyiuzdu7fs06dP3UglKaU8dOiQHDBggOzdu7d89NFHpVarlVJKeenSJXnbbbfJ4OBgOXz4cJmYmNjga2nJKCZV7rsL+tfhf/HF6S/49fZfsbOyM/r+92bs5dEdjzLcczgrp67E0tw4vRRFqU+V+zYuVe5bQUqJJk3DCM8RbZIcAMb4jOHF0S+yL2sfz+95nlptbZscR1EU01IXqbuYpOIkUopTuKv/XW16nLmhcymqLOJfR/6FraUtfx39VzWntaJ0MSpBdDGaVA0Ak/wmtfmx7h14LyXVJXwU+xE9LXvyh8g/qCShKF2IShBdjCZNQ5hr2HUV52uNx4Y8Rll1GZ/HfY69pT0PD3m4XY6rKErbUwmiC8m7lEdsbmy7fkgLIfjj8D9SVl3Gf47/B0tzSx4If6Ddjq8oSttRCaIL2ZW2C4lkit+Udj2umTDjpdEvUa2t5t2j71KtrebhwaonoSidnRrF1IVo0nTF+Uxxl7O5mTmvjn2VW4Jv4T8x/+H9Y++r4n5Kp5eWlsbkyZPp378/AwYMqLv7+aWXXsLHx4chQ4YwZMgQNm/eXNcmNjaW0aNHM2DAAMLDw6moqGh0/2+++SZCCPLy8uqWqXLf6kY5oyurKpPDPh8mXzvwmknjqNXWyhejX5QDVw2U7xx5p+5mHkW5Xh3hRrnMzEx55MgRKaWUxcXFMjQ0VJ46dUr+9a9/lW+88cY121dXV8vw8HAZExMjpZQyLy+vrlz31VJTU+X06dOlv79/XTlxVe5baRP7svbpivNd5+RAxmYmzPjr6L+yoM8CPjnxCa8feh2t1Jo0JkVpKS8vLyIiIgCwt7enf//+dVVUG7Jt2zYGDRrE4MGDAXB1dcXc3LzBbZ988klef/31K0b+qXLfSpvQpGqwt7InwiPC1KFgJsx4YdQLWJtb88XpL7hYeZG/j/m7uuNaabF/HvwnZwrOGHWf/Vz68eyIZw3ePjk5mWPHjjFy5Eiio6N5//33WbNmDZGRkfzrX//C2dmZs2fPIoTgxhtvJDc3l0WLFvHHP/4RuLLc98aNG/Hx8alLJJd1tHLfqgfRBVwuzjfeZ7zRivO11uXRTY8PfZxN5zfxu52/o7y63NRhKUqLlJaWMn/+fN555x0cHBx4+OGHSUxMJCYmBi8vL55++mkAampq2LNnD19++SV79uxhw4YN7NixA9CV+46MjKS8vJxXX32Vl19++ZrjNPTNv8OX+xZCzADeBcyBT6SUK65aL/TrbwLKgXullEebaiuEcAG+BgKBZGChlPKiEMIS+ASI0Me3Rkr5WuteZtcWkxvDxcqLRpn7wZiEEDw46EFcbFx4ef/LPLjtQVZOXYmTjZOpQ1M6mev5pm9s1dXVzJ8/n8WLFzNv3jwAPDw86tY/+OCDzJ49G9B9s584cWLdt/ebbrqJo0ePMnXq1LrtExMTSUpKqus9pKenExERwcGDBztfuW8hhDmwEpgJhAF3CCHCrtpsJhCq/1kGfGBA2+eAHVLKUGCH/jnAAsBaShkODAMeEkIEtvQFdgeaVA0WZhaM8x5n6lAaNL/PfN6a9BZnCs6w5OclpBWnNd9IUToAKSVLly6lf//+PPXUU3XLs7Ky6h5v2LChrgT4jTfeSGxsLOXl5dTU1LBr1y7Cwq78uAwPDycnJ4fk5GSSk5Px9fXl6NGjeHp6drhy34acYhoBJEgpz0spq4C1wJyrtpmD7pu+lFLuB5yEEF7NtJ0DrNY/Xg3cqn8sAVshhAXQA6gCilv06roBqS/ON9JzZJsV5zOGqf5T+b9p/0dBRQF3br6To9lHTR2SojQrOjqazz//nJ07d14xpPWPf/wj4eHhDBo0CI1Gw9tvvw2As7MzTz31FMOHD2fIkCFEREQwa9YsQHcNormq0wMGDGDhwoWEhYUxY8YMVq5cWXeR+4MPPuCBBx4gJCSE4OBgZs6cCcDSpUvJz88nJCSEt956ixUrjDi5UlNDnPTntm5Dd2ro8vMlwPtXbfMTMK7e8x1AZFNtgcKr9nFR/9sSXSLJBcqAZY3EtQw4DBz29/dvcFhXd5B4MVEOXDVQrj291tShGCS5KFnOWj9LDl0zVG5M2GjqcJQOrCMMc+1K2mqYa0N9lauvijS2jSFtrzYCqAW8gSDgaSFE72t2IuVHUspIKWVkr169mtll17UzbScAE/0mmjgSwwQ4BPDlTV8y1H0oz+95nveOvqeGwSpKB2VIgkgH/Oo99wUyDdymqbbZ+tNQ6H/n6JffCWyRUlZLKXOAaHS9EaUB7V2czxgcrR358IYPmRc6j49PfMzTUU9TVl1m6rAURbmKIQniEBAqhAgSQlgBi4CNV22zEbhb6IwCiqSUWc203Qjco398D/CD/nEqMEW/L1tgFGDcAdBdRN6lPE7knjD5zXEtYWluyUujX+KZyGfQpGm4Y9MdnC86b+qwlA5GqnItRtHS97HZBCGlrAEeA7YCp4F1UspTQojlQojl+s02A+eBBOBj4JGm2urbrACmCSHOAdP0z0E36skOOIkuwXwmpYxt0avr4qLSopDITpkgQDcM9p4B9/Dx9I8pqizijp/u4JeUX0wdltJB2NjYkJ+fr5JEK0kpyc/Px8bG5rrbqjmpO7FHdzxKYmEiP8/7udNP1HOh7AJPRz1NbF4s9w28j8eHPo6FmbrRvzurrq4mPT29yWJ3imFsbGzw9fXF0vLKG2mbm5Na/Q/spMqry9mfuZ+FfRd2+uQA4GnryWczPmPFwRV8dvIzYnNjWTF+Rae6tqIYl6WlJUFBQaYOo1tTpTY6qX2Z+6jSVnXa00sNsTK34sXRL/KPcf8gLj+O2368rW4KVUVR2p9KEJ3UzrSd2FvZM9RjqKlDMbqbg29m3ex1eNt687jmcf5x4B9U1laaOixF6XZUguiEarQ17E7fzQTfCR2mOJ+xBToG8sVNX3BX/7v46sxXLN60mPOFapSTorQnlSA6oZicGAorC7vU6aWGWJlb8eyIZ1k5dSU55Tks+HEBq0+tplZba+rQFKVbUAmiE9KkabA0s2ScT8cszmdsE3wnsH7Oesb4jOHNw29y/9b7VcE/RWkHKkF0MlJfnG+E1whsLW1NHU67cevhxnuT3+PVca9y7uI55v84n6/PfK3GyCtKG1IJopM5X3SetJI0pvhNMXUo7U4IwS3Bt7B+znqGug/llQOv8OAvD6rehKK0EZUgOhlNmm7Y50TfzlGcry142nry4Q0f8sKoFziVd4q5G+fyyYlPqNZWmzo0RelSVILoZDSpGga4DsDD1qP5jbswIQQL+y7k+znfM95nPO8efZfbf7qd47nHTR2aonQZKkF0IrnlucTmxXb50UvXw8PWg7cnv817k9+juLKYJZuX8Mr+VyipKjF1aIrS6akE0YlEpUcBdLi5pzuCyf6T+eHWH7iz/52si1/H7A2z2XBug5prQlFaQSWITkSTqsHHzodQp1BTh9Ih2Vra8tyI5/hq9lf42fvx4t4XWbxpMbG5qhiworSEShCdRHl1OQeyDjDZb3KrivPll1ayUpNAWWWNEaPrWAa4DuDzmZ/zj3H/ILs8m8WbF/OXPX8h71KeqUNTlE5FJYhOYm/mXqq0VUzxb93w1re3n+WNrfE89PkRqmq67ukXIQQ3B9/Mj3N/5P6B97MpaROzN8zmkxOfcKnmkqnDU5ROQSWITkKTpsHByoGh7i0vzldcUc2GoxkA7EnI45lvjqPVdu0bzWwtbXly2JN8P+d7hnsM592j79Zdn1AlOxSlaSpBdAI12hp2pe9igu+EVk2is+5QGmVVtfz42Dj+cGNfNh7P5O+b4rrF3cgBDgH8e+q/WTVjFZ49PXlx74vc9uNt7Erb1S1ev6K0hEoQncCxnGMUVRa1anhrrVayel8ywwOdCfd15JFJwdw3NpDPopNZqUkwYrQd2zCPYXxx0xe8NektqrXVPLbzMe7beh8xOTGmDk1ROhyVIDqBy8X5xvqMbfE+tp/OJq3gEveN1c3QJYTghVlhzB3qw5vbzvJ/uxKNFW6HJ4RgWsA0NszZwF9G/oWkoiSW/LyE5duXcyL3hKnDU5QOQyWIDk5KSVRaFCO9RraqON9n0Un4OPVgethvd2CbmQneuG0Qswd58drPZ/h4d/eab8HSzJLb+93Oz/N+5vcRv+dU3inu3Hwnj+54lFP5p0wdnqKYnEoQHVxiYSJpJWmtOr10KrOI/ecLuHt0ABbmV/6TW5ib8c7tQ5gV7sWrm0/zya/dK0kA9LTsydLwpWyZv4UnIp7geO5xFv20iN/t+B1x+XGmDk9RTEYliA7ucnG+SX6TWryPVdHJ9LA0Z9Fw/wbXW5ib8c6iIcwc6Mkrm053u57EZbaWtjwQ/gBb5m3hsSGPcSTnCLf/dDuPbH+EI9lH1MVspdtRCaKD06RpGOg6EPee7i1qn1dayQ8xmcwf5oNjz8anJ7U0N+O9O4bW9STe2Hqm234g2lnZ8dDgh9g6fyu/G/o7TuWf4t4t93L3z3cTlRalynco3YZKEB1YTnkOJ/JOtKr20v8OpFJVq+XeMUHNbns5Sdwxwo+VmkT+8v1Jarv4fRJNsbeyZ9mgZWyZv4XnRz5PTnkOv9v5O+ZvnM+PiT+q8uJKl6cSRAcWlRYF0OLrD1U1Wj7fn8LEPr0IcbczqI25meAfc8N5eFIwXx5I5Ym1x7r0HdeG6GHRgzv63cFP837itfGvAfD8nueZtX4Wq06uoqiyyMQRKkrbUAmiA4tKi8LXzpcQp5AWtd90IpPckkruH9d876E+IQTPzujHczP78VNsFvd8epCicvVt2dLMktm9Z7P+lvWsnLoSHzsf/nXkX0z7dhqv7H+F80Xd89qN0nWpBNFB1RXn829ZcT4pJZ/uSSa4ly0TQt1aFMPyicG8tXAwh1MKmPtBNCn5ZS3aT1cjhGCC7wQ+m/EZ39z8DdMDprP+3HrmfD+H5duXsydjj7pOoXQJKkF0UNGZ0VRpq1p8eulIykVOZBRx39igVlV/nRfhyxdLR1JQVsXc/+zlcHJBi/fVFfVz6ccr417hl9t+4dEhjxJfEM/D2x/m1h9u5cvTX1JcVWzqEBWlxVSC6KA0qRocrR1bXJzv0+gkHGwsmBfh0+pYRvZ2ZcMjY3HsYcmdnxxg/dH0Vu+zq3Ht4crywcvZNn8b/xj3D2wtbFlxcAVT103lL3v+QmxubLcdFaZ0XipBdEA12hp2Z+xmgk/LivOlXyxny8kL3DHSn55WLS/uV1+Qmy3rHx5DhL8TT607zos/nOz2F68bYmluyc3BN/PV7K/4evbXzA6ezbaUbSzevJgFPy7g6zNfU1pVauowFcUgKkF0QHXF+Vo4vPXzfSkIIbh7dKBR43K2teKLpSN5YFwQa/alcMfH+8kurjDqMbqSMNcw/jr6r+xcsJMXRr2AEIJXDrzClG+m8NLel4jJiVG9CqVDMyhBCCFmCCHihRAJQojnGlgvhBDv6dfHCiEimmsrhHARQvwihDin/+1cb90gIcQ+IcQpIcQJIYRNa19oZ6JJ02BlZsVY7+svzldeVcNXB1OZMcATH6ceRo/NwtyMv8wO4993DOV0VjGz3tvDgfP5Rj9OV2JnZcfCvgtZN3sd/7vpf8wInMHmpM0s+XkJt3x/C5+c+IQLZRdMHaaiXKPZBCGEMAdWAjOBMOAOIUTYVZvNBEL1P8uADwxo+xywQ0oZCuzQP0cIYQF8ASyXUg4AJgHdZoyllBJNqoaRXiPpadnzutt/dzSD4ooa7hsbaPzg6rl5sDffPzoWexsL7vh4P+9sP0tNrTrl1BQhBOG9wnl57MvsXLCTl8e8jGsPV949+i7Tv53Osm3L2HR+k5rxTukwDOlBjAASpJTnpZRVwFpgzlXbzAHWSJ39gJMQwquZtnOA1frHq4Fb9Y+nA7FSyuMAUsp8KWW3mforoTCB9NL0Fp1e0molq6KTGOTryLAA5+YbtFIfD3s2PjaWWwZ78872c9z58QEyCtWHmyHsrOyYGzqXVTNWsXnuZh4a/BCpJak89+tzTFmnOwV16MIhNVxWMSlDEoQPkFbvebp+mSHbNNXWQ0qZBaD/fbnYUB9ACiG2CiGOCiH+2FBQQohlQojDQojDubm5BryMzuFycb6JvhOvu+3uc7kk5pZx39jAVg1tvR72Npa8s2goby0czKnMIma+s5ufT2S1y7G7Cj8HPx4d8iib523m0xs/ZYr/FDYnbeb+rfcz7dtpvHHoDU7mnVTXK5R2Z8gQl4Y+aa7+S21sG0PaNhTTOGA4UA7sEEIckVLuuGInUn4EfAQQGRnZZf7naFI1hLuFt6g432fRyfSyt2ZWuHcbRNa0eRG+RPg788TaYzz85VHmR/jy4s1hOPZovECgciUzYcZwz+EM9xzOn0f+mV3pu9ictJn/nfkfa+LW4G/vz4ygGcwMnEmIc8vurleU62FIDyId8Kv33BfINHCbptpm609Dof+dU29fu6SUeVLKcmAzEEE3kFOew8n8ky26OS4hp5RdZ3NZMioAKwvTDE4LdLPlm+VjeGxyCN/HZDD97V1ozuQ031C5Rk/LnswMmsm/p/ybqIVRvDzmZbztvPnkxCfM3TiXeRvn8XHsxyQXJZs6VKULM+ST5BAQKoQIEkJYAYuAjVdtsxG4Wz+aaRRQpD9t1FTbjcA9+sf3AD/oH28FBgkheuovWE8EusWsLa0pzrdqbxJWFmbcObLhOR/ai5WFGc/c2JcNj4zBqYcV9606xDPfHKfoUrcZZ2B0jtaOzA2dy8fTP2bHgh38acSfsLWw5b1j73Hz9zcz94e5vH/sfeIL4tVpKMWohCF/UEKIm4B3AHPgUynlq0KI5QBSyg+F7oT3+8AMdKeF7pNSHm6srX65K7AO8AdSgQVSygL9uruAP6E7HbVZStngdYjLIiMj5eHDh6/vlXdAD29/mJTiFDbN3XRd1xCKyqsZ9doOZg/y4o0Fg9swwutTWVPLv3ck8MGuRNzsrHjp5gHMGOjZbtdHuroLZRfYkbqDHak7OJJ9BK3U4mvnyw0BNzDVfyqDeg3CTKhbnZTG6U/fRza6vit84+gKCaKsuozxa8dzR787+MPwP1xX2//blchrP59h8+PjCfN2aKMIW+5EehHPfhdLXFYxk/r24uVbBuLvev1DeJXG5V/KJyotiu2p29mftZ8abQ3uPdyZ7D+ZyX6TGe45HCtzK1OHqXQwKkF0EtuSt/H0rqf59MZPGe453OB2NbVaJryuwd+1J2uXjW7DCFunplbL6n0pvLUtnhqt5LHJISyb2BtrC3NTh9bllFSVsDt9N9tTtrMnYw8VtRX0sOjBGO8xTPSdyHjf8bj1aFmFX6VraS5BGKdQj9JqmrSWFefbFpdNZlEFL90yoI0iMw4LczOWjgtiVrgXL/90in/9cpYNMRm8MDuMyX1bNp2q0jB7K3tm9Z7FrN6zqKip4OCFg+xO382u9F3sSNUNBhzoOpAJfhOY6DuR/i791Wk/pUGqB9EBVGurmfT1JCb5TeLVca9eV9vbPthLdkkFUc9Mxtys8/wnj4rP4aWNp0jOL2dCn178+ab+9PW0N3VYXZqUkrMXz9Yli9jcWCQS9x7ujPcdzxjvMYz0GomjtaOpQ1XaiepBdALHso9RXFV83aOXYtMLOZxykRdmh3Wq5AAwqa872550Y82+ZN7bcY6Z7+7mjhH+PDWtD6521qYOr0sSQtDXpS99Xfry4KAHKagoYE/GHnal7WJr8la+O/cdZsKMgW4DGeM9hrHeYxnoNrBFFYWVrkH1IDqAfx78J+vi1/Hrol+vq/7Sk1/HsO3UBfY9PxUHm857Q9rFsire2X6WLw6k0tPSnOWTgrlvbKDRSpUrzavR1nAi7wR7M/eyN3MvJ/NOopVa7CztGOk1kjHeYxjjPQZfe19Th6oYkbpI3cFJKZm5fibBTsGsnLrS4HY5xRWM/edOFo8M6PDXHwyVkFPKa5tPs+NMDm521jw6OZg7R/qrC9kmUFRZxIGsA3UJI6tMVz7F396fUV6jGO41nOEew3Ht4WriSJXWUKeYOrhzhefIKM1gafjS62r3xf4UarSSe8cEtk1gJhDibsd/7x3OkZQC3tgaz99+jOPj3ed5fGoo84f5YmmuxvS3F0drR6YHTmd64HSklCQXJ9cli01Jm1h3dh0AIU4hDPcczgjPEUR6ROJk42TawBWjUj0IE/u/4//H+zHvs3PBTnr17GVQm4rqWsau2MlQfyc+ucfwIbGdiZSS6IR83tgWz/G0QgJde/Lo5BBuHeqjEoWJ1WhriMuP4+CFgxy6cIhjOcfqSpT3de5blzCGeQ7Dwarj3Zej/EadYurgFv20CHNhzpezvjS4zbrDafzx21i+fGAkY0O69nh2KSXbT+fw1i9nOZ1VjI9TD5ZP7M2CSD9sLNWpp46guraak/knOZh1kEPZh4jJiaGythKBoJ9LP4a6D2Wox1Ai3CNaVIRSaTsqQXRg2WXZ3PDtDTwR8QQPhD9gUBspJTe9twetVrLl9+O7zfh1KSWa+Bze35nA0dRC3OyseWB8EHeNCsDOWp0p7UiqaquIzY3l0IVDHMk+QmxebF0Pw8fOhwj3iLqEEeQYpMqBmJC6BtGB7UrfBVxfcb795ws4nVXMinnh3SY5gG6I5pR+Hkzu687+8wX8JyqBFT+f4YOoRBaP9Ofu0YF4OnarmWk7LCtzKyI9I4n01H3uVGuriS+I52j2UWJyY4jOjObH8z8CumsdQ3v91sMIcw1TJUE6ENWDMKHl25eTVpzGT3N/MvjDftmawxxKLmDfn6Z2+1MsMWmFfBCVwLa4bMyFYPYgL5aO6024r7rRqyOTUpJWksbRnKMcyznG0eyjJBcnA2BpZkk/l36Eu4UT3iucwW6D8bX37VZfhtqTOsXUQbWkOF9qfjkT39TwyKRg/nBjvzaOsPNIzS9n1d5k1h1Oo7SyhhGBLtw/LpBpYZ6d7gbC7qqgooBjOceIzY0lNjeWU/mn6k5LOVs7M9BtYF3CGNhroLr4bSTqFFMHtSdjD9Xa6us6vbR6XzLmQrBkVGDbBdYJ+bv25MWbw3hyWijrDqezam8Sy784iq9zD+4Y4c+CSF/c7dXpp47MxcaFqf5Tmeo/FdCNlEosTCQ2L5YTuSc4kXeCPRl7kPoJKQMdAhnUaxDhbuEMcB1AH5c+WJurO/CNTfUgTOS5X58jOiMazUKNQaUMSitrGP2PHUzu5857d1xfQb/uplYr+SXuAp/vTyE6IZ/+5uk87n4c38hZDBg1EzM1TLZTKq0q5WT+SU7kntD1NPJiKagoAMBCWBDsFEyYa1jdTx/nPthYqC8GTVE9iA6oWlvN7vTdTPabbHCdm28Pp1FSWcP944LaOLrOz9xMMGOgFzMGenHhwDe4bP0rVhcvwS9fkr7di4yg2widvgwXT9POvqdcHzsrO0Z5jWKU1yhAdy0jsyyTuPy4uh9NmoYNCRsAMBfm1ySNvs59VdK4DipBmMCx7GOUVJUwxW+KQdtrtZJVe5MZ6u/EED+ntg2uq9BqYffreEa9Bt4RVM75iJMHtmNz4ktGnv83NR+sJMZ2FNqhdzFw4gKsrNTImc5GCIGPnQ8+dj5MC5gG6JJGVlkWp/NPcyr/FHEFcexO3833Cd8DuqTR26k3/V3608e5j654oXNfnG2cTfhKOi6VIExAk6bB2tya0d6GTfCjic8hOb+cp6f3bePIuojKEtiwHM78BIPvhNlvY21pw7BbQuGWh0mKP06m5mP6XvgRt+hHyI1+nnPuM+g1Zgkhg8YgzNQpqM5KCIG3nTfedt5MDdBdz5BSkl2erUsY+p7G3sy9bEzcWNfOvYc7fVz60Ne5b13S8Hfw7/aVbNU1iHZ2uThfiFMI709936A2iz/ZT2JOGb8+O1mVmWhOwXn46k7IOwvTX4FRD0MjQyRrqiqJ+/U75NEv6F+6HytRS4qZH9kBN+M/6R48A9RIsa4s/1I+Zy+e5ezFs8QXxBN/MZ7zReep0dYAYG1uTbBTcF3S6OPchz7OfbrUfBnqGkQHc/biWTJKMwy+czr+QgnRCfn8cUZflRyak7gTvrlP9/iu7yC46RFiFlbWDJp6J0y9k+L8HI7u/Bz7s+sZkfQfSPoPZyzDKAi+ldDJd9HLw6cdXoDSnlx7uDK6x+grevLVtdWcLzpP/MX4uqQRlRZVd10DwL2nOyFOIQQ7Bdf9DnYMxs7KzgSvom2pBNHONGkaBIJJfpMM2v6z6CRsLM24Y7i6oNooKWH/f2DbX6BXP1j0Jbj0vq5dOLi6M2rB08DTpJ2PJ/3XNXilbGTMmX9Qc3oFsdaDuRQyi9CJi3Dx8Gub16GYnKW5Zd2kSgTrlkkpybuUV5c0EgsTSShM4Jv4b6iorahr62XrdUXSCHEKobdj7+ua46WjUaeY2tntP92OhZkFX97UfHG+grIqRr+2g3kRvrw2L7wdouuEqi/Bj7+H2LXQ/2a49UOwNtI3OSlJPX2QrL1f4Z2xFT+ZiVYK4m3CKek9i8Bxi3D3CTTOsZROp1ZbS2ZpJucKz9UljcTCRJKKkqjSVtVt52Pno+tl6JNGoEMggY6BHeJmP3WKqQO5UHaBuPw4noh4wqDtvzqYSmWNlvvGBrZtYJ1VUQZ8vRgyj8HkP8P4Z8CYF5iFwD9sJP5hI5FaLYlxh7mw72u8s7bR//RrcPo1Tlv0J99/Bn6j5uIfOkiVhOhGzM3M8XPww8/Bjyn+v41IrNHWkF6SfkXSOFd4jr2Ze+uubwC49XAj0CGQIMegut9BjkF42XphbtYxyuioBNGOdqXpivMZMry1ulbLmn3JjA91o4+HfVuH1vmkHoCv74Lqclj0P+g3q00PJ8zMCB44guCBI5BSknI2hqx963BP+5lx59+G82+TKnzI8piI09BbCBl2A+YWnXcaWKXlLMwsCHTU9RIuj6QC3f1P6SXpJBclk1ScRFJREslFyWxN3kpxVXHddlZmVgQ4BhDkEESgoz5x6B/bWtq272tp16N1c5o0DQEOAQQ5Nn+z2+YTWWQXV6pTSw05sho2PQ2OvnDPRnDv366HF0IQ0HcoAX2HAq+RkxpP8t712CT/wtCsr7G68D+KfrbjnMMo6DOD4NG34uxq2GRQStdlaWZZ10uYzG8DKKSUXKy8qEscRUkkF+t+nyk4w/bU7Wiltm5b9x7uBDoGEuAQUPcT4hTSZnOFq2sQ7aS0qpTxX49ncb/FPDP8mWa3v3VlNEWXqtnx1ETMVME5ndpq2PInOPQxBE+B+f+Fni6mjuoKRYUFxEf/AGe3EFq0F2eKqZFmxFv1p8h7Am6DZxIyaCxmFuq7mdK8qtoq0krSrkgcycXJpBanUlhZCMC0gGm8NemtFu1fXYPoIPZk7qFGW8Nk/+aL8x1NvUhMWiF/u2WASg6XleXBunsgZQ+M+R1MfQnMO96fr6OTCyNm3Qez7qO2poazMbsojNmIy4U9jEn5AFI+oHCjPUkOw9H2noz/8Nn08rm+EVdK92FlblV3gftqhRWFpJSkYGXWdlUAOt7/sC5Kk6rB2dqZIb2GNLvtZ9HJ2NtYcNuwtuk2djpZsbD2TijNgbkfweDbTR2RQcwtLOgTORUideehC7LTOX9wE7XndtC7+AC9YnZCzAskm/mR5Toaq743EBI5DUenjtUrUjomJxsnnGyc2vQYKkG0g2ptNb9m/MoUvynNjk7IKrrE5hNZ3DcmEFs1lSac/A6+f1R3Kun+LeATYeqIWszFwxeXmx8CHkJbq+Vc3CHyj2/GLn03ETkbsM5dR82vZsRbhnLRfSR2fScREnkDNrZd585dpXNRn0Dt4Gj2UUqqSgw6vfT5vhSklNwzJrDtA+vItLWw8xXY8xb4jYKFa8Dew9RRGY2ZuRmh4SMJDR8JQNWlUs4c01AUtxOH7P0My/gSy8w11Ow046xVHwrcR9IjdCK9I6Zi7+Bk2uCVbkMliHZQV5zPq+nifJeqavnfwVSmhXng59J5775stYoi+O4BOLcNht0LM98Ai65dbdWqhx39xtwMY24GoLSkiFNHdlAWH4VL7kGGpX+BZcZqqjXmnLEMocA1ApveYwgYMhlXdWe30kYMShBCiBnAu4A58ImUcsVV64V+/U1AOXCvlPJoU22FEC7A10AgkAwslFJerLdPfyAOeElK+WbLX6JpSSnRpGoY5TWq2Vvuv4/JoLC8mvvHduM5H3LPwto74GIyzHoLhi81dUQmYWfvyJBJ82DSPADKS4s4e3QnZfEaHHIPM+zCt1hnfwX7IF14keM0GK3vSNwHTMQ3dAhm5h3jRiulc2s2QQghzIGVwDQgHTgkhNgopYyrt9lMIFT/MxL4ABjZTNvngB1SyhVCiOf0z5+tt8+3gZ9b+wJN7ezFs2SWZbJs0LImt5NS8ll0EmFeDowI6qYXKc9u1fUczK3g7o0QONbUEXUYPe0cGTBhLkyYC0BVxSXOxEZzMf5XrLMOEXhxLy4Xt8AJKMaWJJswSj0isQseQ2D4WBydXU38CpTOyJAexAggQUp5HkAIsRaYg+7b/WVzgDVSd1PFfiGEkxDCC13voLG2c4BJ+vargSj0CUIIcStwHihr+UvrGHam7UQgmOg3scntohPyOZtdypsLBne/cg1S6q417Pg7eIbr7ox2UqdNmmJl04N+I26AETcAoK3VkpJ4guxTuyFtPx6FxxmsH1ar3SFIMfch134A0mcYrn1G499/OBbWPUz8KpSOzpAE4QOk1Xuejq6X0Nw2Ps209ZBSZgFIKbOEEO4AQghbdIliGtDoHWVCiGXAMgB//45b6VSTqmFQr0G49XBrcrtPo5Nws7Pi5sFe7RRZB1FVBj88BqfWw8D5cMv7YNWNr7+0kJm5GQF9BhPQZzDwOwBKC3NJjf2VkvMHsMmOIahoP65FWyEOqjZYcM6yNwVOAxG+w3DrOwb/0EFYqBv4lHoM+Wto6Ovs1bdfN7aNIW2v9jfgbSllaVPfpKWUHwEfge5O6mb2aRIXyi5wuuA0v4/4fZPbJeWVsfNMDk9MDcXaohudO76YAmsXQ/ZJuOFvMPaJRif3Ua6fnVMvwibMgwm66xhSqyUjNYGMU3uoTj2C48VYBuZuxjZvPcRAqexBilVvSpzCMPMeTK/Q4fj1GYqFlbVpX4hiMoYkiHSgfn/fF8g0cBurJtpmCyG89L0HLyBHv3wkcJsQ4nXACdAKISqklIZNv9aBRKVFATQ7vHVVdBKW5oLFozpuT8jokvfAuruhtgYWfwOh00wdUZcnzMzwCeyDT2Af4H4AamtqSEmIJT9+L7XpR3AoPM2gnI30zP0GjkOVtCDRMpCLDv2QnoOwDxyGX/9IbO2dTPpalPZhSII4BIQKIYKADGARcOdV22wEHtNfYxgJFOk/+HObaLsRuAdYof/9A4CUcvzlnQohXgJKO2NygHrF+RwaH5VUdKmab46kc/Mgb9ztbdoxOhOREg59Alue003qs+grcAsxdVTdlrmFBQH9Igjo99sNiNqaGlIST5Jz9iDV6THYXYwjuGAXzgU/QRxoNwlSzbzJtu1DlWsYPXzD8QyNwMs/VM3n3cU0myCklDVCiMeAreiGqn4qpTwlhFiuX/8hsBndENcEdMNc72uqrX7XK4B1QoilQCqwwKivzMRKqko4eOEgd/W/q8mLzt8cTqO8qpb7usPQ1ppKXRXWY59Dnxkw7yOwUXcJdzRmFhYE9B1CQN8hdcukVktWeiLZ8YeoSDtGj/xT+JWdxLNUAylAtO4UVYZVIMUOfRDu/bEPGIxXn2E4uHSdGxy7G1XNtY1sSdrCH3b/gdUzVhPh0XB5iFqtZOIbGrwcbfhm+Zh2jrCdlVyAr5dA+kHdxD6T/2zcyX0UkygvLiA9/giFycfRZp/CoegsPlVJOIrfBiDm4swF6yBKHfsgPMJwCBiET/BgHJ276XDuDkRVczWRnWk7cbFxYXCvwY1u80tcNukXL/Hnm9p3PoN2l3EE1t4FFYWwYBUMmGvqiBQj6engQp/h02D4b9eQpFZLRnoyuYnHKE8/gUXeGZzLzhGa/R02OWvhhG67bFzJsfan3CEY0asvDn4D8AoehGMvXzVYoYNQCaINVGur2ZO+h6kBU5sszvdZdBI+Tj2YFtaFu+AxX8GPT4CdByzdprvPQenShJkZPv698fHvDcyvW66tqSEr9Qx552MozzyNef45HMqSCMrZhF3ut3V3VhVjS5alH8W2QdS6hGLl1R/XgAF4BvbDWo2oalcqQbSBI9lHKKkuYbJf46OXTmUWcSCpgOdv6oeFeRc81VJbA7+8CPtXQuB4WLAabNXdvN2ZmYUFXr0H4tV74BXLtbVaMjOTyDkfS1lGHGZ557ArTSKwcD+9Cn/W3TIbDZXSgmQzTwps/Ki0D0S4BWPv1Qe3wDDcfXojOsg8zl2JShBtQJOqL87n3Xhxvs+ik+lhac7tkV1waGt5AXx7H5yPghEPwY2vgrman1lpmJm5Gd5+wXj7BQNXnn4sLswjO/EExekn0WbHY1mcjHN5Kl7lh7HJqa7rdVRISy6Ye1HYw59KhwDMXIPp4dUHV/8wPLwDVW2qFlIJwsiklGjSNIz2Gk0Pi4ZLGeSWVLIxJpPbh/vh2LOLfXBmx+mK7RVn6u6Kjlhi6oiUTszByQ2HYZNh2JW9cW1tLRcyk8lLiaM0M57avERsSpJxKU/Bu/QA1lnVcFK37SVpRaa5F4XWvlTa+4NzID09euPs0wcP/1Bsetia4JV1DipBGFn8xXiyyrJYPnh5o9v870AqVbVa7h0b2H6BtYfTP8L6h8DaDu7dBH4jTB2R0kWZmZvj6ReMp18wcPMV62pqashMTyQ/7QyXLpxF5idiU5xMr4oU3MsP6noe8b9tn4sLeZZelNv6UusYgIVrILYewTj79MHNK6Bb9z5UgjAyTaoGgWCC74QG11fW1PL5/hQm9e1FcC+7do6ujWi1sOufsGsF+AyD278AB29TR6V0UxYWFngH9sU7sO8166S2lvycDHJT4ym9kEBVXhIWRSn0LM/Ap/AI7he3YZby29D/KmnBBTN3Cq28KbP1pdbRHyvXAOzcg3D16Y2bpz/mXTiBqARhZJo0DYN7DW60ON+m2CzySiu7zo1xlSWwYTmc+QkG3wmz3wbLbnBHuNIpCTNzXD39cfX0R1cP9Erl5WXkpidwMTOBipzzcDEFq5JUHCoy8C84g1NBKST9tn2VNOeCmRuFlh5c6ulFrb0fFs5+9OgViKNnIG6+wVj3sG+/F2hkKkEY0eXifE8Oe7LB9VJKPo1OIsTdjgmhTVd37RTyE3XF9vLOwowVMHK5Gr+udGo9e9rWq4p7rYqSi+RmJFCYlcSl3GS0hWlYlGRgW5GFb+ERel3chnnalTcfX8SBfPNelFh7UmHrjdbBD0sXP2x7BeDsEYCblz9WVh1zxkSVIIxIk6YBaHR46+GUi5zMKOaVWwd2/jkfEnboRioJM1iyHnpPMnVEitLmbOyd8es3HL9+wxtcX1FRQUZGMkXZSZTnJFF7MQ3zkgxsyjNxqUilV/lheuZWQuJvbWqlIEc4U2jhRqm1B9U9PcDBG0tnX3q6+eHkGYiLRwBWJriYrhKEEWlSNQQ6BBLk2PDpo8+ik3DsYcm8CJ92jsyIpIR9K+GXF6BXP93kPi5d5HSZorSSjY0N/sH9ILhfwxtISWlRPgWZCRRnp1CRn0ZtUQZmJRewuXQBl0vJuJYexj730jVNC7GnwNyVUit3Knp4UGvnjbmjN05BQ+gTMalNXo9KEEZSUlXCoexDLOnf8LDO9IvlbDl5gQcn9KanVSd926sv6e6Kjv0a+t8Ct36gG7GkKIphhMDOyQ07JzcIG9XgJlJKiosvkp+ZQlGOLonUFGZiUZaJdXk2dlU5eF86i1tBIQBHkqeAShAd256MPdRoaxqd++HzfSkIIbh7dGD7BmYsRRnw9WLIPKYrtDf+GVVsT1HagBACB0cXHBxdoP/QRrerrLxEflYqbTkHpUoQRqJJ1eBi48Igt0HXrCuvquGrg6nMGOCJj1MnnAc4db+uEmt1ue6UUr9Zpo5IUbo9a+seDQ7lNSb1FdAIqmur2ZOxh4m+Exsszvfd0QyKK2q4f1xg+wfXWkdWwarZulNJD2xXyUFRuhHVgzCCw9mHGy3Op9VKPotOYpCvIxH+ziaIroVqq3Wzvh36BIKnwG2fQo9OFL+iKK2mehBGoEnTYGNuwyjvay867T6Xy/ncMu4fG9R5hraW5sKaObrkMOZxWPytSg6K0g2pHkQrXS7ON8p7VIPF+T6NTsbd3pqbwtvyUpIRZR3X3fxWlgvzPoZBC00dkaIoJqJ6EK10puAMF8ouMMVvyjXrEnJK2H02lyWjArCy6ARv9cnv4L83gtTC/VtUclCUbk71IFpJk9Z4cb7PopOxsjDjzpEdfM4HbS3s/DvseRv8RsHtn4Odu6mjUhTFxFSCaKWotCiGuA/BtceVs6UVllex/mgGtw7xxtWuA0+TeKkQvnsAEn6BYffCzDfAomPWhVEUpX11gvMeHVdWaRanC043OHpp7aE0LlXXduyqrbln4ZOpcF4Ds96Cm99VyUFRlDqqB9EKl4vzTfKbdMXymlota/YmM7q3K/29HEwQmQHObtX1HMyt4O6NEDjW1BEpitLBqB5EK2jSGi7Ot/VUNplFFdzXEWeMkxJ2vwn/u11XZG9ZlEoOiqI0SPUgWqi4qpjDFw6zZMC1xfk+i07C36UnU/t7mCCyJlSVwQ+PwqkNMPA2uOXfYNXT1FEpitJBqQTRQnvS91Aja64Z3hqbXsjhlIu8MDsMc7MOdGPcxRTd/Q3ZJ+GGv8HYJ9TkPoqiNEkliBaKSovCxcaFcLfwK5Z/Fp2MnbUFCyN9TRNYQ5J+hXV364azLv4GQq+dalFRFOVq6hpEC1TXVvNrxq9M8pt0RXG+nOIKforN5LZhvtjbWJowQj0p4cBHurIZtm7w4E6VHBRFMZjqQbTAoexDlFaXXjO89Yv9KdRoJfeOCTRNYPXVVMKmp+HY59Bnhq5shk0HHVGlKEqHpBJEC2hSdcX5RnqNrFtWUV3LlwdSmdrPnUC39p879golF3TzN6Qf1E3sM/nPanIfRVGum0oQ10lKSVR6FKO9R19RnG/j8Uzyy6q439Q3xqUf0c38VlEEC1bBgLmmjUdRlE7LoK+VQogZQoh4IUSCEOK5BtYLIcR7+vWxQoiI5toKIVyEEL8IIc7pfzvrl08TQhwRQpzQ/762Cp4JnS44zYWyC1ecXpJS8umeJPp62DM62LWJ1m0s5iv4bCaYW8LSbSo5KIrSKs0mCCGEObASmAmEAXcIIcKu2mwmEKr/WQZ8YEDb54AdUspQYIf+OUAecLOUMhy4B/i8xa+uDWjSNJgJMyb6Taxbtv98AWculHD/uEDTzPlQWwNb/gTfLwe/EfBgFHiGN9tMURSlKYb0IEYACVLK81LKKmAtMOeqbeYAa6TOfsBJCOHVTNs5wGr949XArQBSymNSykz98lOAjRCiw1S7i0qLYkivIbjYuNQt+zQ6CeeelswZ4tP+AZUXwBfzYP9/YMRDsGQD2JqwF6MoSpdhSILwAdLqPU/XLzNkm6baekgpswD0vxuqLz0fOCalrLx6hRBimRDisBDicG5urgEvo/UySzM5U3DmitNLqfnlbD+dzeKRAdhYXjsfdZvKjoOPJ0PqPpizEm56XXd6SVEUxQgMSRANnTORBm5jSNuGDyrEAOCfwEMNrZdSfiSljJRSRvbq1cuQXbZaQ8X5Vu1NxlwIlowOaJcY6sRthE9ugOoKuHczDL2rfY+vKEqXZ8gopnTAr95zXyDTwG2smmibLYTwklJm6U9H5VzeSAjhC2wA7pZSJhryQtqDJk1DkGMQgY6BAJRUVLPucBqzBnnh4WDTPkFotbBrBez6J/gMg9u/BIdOMp2poiidiiE9iENAqBAiSAhhBSwCNl61zUbgbv1oplFAkf60UVNtN6K7CI3+9w8AQggnYBPwJylldMtfmnEVVxVz5MKRK04vfXskndLKmvab86GyBL6+S5ccBt+p6zmo5KAoShtptgchpawRQjwGbAXMgU+llKeEEMv16z8ENgM3AQlAOXBfU231u14BrBNCLAVSgQX65Y8BIcALQogX9MumSynrehim8Gv6r9TImroEodVKVu1NJsLfiSF+Tm0fQH4irL0T8s7BjBUwcrkqtqcoSpsy6EY5KeVmdEmg/rIP6z2WwKOGttUvzwemNrD8FeAVQ+JqT1FpUbjauDKo1yAAdp7JISW/nGem9237gyfsgG/vA2EGS9ZD70ltf0xFUbo9VX/BANW11ezJ2MMkv0mYCd1b9tneJLwcbZgx0LPtDiwl7P03fHkbOPjCgxqVHBRFaTeq1IYBDl3QFee7PHrpzIViohPy+eOMvliat1GOrb4EPz4BsV9D/1vg1g/A2q5tjqUoitIAlSAMsDNtJz0sejDKaxQAq6KTsbE0447h/m1zwKJ03eQ+WTG6Qnvjn1HF9hRFaXcqQTRDSklUWhSjvUZjY2FDQVkVG45lMC/CF2dbK+MfMHW/bqRSdQUs+gr63WT8YyiKohhAfS1tRlxBHNnl2Uz2141e+upgKpU1Wu4fG2j8gx1ZBatmg7U9PLBdJQdFUUxK9SCaEZUWpSvO5zuR6lota/YlMz7UjVAPe+MdpKYKtjwHh/8LwVPgtk+hh7Px9q8oitICqgfRDE2qhiG9huBs48zmE1lkF1cad86H0lz4/FZdchjzOCz+ViUHRVE6BJUgmpBRmkH8xfi6m+M+jU6mt5stE/sYqfZT1nH4aBJkHNFNCTr972DWzgX/FEVRGqESRBOi0qIAmOw/maOpFzmeVsi9YwMxMzPCHcwnvoX/3ghIuH8LDFrY+n0qiqIYkUoQTdCkaujt2JsAhwA+3ZOEvY0F8yN8W7dTbS388lf4bil4D4FlUeA91BjhKoqiGJVKEI0oqizicPZhJvtNJqvoEj+fvMCi4X7YWrfiuv6lQvjf7RD9Dgy7D+7eCHYNTYOhKIpiemoUUyP2ZOyhVtYy2X8ya/alIKXk7tGBLd9h7ln4ahEUpsCst2D4UqPFqiiK0hZUgmiEJk2Dq40rIQ5hfHVQw/QwT/xcerZsZ/FbYP2DYG4F9/wIAWOMG6yiKEobUKeYGlBVW1VXnO+HmCwKy6u5ryU3xkkJu9/U9RxcgnTXG1RyUBSlk1A9iAYcunCIsuoyJvlN4tVvkhjg7cCIIJfr20lVGXz/CMR9DwNvg1v+DVYt7IEoiqKYgOpBNECTpqGHRQ9qy0I4l1PKfWODENczOc/FFPjvdIj7Aaa9DPM/UclBUZROR/UgriKlRJOmYYz3GL7cn4WbnRU3D76OaT2TfoV1d+uGsy7+FkJvaLtgFUVR2pDqQVwlriCOnPIcBjqNZueZHBaPDMDawoC7m6WEA/8Ha+aArRs8uFMlB0VROjXVg7iKJlWDmTAjIdkPK/MiFo8yYM6HmkrY9BQc+wL6zIR5H4GNQ9sHqyiK0oZUgriKJk3DILchbNxfzOzBXrjb2zTdoOSCbv6G9EMw4Q8w6Xk1uY+iKF2C+iSrJ70knbMXz2JfO5jyqtrmq7amH9EV28s+BQtWw5S/qOSgKEqXoXoQ9Vwuzncs3ocRgS4M9HFsfOOY/8GPvwd7D1j6C3gObI8QFUVR2o36uluPJk2Dh00AWXl23D8usOGNamtgy5/g+4fBbwQ8GKWSg6IoXZLqQegVVRZxJPsITtXT8XHqwbQwz2s3Ki+Ab+6FpF0wcjlMfwXMLds9VkVRlPagEoTerxm/UitrSU0L4k9TAjG/es6H7FPw1R1QkgVzVsLQu0wTqKIoSjtRCUJPk6rBCkdqtQEsHO535cq4jbBhOVjbw72bwW+4aYJUFEVpRypBoCvO92vGHi4VDuS2Yf449tCfNtJqIeo12P06+ETC7V+Aw3XcVa0oitKJqQQBHLxwkEs15VSWhHHPmEDdwopi2PAQxG+GIYt1czhYNnNPhKIoSheiEgSwPWUnaK0Y5zOK4F52kJ8Ia++EvHMw458w8iG4nmJ9iqIoXUC3TxBaqWVb0k6qS0NZemMfSNgO394PwgyWbIDeE00doqIoikl0+/sg4vLiKKnJx1UMZXzuV/DlAnDwhQc1KjkoitKtGZQghBAzhBDxQogEIcRzDawXQoj39OtjhRARzbUVQrgIIX4RQpzT/3aut+5P+u3jhRA3tvZFNuXLkz8jpeA/VkcQv7wA/WbD0m26GeAURVG6sWYThBDCHFgJzATCgDuEEGFXbTYTCNX/LAM+MKDtc8AOKWUosEP/HP36RcAAYAbwH/1+2sSulG2EVUgGZ26FyX+BhWvA2q6tDqcoitJpGNKDGAEkSCnPSymrgLXAnKu2mQOskTr7ASchhFczbecAq/WPVwO31lu+VkpZKaVMAhL0+zG6YzE/UiIucOOlUlj0FUz8g7oYrSiKomdIgvAB0uo9T9cvM2Sbptp6SCmzAPS/3a/jeAghlgkhDgshDufm5hrwMq5VauPKmEvWDJv2IfS7qUX7UBRF6aoMGcXU0FdqaeA2hrRtyfGQUn4EfAQQGRnZ3D4bNL7fGMb3O9ySpoqiKF2eIT2IdKB+7QlfINPAbZpqm60/DYX+d851HE9RFEVpY4YkiENAqBAiSAhhhe4C8sarttkI3K0fzTQKKNKfNmqq7UbgHv3je4Af6i1fJISwFkIEobvwfbCFr09RFEVpoWZPMUkpa4QQjwFbAXPgUynlKSHEcv36D4HNwE3oLiiXA/c11Va/6xXAOiHEUiAVWKBvc0oIsQ6IA2qAR6WUtcZ6wYqiKIphhJQtOn3foURGRsrDh9W1BEVRlOshhDgipYxsbH23v5NaURRFaZhKEIqiKEqDVIJQFEVRGqQShKIoitKgLnGRWgiRC6S0YhduQJ6RwjEmFdf1UXFdHxXX9emKcQVIKXs1trJLJIjWEkIcbupKvqmouK6Piuv6qLiuT3eMS51iUhRFURqkEoSiKIrSIJUgdD4ydQCNUHFdHxXX9VFxXZ9uF5e6BqEoiqI0SPUgFEVRlAapBKEoiqI0qFsnCCHEDCFEvBAiQQjxXDscz08IoRFCnBZCnBJCPKFf/pIQIkMIEaP/ualemz/p44sXQtxYb/kwIcQJ/br3hGjdXKlCiGT9/mKEEIf1y1yEEL8IIc7pfzu3Z1xCiL713pMYIUSxEOL3pni/hBCfCiFyhBAn6y0z2vujL2//tX75ASFEYCviekMIcUYIESuE2CCEcNIvDxRCXKr3vn3YznEZ7d/NyHF9XS+mZCFEjAner8Y+G0z7Nyal7JY/6MqPJwK9ASvgOBDWxsf0AiL0j+2Bs0AY8BLwTAPbh+njsgaC9PGa69cdBEajm4HvZ2BmK2NLBtyuWvY68Jz+8XPAP9s7rqv+vS4AAaZ4v4AJQARwsi3eH+AR4EP940XA162IazpgoX/8z3pxBdbf7qr9tEdcRvt3M2ZcV63/F/CiCd6vxj4bTPo31p17ECOABCnleSllFbAWmNOWB5RSZkkpj+oflwCnaWC+7XrmAGullJVSyiR0822MELoZ+ByklPuk7l97DXBrG4Q8B1itf7y63jFMEddUIFFK2dQd820Wl5RyN1DQwPGM9f7U39e3wFRDejkNxSWl3CalrNE/3Y9uVsZGtVdcTTDp+3WZvv1C4Kum9tFGcTX22WDSv7HunCB8gLR6z9Np+sPaqPTdu6HAAf2ix/SnBD6t141sLEYf/eOrl7eGBLYJIY4IIZbpl3lI3cyA6H+7myCuyxZx5X9cU79fYNz3p66N/sO9CHA1Qoz3o/sWeVmQEOKYEGKXEGJ8vWO3V1zG+ndri/drPJAtpTxXb1m7v19XfTaY9G+sOyeIhjJnu4z5FULYAd8Bv5dSFgMfAMHAECALXTe3qRjbIvaxUsoIYCbwqBBiQhPbtmdcCN10tbcA3+gXdYT3qykticPoMQoh/oxuVsYv9YuyAH8p5VDgKeB/QgiHdozLmP9ubfFvegdXfglp9/ergc+GRjdt5DhGja07J4h0wK/ec18gs60PKoSwRPcH8KWUcj2AlDJbSlkrpdQCH6M7/dVUjOlcedqg1bFLKTP1v3OADfoYsvVd1svd6pz2jktvJnBUSpmtj9Hk75eeMd+fujZCCAvAEcNP0VxDCHEPMBtYrD/VgP50RL7+8RF05637tFdcRv53M/b7ZQHMA76uF2+7vl8NfTZg4r+x7pwgDgGhQogg/TfURcDGtjyg/nzff4HTUsq36i33qrfZXODyCIuNwCL96IMgIBQ4qO9qlgghRun3eTfwQyvishVC2F9+jO4i50n98e/Rb3ZPvWO0S1z1XPHNztTvVz3GfH/q7+s2YOflD/brJYSYATwL3CKlLK+3vJcQwlz/uLc+rvPtGJcx/92MFpfeDcAZKWXd6Zn2fL8a+2zA1H9jzV3F7so/wE3oRgskAn9uh+ONQ9eliwVi9D83AZ8DJ/TLNwJe9dr8WR9fPPVG3gCR6P6DJQLvo78rvoVx9UY3IuI4cOrye4Hu/OQO4Jz+t0t7xqXfX08gH3Cst6zd3y90CSoLqEb3TWypMd8fwAbdKbQEdKNQercirgR055ov/41dHrkyX//vexw4CtzcznEZ7d/NmHHpl68Cll+1bXu+X419Npj0b0yV2lAURVEa1J1PMSmKoihNUAlCURRFaZBKEIqiKEqDVIJQFEVRGqQShKIoitIglSAURVGUBqkEoSiKojTo/wFGXsvR0y146wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "# Three settings of the lrate hyperparameters.\n",
    "opts = [NoamOpt(512, 1, 4000, None),\n",
    "        NoamOpt(512, 1, 8000, None),\n",
    "        NoamOpt(256, 1, 4000, None)]\n",
    "plt.plot(np.arange(1, 20000), [[opt.rate(i) for opt in opts] for i in range(1, 20000)])\n",
    "plt.legend([\"512:4000\", \"512:8000\", \"256:4000\"])\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\" python\n",
    "预测softmax_res = [0.1, 0.3, 0.6]\n",
    "真实结果是[2] -->[0,0,1]\n",
    "如果不做平滑直接是这两者计算交叉信息熵\n",
    "如果做了平滑可以把真实结果修改为\n",
    "[0.1,0.1,0.8]\n",
    "计算他们的kl散度\"\"\"\n",
    "class LabelSmoothing(nn.Module):\n",
    "    \"Implement label smoothing.\"\n",
    "    \"\"\"\n",
    "    size:预测词典长度\n",
    "    smoothing:分给真实标签，1-smoothing，其余的smoothing/(n-1)\n",
    "    \"\"\"\n",
    "    def __init__(self, size, padding_idx, smoothing=0.0):\n",
    "        super(LabelSmoothing, self).__init__()\n",
    "        self.criterion = nn.KLDivLoss(size_average=False)\n",
    "        self.padding_idx = padding_idx\n",
    "        self.confidence = 1.0 - smoothing\n",
    "        self.smoothing = smoothing\n",
    "        self.size = size\n",
    "        self.true_dist = None\n",
    "\n",
    "    def forward(self, x, target):\n",
    "        assert x.size(1) == self.size\n",
    "        true_dist = x.data.clone()\n",
    "        true_dist.fill_(self.smoothing / (self.size - 2))\n",
    "        true_dist.scatter_(1, target.long().data.unsqueeze(1), self.confidence)\n",
    "        #scatter_(dim, index, src):\n",
    "        true_dist[:, self.padding_idx] = 0\n",
    "        mask = torch.nonzero(target.data == self.padding_idx)\n",
    "        if mask.dim() > 0:\n",
    "            true_dist.index_fill_(0, mask.squeeze(), 0.0)\n",
    "        self.true_dist = true_dist\n",
    "        return self.criterion(x, Variable(true_dist, requires_grad=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#计算损失\n",
    "class SimpleLossCompute:\n",
    "    \"A simple loss compute and train function.\"\n",
    "    def __init__(self, generator, criterion, opt=None):\n",
    "        self.generator = generator\n",
    "        self.criterion = criterion\n",
    "        self.opt = opt\n",
    "\n",
    "    def __call__(self, x, y, norm):\n",
    "        \"\"\"\n",
    "        norm 非pad的单词个数\n",
    "        \"\"\"\n",
    "        x = self.generator(x)\n",
    "        loss = self.criterion(x.contiguous().view(-1, x.size(-1)), y.contiguous().view(-1)) / norm\n",
    "        loss.backward()\n",
    "        if self.opt is not None:\n",
    "            self.opt.step()\n",
    "            self.opt.optimizer.zero_grad()\n",
    "        return loss.item() * norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "#单个训练\n",
    "def run_epoch(data_iter, model, loss_compute, epoch):\n",
    "    \"Standard Training and Logging Function\"\n",
    "    start = time.time()\n",
    "    total_tokens = 0\n",
    "    total_loss = 0\n",
    "    tokens = 0\n",
    "    for i, batch in enumerate(data_iter):\n",
    "        batch.src = torch.tensor(batch.src).cuda()\n",
    "        batch.trg = torch.tensor(batch.trg).cuda()\n",
    "        batch.src_mask = torch.tensor(batch.src_mask).cuda()\n",
    "        batch.trg_mask = torch.tensor(batch.trg_mask).cuda()\n",
    "        batch.trg_y = torch.tensor(batch.trg_y).cuda()\n",
    "        batch.ntokens = torch.tensor(batch.ntokens).cuda()\n",
    "        out = model.forward(batch.src, batch.trg, batch.src_mask, batch.trg_mask)\n",
    "        loss = loss_compute(out, batch.trg_y, batch.ntokens)\n",
    "        total_loss += loss\n",
    "        total_tokens += batch.ntokens\n",
    "        tokens += batch.ntokens\n",
    "        if i % 50 == 1:\n",
    "            elapsed = time.time() - start\n",
    "            print(\"Epoch Step: %d Loss: %f Tokens per Sec: %f\" %\n",
    "                    (epoch, loss / batch.ntokens, tokens / elapsed))\n",
    "            start = time.time()\n",
    "            tokens = 0\n",
    "    return total_loss / total_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def train(data, model, criterion, optimizer):\n",
    "    \"\"\"\n",
    "    训练并保存模型\n",
    "    \"\"\"\n",
    "    # 初始化模型在dev集上的最优Loss为一个较大值\n",
    "    best_dev_loss = 1e5\n",
    "    model.cuda()\n",
    "    #data.train_data.cuda()\n",
    "    #data.dev_data.cuda()\n",
    "    for epoch in range(EPOCHS):\n",
    "        # 模型训练\n",
    "        model.train()\n",
    "        #run_epoch(data.train_data, model, SimpleLossCompute(model.generator, criterion, optimizer), epoch)\n",
    "        run_epoch(data.train_data, model, SimpleLossCompute(model.generator, criterion, optimizer), epoch)\n",
    "        #\n",
    "        model.eval()\n",
    "\n",
    "        # 在dev集上进行loss评估\n",
    "        print('>>>>> Evaluate')\n",
    "        #dev_loss = run_epoch(data.dev_data, model, SimpleLossCompute(model.generator, criterion, None), epoch)\n",
    "        dev_loss = run_epoch(data.dev_data, model, SimpleLossCompute(model.generator, criterion, None), epoch)\n",
    "        print('<<<<< Evaluate loss: %f' % dev_loss)\n",
    "\n",
    "        # TODO: 如果当前epoch的模型在dev集上的loss优于之前记录的最优loss则保存当前模型，并更新最优loss值\n",
    "        if dev_loss < best_dev_loss:\n",
    "            torch.save(model.state_dict(), SAVE_FILE)\n",
    "            best_dev_loss = dev_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 初始化参数设置\n",
    "\n",
    "\n",
    "UNK = 0  # 未登录词的标识符对应的词典id\n",
    "PAD = 1  # padding占位符对应的词典id\n",
    "BATCH_SIZE = 32  # 每批次训练数据数量\n",
    "EPOCHS = 30  # 训练轮数\n",
    "LAYERS = 6  # transformer中堆叠的encoder和decoder block层数\n",
    "H_NUM = 8  # multihead attention hidden个数\n",
    "D_MODEL = 256  # embedding维数\n",
    "D_FF = 2048  # feed forward第一个全连接层维数\n",
    "DROPOUT = 0.1  # dropout比例\n",
    "MAX_LENGTH = 40  # 最大句子长度\n",
    "\n",
    "TRAIN_FILE = './data/training.txt'  # 训练集数据文件\n",
    "VAIL_FILE = \"./data/validation.txt\"  # 验证(开发)集数据文件\n",
    "SAVE_FILE = './save/model.pt'  # 模型保存路径(注意如当前目录无save文件夹需要自己创建)\n",
    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n",
    "\n",
    "def seq_padding(X, padding=0):\n",
    "    \"\"\"\n",
    "    对一个batch批次(以单词id表示)的数据进行padding填充对齐长度\n",
    "    \"\"\"\n",
    "    # 计算该批次数据各条数据句子长度\n",
    "    L = [len(x) for x in X]\n",
    "    # 获取该批次数据最大句子长度\n",
    "    ML = max(L)\n",
    "    # 对X中各条数据x进行遍历，如果长度短于该批次数据最大长度ML，则以padding id填充缺失长度ML-len(x)\n",
    "    return np.array([\n",
    "        np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X\n",
    "    ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from nltk import word_tokenize\n",
    "class PrepareData:\n",
    "    def __init__(self, train_file, vail_file):\n",
    "        # 读取数据 并分词\n",
    "        self.train_en, self.train_cn = self.load_data(train_file)\n",
    "        self.dev_en, self.dev_cn = self.load_data(vail_file)\n",
    "\n",
    "        # 构建单词表\n",
    "        self.en_word_dict, self.en_total_words, self.en_index_dict = self.build_dict(self.train_en)\n",
    "        self.cn_word_dict, self.cn_total_words, self.cn_index_dict = self.build_dict(self.train_cn)\n",
    "\n",
    "        # id化\n",
    "        self.train_en, self.train_cn = self.wordToID(self.train_en, self.train_cn, self.en_word_dict, self.cn_word_dict)\n",
    "        self.dev_en, self.dev_cn = self.wordToID(self.dev_en, self.dev_cn, self.en_word_dict, self.cn_word_dict)\n",
    "\n",
    "        # 划分batch + padding + mask\n",
    "        self.train_data = self.splitBatch(self.train_en, self.train_cn, BATCH_SIZE)\n",
    "        self.dev_data = self.splitBatch(self.dev_en, self.dev_cn, BATCH_SIZE)\n",
    "\n",
    "    def load_data(self, path):\n",
    "        \"\"\"\n",
    "        读取翻译前(英文)和翻译后(中文)的数据文件\n",
    "        每条数据都进行分词，然后构建成包含起始符(BOS)和终止符(EOS)的单词(中文为字符)列表\n",
    "        形式如：en = [['BOS', 'i', 'love', 'you', 'EOS'], ['BOS', 'me', 'too', 'EOS'], ...]\n",
    "                cn = [['BOS', '我', '爱', '你', 'EOS'], ['BOS', '我', '也', '是', 'EOS'], ...]\n",
    "        \"\"\"\n",
    "        en = []\n",
    "        cn = []\n",
    "        # TODO ...\n",
    "        with open(path, 'r', encoding='utf-8') as fin:\n",
    "            \n",
    "            for line in fin.readlines():\n",
    "                list_content = line.split('\\t')\n",
    "                # print(list_content)\n",
    "                en.append(['BOS'] + word_tokenize(list_content[0]) + ['EOS'])\n",
    "                cn.append(['BOS'] + word_tokenize(\" \".join(list_content[1])) + ['EOS'])\n",
    "                \n",
    "        return en, cn\n",
    "\n",
    "    def build_dict(self, sentences, max_words = 50000):\n",
    "        \"\"\"\n",
    "        传入load_data构造的分词后的列表数据\n",
    "        构建词典(key为单词，value为id值)\n",
    "        \"\"\"\n",
    "        # 对数据中所有单词进行计数\n",
    "        word_count = Counter()\n",
    "\n",
    "        for sentence in sentences:\n",
    "            for s in sentence:\n",
    "                word_count[s] += 1\n",
    "        # 只保留最高频的前max_words数的单词构建词典\n",
    "        # 并添加上UNK和PAD两个单词，对应id已经初始化设置过\n",
    "        ls = word_count.most_common(max_words)\n",
    "        # 统计词典的总词数\n",
    "        total_words = len(ls) + 2\n",
    "\n",
    "        word_dict = {w[0]: index + 2 for index, w in enumerate(ls)}\n",
    "        word_dict['UNK'] = UNK\n",
    "        word_dict['PAD'] = PAD\n",
    "        # 再构建一个反向的词典，供id转单词使用\n",
    "        index_dict = {v: k for k, v in word_dict.items()}\n",
    "\n",
    "        return word_dict, total_words, index_dict\n",
    "\n",
    "    def wordToID(self, en, cn, en_dict, cn_dict, sort=True):\n",
    "        \"\"\"\n",
    "        该方法可以将翻译前(英文)数据和翻译后(中文)数据的单词列表表示的数据\n",
    "        均转为id列表表示的数据\n",
    "        如果sort参数设置为True，则会以翻译前(英文)的句子(单词数)长度排序\n",
    "        以便后续分batch做padding时，同批次各句子需要padding的长度相近减少padding量\n",
    "        \"\"\"\n",
    "        # 计算英文数据条数\n",
    "        length = len(en)\n",
    "\n",
    "        # TODO: 将翻译前(英文)数据和翻译后(中文)数据都转换为id表示的形式\n",
    "        out_en_ids = [[en_dict.get(w, 0) for w in sent] for sent in en]\n",
    "        out_cn_ids = [[cn_dict.get(w, 0) for w in sent] for sent in cn]\n",
    "\n",
    "        # 构建一个按照句子长度排序的函数\n",
    "        def len_argsort(seq):\n",
    "            \"\"\"\n",
    "            传入一系列句子数据(分好词的列表形式)，\n",
    "            按照句子长度排序后，返回排序后原来各句子在数据中的索引下标\n",
    "            \"\"\"\n",
    "            return sorted(range(len(seq)), key=lambda x: len(seq[x]))\n",
    "\n",
    "        # 把中文和英文按照同样的顺序排序\n",
    "        if sort:\n",
    "            # 以英文句子长度排序的(句子下标)顺序为基准\n",
    "            sorted_index = len_argsort(out_en_ids)\n",
    "\n",
    "            # TODO: 对翻译前(英文)数据和翻译后(中文)数据都按此基准进行排序\n",
    "            out_en_ids = [out_en_ids[i] for i in sorted_index]\n",
    "            out_cn_ids = [out_cn_ids[i] for i in sorted_index]\n",
    "\n",
    "        return out_en_ids, out_cn_ids\n",
    "\n",
    "    def splitBatch(self, en, cn, batch_size, shuffle=True):\n",
    "        \"\"\"\n",
    "        将以单词id列表表示的翻译前(英文)数据和翻译后(中文)数据\n",
    "        按照指定的batch_size进行划分\n",
    "        如果shuffle参数为True，则会对这些batch数据顺序进行随机打乱\n",
    "        \"\"\"\n",
    "        # 在按数据长度生成的各条数据下标列表[0, 1, ..., len(en)-1]中\n",
    "        # 每隔指定长度(batch_size)取一个下标作为后续生成batch的起始下标\n",
    "        idx_list = np.arange(0, len(en), batch_size)\n",
    "        # 如果shuffle参数为True，则将这些各batch起始下标打乱\n",
    "        if shuffle:\n",
    "            np.random.shuffle(idx_list)\n",
    "        # 存放各个batch批次的句子数据索引下标\n",
    "        batch_indexs = []\n",
    "        for idx in idx_list:\n",
    "            # 注意，起始下标最大的那个batch可能会超出数据大小\n",
    "            # 因此要限定其终止下标不能超过数据大小\n",
    "\n",
    "            batch_indexs.append(np.arange(idx, min(idx + batch_size, len(en))))\n",
    "\n",
    "        # 按各batch批次的句子数据索引下标，构建实际的单词id列表表示的各batch句子数据\n",
    "        batches = []\n",
    "        for batch_index in batch_indexs:\n",
    "            # 按当前batch的各句子下标(数组批量索引)提取对应的单词id列表句子表示数据\n",
    "            batch_en = [en[index] for index in batch_index]\n",
    "            batch_cn = [cn[index] for index in batch_index]\n",
    "            # 对当前batch的各个句子都进行padding对齐长度\n",
    "            # 维度为：batch数量×batch_size×每个batch最大句子长度\n",
    "            batch_cn = seq_padding(batch_cn)\n",
    "            batch_en = seq_padding(batch_en)\n",
    "            # 将当前batch的英文和中文数据添加到存放所有batch数据的列表中\n",
    "            #\n",
    "            batches.append(Batch(torch.from_numpy(batch_en), torch.from_numpy(batch_cn)))\n",
    "\n",
    "        return batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "predata = PrepareData(TRAIN_FILE, VAIL_FILE)\n",
    "en,cn =predata.load_data(TRAIN_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['BOS', 'it', \"'s\", 'none', 'of', 'your', 'concern', '.', 'EOS']\n",
      "['BOS', '这', '不', '关', '你', '的', '事', '。', 'EOS']\n"
     ]
    }
   ],
   "source": [
    "print(en[0])\n",
    "print(cn[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 27, 13, 597, 10, 3]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predata.train_en[1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "was\n",
      "tom\n",
      "asleep\n",
      "?\n"
     ]
    }
   ],
   "source": [
    "print(predata.en_index_dict.get(27))\n",
    "print(predata.en_index_dict.get(13))\n",
    "print(predata.en_index_dict.get(597))\n",
    "print(predata.en_index_dict.get(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([   2,   52, 1133,   11,    6, 1301,   10,    3], dtype=torch.int32)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"*********\")\n",
    "predata.train_data[10].src[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "life\n",
      "is\n",
      "beautiful\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "print(predata.en_index_dict.get(225))\n",
    "print(predata.en_index_dict.get(11))\n",
    "print(predata.en_index_dict.get(248))\n",
    "print(predata.en_index_dict.get(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "src_vocab 5580\n",
      "tgt_vocab 2527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/3791962198.py:21: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_.\n",
      "  nn.init.xavier_uniform(p)\n"
     ]
    }
   ],
   "source": [
    "# 数据预处理\n",
    "data = PrepareData(TRAIN_FILE, VAIL_FILE)\n",
    "src_vocab = len(data.en_word_dict)\n",
    "tgt_vocab = len(data.cn_word_dict)\n",
    "print(\"src_vocab %d\" % src_vocab)\n",
    "print(\"tgt_vocab %d\" % tgt_vocab)\n",
    "\n",
    "# 初始化模型\n",
    "model = make_model(\n",
    "                    src_vocab,\n",
    "                    tgt_vocab,\n",
    "                    LAYERS,\n",
    "                    D_MODEL,\n",
    "                    D_FF,\n",
    "                    H_NUM,\n",
    "                    DROPOUT\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\miniconda3\\envs\\Python38\\lib\\site-packages\\torch\\nn\\_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>> start train\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/2948464230.py:12: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch.src = torch.tensor(batch.src).cuda()\n",
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/2948464230.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch.trg = torch.tensor(batch.trg).cuda()\n",
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/2948464230.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch.src_mask = torch.tensor(batch.src_mask).cuda()\n",
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/2948464230.py:15: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch.trg_mask = torch.tensor(batch.trg_mask).cuda()\n",
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/2948464230.py:16: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch.trg_y = torch.tensor(batch.trg_y).cuda()\n",
      "C:\\Users\\laodi\\AppData\\Local\\Temp/ipykernel_8920/2948464230.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  batch.ntokens = torch.tensor(batch.ntokens).cuda()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Step: 0 Loss: 7.916300 Tokens per Sec: 1112.715698\n",
      "Epoch Step: 0 Loss: 6.850322 Tokens per Sec: 3765.808105\n",
      "Epoch Step: 0 Loss: 6.199982 Tokens per Sec: 3649.751953\n",
      "Epoch Step: 0 Loss: 5.359284 Tokens per Sec: 3730.557861\n",
      "Epoch Step: 0 Loss: 4.546889 Tokens per Sec: 3597.563965\n",
      "Epoch Step: 0 Loss: 4.696634 Tokens per Sec: 3499.347168\n",
      "Epoch Step: 0 Loss: 4.127663 Tokens per Sec: 3584.738525\n",
      "Epoch Step: 0 Loss: 4.657183 Tokens per Sec: 3716.832275\n",
      "Epoch Step: 0 Loss: 4.270918 Tokens per Sec: 3574.641113\n",
      "Epoch Step: 0 Loss: 4.310666 Tokens per Sec: 3531.087646\n",
      "Epoch Step: 0 Loss: 4.376198 Tokens per Sec: 3624.461914\n",
      "Epoch Step: 0 Loss: 3.695674 Tokens per Sec: 3621.164307\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 0 Loss: 3.806527 Tokens per Sec: 4972.560547\n",
      "<<<<< Evaluate loss: 3.837944\n",
      "Epoch Step: 1 Loss: 4.211561 Tokens per Sec: 3801.089355\n",
      "Epoch Step: 1 Loss: 3.753890 Tokens per Sec: 3723.572266\n",
      "Epoch Step: 1 Loss: 3.672273 Tokens per Sec: 3608.841797\n",
      "Epoch Step: 1 Loss: 3.687582 Tokens per Sec: 3686.108887\n",
      "Epoch Step: 1 Loss: 3.186748 Tokens per Sec: 3532.196533\n",
      "Epoch Step: 1 Loss: 3.399110 Tokens per Sec: 3525.859619\n",
      "Epoch Step: 1 Loss: 2.972846 Tokens per Sec: 3605.849609\n",
      "Epoch Step: 1 Loss: 3.640291 Tokens per Sec: 3722.873291\n",
      "Epoch Step: 1 Loss: 3.105339 Tokens per Sec: 3607.247314\n",
      "Epoch Step: 1 Loss: 3.125976 Tokens per Sec: 3575.620117\n",
      "Epoch Step: 1 Loss: 3.417571 Tokens per Sec: 3658.618896\n",
      "Epoch Step: 1 Loss: 2.732367 Tokens per Sec: 3653.248535\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 1 Loss: 2.766627 Tokens per Sec: 5008.386230\n",
      "<<<<< Evaluate loss: 2.934088\n",
      "Epoch Step: 2 Loss: 3.373374 Tokens per Sec: 3871.402588\n",
      "Epoch Step: 2 Loss: 2.880805 Tokens per Sec: 3762.189209\n",
      "Epoch Step: 2 Loss: 2.885793 Tokens per Sec: 3669.136963\n",
      "Epoch Step: 2 Loss: 2.913428 Tokens per Sec: 3741.989502\n",
      "Epoch Step: 2 Loss: 2.616311 Tokens per Sec: 3599.393799\n",
      "Epoch Step: 2 Loss: 2.688684 Tokens per Sec: 3532.499023\n",
      "Epoch Step: 2 Loss: 2.244415 Tokens per Sec: 3594.281494\n",
      "Epoch Step: 2 Loss: 2.916976 Tokens per Sec: 3708.345459\n",
      "Epoch Step: 2 Loss: 2.432611 Tokens per Sec: 3606.149902\n",
      "Epoch Step: 2 Loss: 2.530537 Tokens per Sec: 3572.989502\n",
      "Epoch Step: 2 Loss: 2.988364 Tokens per Sec: 3663.144043\n",
      "Epoch Step: 2 Loss: 2.177401 Tokens per Sec: 3645.978027\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 2 Loss: 2.201595 Tokens per Sec: 5063.965820\n",
      "<<<<< Evaluate loss: 2.406515\n",
      "Epoch Step: 3 Loss: 2.936566 Tokens per Sec: 3878.763672\n",
      "Epoch Step: 3 Loss: 2.384123 Tokens per Sec: 3750.021240\n",
      "Epoch Step: 3 Loss: 2.271882 Tokens per Sec: 3636.100586\n",
      "Epoch Step: 3 Loss: 2.567088 Tokens per Sec: 3728.960205\n",
      "Epoch Step: 3 Loss: 2.172437 Tokens per Sec: 3579.147217\n",
      "Epoch Step: 3 Loss: 2.448194 Tokens per Sec: 3524.074951\n",
      "Epoch Step: 3 Loss: 1.858288 Tokens per Sec: 3584.874512\n",
      "Epoch Step: 3 Loss: 2.543489 Tokens per Sec: 3686.342529\n",
      "Epoch Step: 3 Loss: 2.135692 Tokens per Sec: 3593.981445\n",
      "Epoch Step: 3 Loss: 2.212458 Tokens per Sec: 3547.894043\n",
      "Epoch Step: 3 Loss: 2.702364 Tokens per Sec: 3639.449707\n",
      "Epoch Step: 3 Loss: 1.838471 Tokens per Sec: 3630.182861\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 3 Loss: 1.846370 Tokens per Sec: 4968.564941\n",
      "<<<<< Evaluate loss: 2.049555\n",
      "Epoch Step: 4 Loss: 2.590827 Tokens per Sec: 3821.169189\n",
      "Epoch Step: 4 Loss: 2.068911 Tokens per Sec: 3770.975342\n",
      "Epoch Step: 4 Loss: 1.981795 Tokens per Sec: 3661.687256\n",
      "Epoch Step: 4 Loss: 2.258283 Tokens per Sec: 3756.132812\n",
      "Epoch Step: 4 Loss: 1.816726 Tokens per Sec: 3584.062500\n",
      "Epoch Step: 4 Loss: 2.034517 Tokens per Sec: 3504.688965\n",
      "Epoch Step: 4 Loss: 1.496802 Tokens per Sec: 3574.731445\n",
      "Epoch Step: 4 Loss: 2.114559 Tokens per Sec: 3697.989746\n",
      "Epoch Step: 4 Loss: 1.768749 Tokens per Sec: 3588.446777\n",
      "Epoch Step: 4 Loss: 1.878029 Tokens per Sec: 3558.373291\n",
      "Epoch Step: 4 Loss: 2.428393 Tokens per Sec: 3636.972900\n",
      "Epoch Step: 4 Loss: 1.446239 Tokens per Sec: 3627.458252\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 4 Loss: 1.446238 Tokens per Sec: 5043.002930\n",
      "<<<<< Evaluate loss: 1.728554\n",
      "Epoch Step: 5 Loss: 2.208131 Tokens per Sec: 3838.732178\n",
      "Epoch Step: 5 Loss: 1.723847 Tokens per Sec: 3751.075684\n",
      "Epoch Step: 5 Loss: 1.706077 Tokens per Sec: 3644.274902\n",
      "Epoch Step: 5 Loss: 1.976987 Tokens per Sec: 3721.688232\n",
      "Epoch Step: 5 Loss: 1.537361 Tokens per Sec: 3574.578369\n",
      "Epoch Step: 5 Loss: 1.755641 Tokens per Sec: 3511.262939\n",
      "Epoch Step: 5 Loss: 1.290131 Tokens per Sec: 3571.714355\n",
      "Epoch Step: 5 Loss: 1.807915 Tokens per Sec: 3694.001709\n",
      "Epoch Step: 5 Loss: 1.665193 Tokens per Sec: 3584.867920\n",
      "Epoch Step: 5 Loss: 1.581023 Tokens per Sec: 3541.381592\n",
      "Epoch Step: 5 Loss: 2.127841 Tokens per Sec: 3638.475098\n",
      "Epoch Step: 5 Loss: 1.128475 Tokens per Sec: 3626.393799\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 5 Loss: 1.206129 Tokens per Sec: 5001.444824\n",
      "<<<<< Evaluate loss: 1.487639\n",
      "Epoch Step: 6 Loss: 1.979705 Tokens per Sec: 3873.282959\n",
      "Epoch Step: 6 Loss: 1.500282 Tokens per Sec: 3743.122314\n",
      "Epoch Step: 6 Loss: 1.451709 Tokens per Sec: 3639.234619\n",
      "Epoch Step: 6 Loss: 1.626397 Tokens per Sec: 3722.813232\n",
      "Epoch Step: 6 Loss: 1.307321 Tokens per Sec: 3574.053711\n",
      "Epoch Step: 6 Loss: 1.645736 Tokens per Sec: 3506.976074\n",
      "Epoch Step: 6 Loss: 1.080805 Tokens per Sec: 3569.221191\n",
      "Epoch Step: 6 Loss: 1.579181 Tokens per Sec: 3718.689453\n",
      "Epoch Step: 6 Loss: 1.389472 Tokens per Sec: 3605.716797\n",
      "Epoch Step: 6 Loss: 1.377890 Tokens per Sec: 3568.348633\n",
      "Epoch Step: 6 Loss: 1.976640 Tokens per Sec: 3638.781982\n",
      "Epoch Step: 6 Loss: 1.049360 Tokens per Sec: 3621.963867\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 6 Loss: 1.082873 Tokens per Sec: 4986.917480\n",
      "<<<<< Evaluate loss: 1.289222\n",
      "Epoch Step: 7 Loss: 1.744697 Tokens per Sec: 3841.241211\n",
      "Epoch Step: 7 Loss: 1.295328 Tokens per Sec: 3749.579590\n",
      "Epoch Step: 7 Loss: 1.311185 Tokens per Sec: 3631.098633\n",
      "Epoch Step: 7 Loss: 1.519469 Tokens per Sec: 3727.271240\n",
      "Epoch Step: 7 Loss: 1.075763 Tokens per Sec: 3573.507324\n",
      "Epoch Step: 7 Loss: 1.439586 Tokens per Sec: 3507.550049\n",
      "Epoch Step: 7 Loss: 0.880093 Tokens per Sec: 3572.928711\n",
      "Epoch Step: 7 Loss: 1.392691 Tokens per Sec: 3683.128662\n",
      "Epoch Step: 7 Loss: 1.306534 Tokens per Sec: 3581.026367\n",
      "Epoch Step: 7 Loss: 1.276873 Tokens per Sec: 3544.420654\n",
      "Epoch Step: 7 Loss: 1.865419 Tokens per Sec: 3633.414062\n",
      "Epoch Step: 7 Loss: 0.831080 Tokens per Sec: 3617.093750\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 7 Loss: 0.962042 Tokens per Sec: 4805.840332\n",
      "<<<<< Evaluate loss: 1.139322\n",
      "Epoch Step: 8 Loss: 1.547533 Tokens per Sec: 3833.167236\n",
      "Epoch Step: 8 Loss: 1.106925 Tokens per Sec: 3761.095215\n",
      "Epoch Step: 8 Loss: 1.113127 Tokens per Sec: 3656.364014\n",
      "Epoch Step: 8 Loss: 1.288819 Tokens per Sec: 3742.654297\n",
      "Epoch Step: 8 Loss: 0.835984 Tokens per Sec: 3598.139893\n",
      "Epoch Step: 8 Loss: 1.248488 Tokens per Sec: 3536.532471\n",
      "Epoch Step: 8 Loss: 0.838835 Tokens per Sec: 3601.052246\n",
      "Epoch Step: 8 Loss: 1.125023 Tokens per Sec: 3711.377197\n",
      "Epoch Step: 8 Loss: 1.065000 Tokens per Sec: 3606.070068\n",
      "Epoch Step: 8 Loss: 1.123767 Tokens per Sec: 3567.829102\n",
      "Epoch Step: 8 Loss: 1.662162 Tokens per Sec: 3654.965088\n",
      "Epoch Step: 8 Loss: 0.726159 Tokens per Sec: 3630.876953\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 8 Loss: 0.807342 Tokens per Sec: 5041.283691\n",
      "<<<<< Evaluate loss: 0.981080\n",
      "Epoch Step: 9 Loss: 1.445839 Tokens per Sec: 3865.356689\n",
      "Epoch Step: 9 Loss: 0.968391 Tokens per Sec: 3759.124268\n",
      "Epoch Step: 9 Loss: 0.998374 Tokens per Sec: 3656.776367\n",
      "Epoch Step: 9 Loss: 1.157540 Tokens per Sec: 3742.854736\n",
      "Epoch Step: 9 Loss: 0.702632 Tokens per Sec: 3588.668213\n",
      "Epoch Step: 9 Loss: 1.186265 Tokens per Sec: 3536.921631\n",
      "Epoch Step: 9 Loss: 0.652180 Tokens per Sec: 3599.880615\n",
      "Epoch Step: 9 Loss: 0.970052 Tokens per Sec: 3709.515137\n",
      "Epoch Step: 9 Loss: 1.053788 Tokens per Sec: 3596.962158\n",
      "Epoch Step: 9 Loss: 1.028622 Tokens per Sec: 3569.940430\n",
      "Epoch Step: 9 Loss: 1.443477 Tokens per Sec: 3667.018066\n",
      "Epoch Step: 9 Loss: 0.636663 Tokens per Sec: 3635.863281\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 9 Loss: 0.659722 Tokens per Sec: 5006.079590\n",
      "<<<<< Evaluate loss: 0.845478\n",
      "Epoch Step: 10 Loss: 1.218450 Tokens per Sec: 3911.747070\n",
      "Epoch Step: 10 Loss: 0.833399 Tokens per Sec: 3762.958252\n",
      "Epoch Step: 10 Loss: 0.872259 Tokens per Sec: 3655.055664\n",
      "Epoch Step: 10 Loss: 0.991402 Tokens per Sec: 3743.888184\n",
      "Epoch Step: 10 Loss: 0.598337 Tokens per Sec: 3581.749268\n",
      "Epoch Step: 10 Loss: 0.994663 Tokens per Sec: 3517.141113\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Step: 10 Loss: 0.547212 Tokens per Sec: 3596.836670\n",
      "Epoch Step: 10 Loss: 0.923535 Tokens per Sec: 3710.739502\n",
      "Epoch Step: 10 Loss: 0.845837 Tokens per Sec: 3599.687012\n",
      "Epoch Step: 10 Loss: 0.823272 Tokens per Sec: 3569.589600\n",
      "Epoch Step: 10 Loss: 1.357319 Tokens per Sec: 3658.924561\n",
      "Epoch Step: 10 Loss: 0.577303 Tokens per Sec: 3640.517090\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 10 Loss: 0.547276 Tokens per Sec: 5072.104492\n",
      "<<<<< Evaluate loss: 0.701767\n",
      "Epoch Step: 11 Loss: 1.246732 Tokens per Sec: 3852.861328\n",
      "Epoch Step: 11 Loss: 0.755828 Tokens per Sec: 3769.166016\n",
      "Epoch Step: 11 Loss: 0.768767 Tokens per Sec: 3661.545898\n",
      "Epoch Step: 11 Loss: 0.840176 Tokens per Sec: 3738.118896\n",
      "Epoch Step: 11 Loss: 0.554952 Tokens per Sec: 3588.847656\n",
      "Epoch Step: 11 Loss: 0.838894 Tokens per Sec: 3521.654541\n",
      "Epoch Step: 11 Loss: 0.454424 Tokens per Sec: 3623.396973\n",
      "Epoch Step: 11 Loss: 0.820111 Tokens per Sec: 3720.817627\n",
      "Epoch Step: 11 Loss: 0.768049 Tokens per Sec: 3602.748291\n",
      "Epoch Step: 11 Loss: 0.716298 Tokens per Sec: 3560.416748\n",
      "Epoch Step: 11 Loss: 1.269902 Tokens per Sec: 3655.244385\n",
      "Epoch Step: 11 Loss: 0.477760 Tokens per Sec: 3630.958984\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 11 Loss: 0.462678 Tokens per Sec: 5066.434570\n",
      "<<<<< Evaluate loss: 0.597394\n",
      "Epoch Step: 12 Loss: 0.942163 Tokens per Sec: 3873.508545\n",
      "Epoch Step: 12 Loss: 0.655567 Tokens per Sec: 3760.736328\n",
      "Epoch Step: 12 Loss: 0.692683 Tokens per Sec: 3659.166016\n",
      "Epoch Step: 12 Loss: 0.678960 Tokens per Sec: 3736.459229\n",
      "Epoch Step: 12 Loss: 0.453044 Tokens per Sec: 3585.257324\n",
      "Epoch Step: 12 Loss: 0.748114 Tokens per Sec: 3531.636475\n",
      "Epoch Step: 12 Loss: 0.444434 Tokens per Sec: 3578.612549\n",
      "Epoch Step: 12 Loss: 0.669528 Tokens per Sec: 3716.346924\n",
      "Epoch Step: 12 Loss: 0.585978 Tokens per Sec: 3611.818115\n",
      "Epoch Step: 12 Loss: 0.786217 Tokens per Sec: 3559.856201\n",
      "Epoch Step: 12 Loss: 1.184539 Tokens per Sec: 3653.444336\n",
      "Epoch Step: 12 Loss: 0.454730 Tokens per Sec: 3644.269775\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 12 Loss: 0.397481 Tokens per Sec: 5022.643555\n",
      "<<<<< Evaluate loss: 0.517709\n",
      "Epoch Step: 13 Loss: 0.851758 Tokens per Sec: 3854.845459\n",
      "Epoch Step: 13 Loss: 0.627942 Tokens per Sec: 3765.050293\n",
      "Epoch Step: 13 Loss: 0.632504 Tokens per Sec: 3657.989502\n",
      "Epoch Step: 13 Loss: 0.696077 Tokens per Sec: 3744.530762\n",
      "Epoch Step: 13 Loss: 0.416753 Tokens per Sec: 3587.745850\n",
      "Epoch Step: 13 Loss: 0.675526 Tokens per Sec: 3529.906250\n",
      "Epoch Step: 13 Loss: 0.345285 Tokens per Sec: 3593.795166\n",
      "Epoch Step: 13 Loss: 0.631868 Tokens per Sec: 3708.231689\n",
      "Epoch Step: 13 Loss: 0.555527 Tokens per Sec: 3614.919434\n",
      "Epoch Step: 13 Loss: 0.654844 Tokens per Sec: 3564.710205\n",
      "Epoch Step: 13 Loss: 1.003409 Tokens per Sec: 3655.942627\n",
      "Epoch Step: 13 Loss: 0.365354 Tokens per Sec: 3637.906738\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 13 Loss: 0.320542 Tokens per Sec: 4976.606445\n",
      "<<<<< Evaluate loss: 0.437382\n",
      "Epoch Step: 14 Loss: 0.875060 Tokens per Sec: 3870.179199\n",
      "Epoch Step: 14 Loss: 0.551922 Tokens per Sec: 3761.967773\n",
      "Epoch Step: 14 Loss: 0.569612 Tokens per Sec: 3660.136719\n",
      "Epoch Step: 14 Loss: 0.552842 Tokens per Sec: 3736.183105\n",
      "Epoch Step: 14 Loss: 0.367177 Tokens per Sec: 3583.175781\n",
      "Epoch Step: 14 Loss: 0.594553 Tokens per Sec: 3528.736572\n",
      "Epoch Step: 14 Loss: 0.290367 Tokens per Sec: 3582.742676\n",
      "Epoch Step: 14 Loss: 0.505968 Tokens per Sec: 3712.242676\n",
      "Epoch Step: 14 Loss: 0.514413 Tokens per Sec: 3603.219238\n",
      "Epoch Step: 14 Loss: 0.548591 Tokens per Sec: 3564.457275\n",
      "Epoch Step: 14 Loss: 0.935571 Tokens per Sec: 3657.238281\n",
      "Epoch Step: 14 Loss: 0.357621 Tokens per Sec: 3628.959717\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 14 Loss: 0.315614 Tokens per Sec: 4951.098145\n",
      "<<<<< Evaluate loss: 0.372165\n",
      "Epoch Step: 15 Loss: 0.756656 Tokens per Sec: 3887.011719\n",
      "Epoch Step: 15 Loss: 0.555630 Tokens per Sec: 3805.934814\n",
      "Epoch Step: 15 Loss: 0.459221 Tokens per Sec: 3669.019775\n",
      "Epoch Step: 15 Loss: 0.428391 Tokens per Sec: 3760.099854\n",
      "Epoch Step: 15 Loss: 0.284569 Tokens per Sec: 3567.843994\n",
      "Epoch Step: 15 Loss: 0.562709 Tokens per Sec: 3528.130859\n",
      "Epoch Step: 15 Loss: 0.304726 Tokens per Sec: 3599.099121\n",
      "Epoch Step: 15 Loss: 0.477549 Tokens per Sec: 3702.171387\n",
      "Epoch Step: 15 Loss: 0.475146 Tokens per Sec: 3582.170654\n",
      "Epoch Step: 15 Loss: 0.534531 Tokens per Sec: 3553.735840\n",
      "Epoch Step: 15 Loss: 0.878425 Tokens per Sec: 3643.786621\n",
      "Epoch Step: 15 Loss: 0.294523 Tokens per Sec: 3635.284668\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 15 Loss: 0.267282 Tokens per Sec: 5041.886719\n",
      "<<<<< Evaluate loss: 0.317379\n",
      "Epoch Step: 16 Loss: 0.691181 Tokens per Sec: 3861.476562\n",
      "Epoch Step: 16 Loss: 0.439124 Tokens per Sec: 3752.440674\n",
      "Epoch Step: 16 Loss: 0.389241 Tokens per Sec: 3646.773438\n",
      "Epoch Step: 16 Loss: 0.454532 Tokens per Sec: 3730.275146\n",
      "Epoch Step: 16 Loss: 0.267379 Tokens per Sec: 3585.681641\n",
      "Epoch Step: 16 Loss: 0.500666 Tokens per Sec: 3533.769775\n",
      "Epoch Step: 16 Loss: 0.251606 Tokens per Sec: 3590.815430\n",
      "Epoch Step: 16 Loss: 0.414819 Tokens per Sec: 3703.114014\n",
      "Epoch Step: 16 Loss: 0.438771 Tokens per Sec: 3600.508301\n",
      "Epoch Step: 16 Loss: 0.504917 Tokens per Sec: 3560.810547\n",
      "Epoch Step: 16 Loss: 0.724446 Tokens per Sec: 3651.050537\n",
      "Epoch Step: 16 Loss: 0.280775 Tokens per Sec: 3630.936523\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 16 Loss: 0.226322 Tokens per Sec: 4940.551270\n",
      "<<<<< Evaluate loss: 0.276531\n",
      "Epoch Step: 17 Loss: 0.617383 Tokens per Sec: 3787.214844\n",
      "Epoch Step: 17 Loss: 0.342433 Tokens per Sec: 3762.070312\n",
      "Epoch Step: 17 Loss: 0.372373 Tokens per Sec: 3650.135254\n",
      "Epoch Step: 17 Loss: 0.342552 Tokens per Sec: 3733.100586\n",
      "Epoch Step: 17 Loss: 0.229554 Tokens per Sec: 3587.238281\n",
      "Epoch Step: 17 Loss: 0.447600 Tokens per Sec: 3519.414307\n",
      "Epoch Step: 17 Loss: 0.231977 Tokens per Sec: 3590.730469\n",
      "Epoch Step: 17 Loss: 0.432946 Tokens per Sec: 3701.765381\n",
      "Epoch Step: 17 Loss: 0.354043 Tokens per Sec: 3599.042236\n",
      "Epoch Step: 17 Loss: 0.382793 Tokens per Sec: 3556.233154\n",
      "Epoch Step: 17 Loss: 0.753792 Tokens per Sec: 3652.736328\n",
      "Epoch Step: 17 Loss: 0.223186 Tokens per Sec: 3634.897949\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 17 Loss: 0.197998 Tokens per Sec: 5033.580078\n",
      "<<<<< Evaluate loss: 0.235848\n",
      "Epoch Step: 18 Loss: 0.576628 Tokens per Sec: 3862.904053\n",
      "Epoch Step: 18 Loss: 0.357438 Tokens per Sec: 3764.109863\n",
      "Epoch Step: 18 Loss: 0.342220 Tokens per Sec: 3656.947754\n",
      "Epoch Step: 18 Loss: 0.353062 Tokens per Sec: 3742.237305\n",
      "Epoch Step: 18 Loss: 0.186360 Tokens per Sec: 3582.921631\n",
      "Epoch Step: 18 Loss: 0.349060 Tokens per Sec: 3527.646729\n",
      "Epoch Step: 18 Loss: 0.285805 Tokens per Sec: 3582.617676\n",
      "Epoch Step: 18 Loss: 0.381555 Tokens per Sec: 3707.672852\n",
      "Epoch Step: 18 Loss: 0.302295 Tokens per Sec: 3591.804199\n",
      "Epoch Step: 18 Loss: 0.389941 Tokens per Sec: 3559.389893\n",
      "Epoch Step: 18 Loss: 0.542572 Tokens per Sec: 3647.250244\n",
      "Epoch Step: 18 Loss: 0.239037 Tokens per Sec: 3631.026855\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 18 Loss: 0.154075 Tokens per Sec: 5002.046875\n",
      "<<<<< Evaluate loss: 0.218445\n",
      "Epoch Step: 19 Loss: 0.516835 Tokens per Sec: 3850.075439\n",
      "Epoch Step: 19 Loss: 0.324070 Tokens per Sec: 3753.662354\n",
      "Epoch Step: 19 Loss: 0.347292 Tokens per Sec: 3644.760498\n",
      "Epoch Step: 19 Loss: 0.273748 Tokens per Sec: 3736.894775\n",
      "Epoch Step: 19 Loss: 0.155445 Tokens per Sec: 3582.258057\n",
      "Epoch Step: 19 Loss: 0.355140 Tokens per Sec: 3521.632568\n",
      "Epoch Step: 19 Loss: 0.210272 Tokens per Sec: 3590.491699\n",
      "Epoch Step: 19 Loss: 0.308598 Tokens per Sec: 3704.419922\n",
      "Epoch Step: 19 Loss: 0.349350 Tokens per Sec: 3599.373535\n",
      "Epoch Step: 19 Loss: 0.396999 Tokens per Sec: 3554.836182\n",
      "Epoch Step: 19 Loss: 0.518145 Tokens per Sec: 3647.095947\n",
      "Epoch Step: 19 Loss: 0.181226 Tokens per Sec: 3619.392822\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 19 Loss: 0.159093 Tokens per Sec: 5023.954102\n",
      "<<<<< Evaluate loss: 0.189328\n",
      "Epoch Step: 20 Loss: 0.481066 Tokens per Sec: 3856.267578\n",
      "Epoch Step: 20 Loss: 0.243805 Tokens per Sec: 3756.186768\n",
      "Epoch Step: 20 Loss: 0.238888 Tokens per Sec: 3642.569092\n",
      "Epoch Step: 20 Loss: 0.283835 Tokens per Sec: 3729.776855\n",
      "Epoch Step: 20 Loss: 0.219613 Tokens per Sec: 3592.463379\n",
      "Epoch Step: 20 Loss: 0.312789 Tokens per Sec: 3522.565918\n",
      "Epoch Step: 20 Loss: 0.192007 Tokens per Sec: 3587.991699\n",
      "Epoch Step: 20 Loss: 0.270803 Tokens per Sec: 3708.830322\n",
      "Epoch Step: 20 Loss: 0.295812 Tokens per Sec: 3591.210938\n",
      "Epoch Step: 20 Loss: 0.285904 Tokens per Sec: 3551.645996\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Step: 20 Loss: 0.460608 Tokens per Sec: 3649.641113\n",
      "Epoch Step: 20 Loss: 0.178993 Tokens per Sec: 3623.052490\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 20 Loss: 0.136638 Tokens per Sec: 5012.429199\n",
      "<<<<< Evaluate loss: 0.170570\n",
      "Epoch Step: 21 Loss: 0.483931 Tokens per Sec: 3821.250488\n",
      "Epoch Step: 21 Loss: 0.267406 Tokens per Sec: 3752.152100\n",
      "Epoch Step: 21 Loss: 0.250503 Tokens per Sec: 3646.004150\n",
      "Epoch Step: 21 Loss: 0.306363 Tokens per Sec: 3729.448730\n",
      "Epoch Step: 21 Loss: 0.149371 Tokens per Sec: 3576.106201\n",
      "Epoch Step: 21 Loss: 0.266933 Tokens per Sec: 3517.546143\n",
      "Epoch Step: 21 Loss: 0.167823 Tokens per Sec: 3587.124268\n",
      "Epoch Step: 21 Loss: 0.300337 Tokens per Sec: 3700.859131\n",
      "Epoch Step: 21 Loss: 0.262505 Tokens per Sec: 3603.251953\n",
      "Epoch Step: 21 Loss: 0.244440 Tokens per Sec: 3553.473145\n",
      "Epoch Step: 21 Loss: 0.441986 Tokens per Sec: 3657.662598\n",
      "Epoch Step: 21 Loss: 0.159592 Tokens per Sec: 3640.478027\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 21 Loss: 0.111908 Tokens per Sec: 5017.434570\n",
      "<<<<< Evaluate loss: 0.157509\n",
      "Epoch Step: 22 Loss: 0.364918 Tokens per Sec: 3847.724121\n",
      "Epoch Step: 22 Loss: 0.236768 Tokens per Sec: 3760.429932\n",
      "Epoch Step: 22 Loss: 0.279018 Tokens per Sec: 3641.094482\n",
      "Epoch Step: 22 Loss: 0.268455 Tokens per Sec: 3732.921631\n",
      "Epoch Step: 22 Loss: 0.157582 Tokens per Sec: 3584.382324\n",
      "Epoch Step: 22 Loss: 0.266914 Tokens per Sec: 3528.033691\n",
      "Epoch Step: 22 Loss: 0.154487 Tokens per Sec: 3593.885742\n",
      "Epoch Step: 22 Loss: 0.245211 Tokens per Sec: 3702.934326\n",
      "Epoch Step: 22 Loss: 0.240332 Tokens per Sec: 3590.698242\n",
      "Epoch Step: 22 Loss: 0.267480 Tokens per Sec: 3564.431885\n",
      "Epoch Step: 22 Loss: 0.371846 Tokens per Sec: 3648.698730\n",
      "Epoch Step: 22 Loss: 0.135443 Tokens per Sec: 3639.419189\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 22 Loss: 0.106880 Tokens per Sec: 5014.321289\n",
      "<<<<< Evaluate loss: 0.139024\n",
      "Epoch Step: 23 Loss: 0.393818 Tokens per Sec: 3835.020508\n",
      "Epoch Step: 23 Loss: 0.251954 Tokens per Sec: 3761.751221\n",
      "Epoch Step: 23 Loss: 0.184623 Tokens per Sec: 3654.068604\n",
      "Epoch Step: 23 Loss: 0.140115 Tokens per Sec: 3736.680664\n",
      "Epoch Step: 23 Loss: 0.141995 Tokens per Sec: 3585.927002\n",
      "Epoch Step: 23 Loss: 0.259978 Tokens per Sec: 3519.525879\n",
      "Epoch Step: 23 Loss: 0.132132 Tokens per Sec: 3584.879639\n",
      "Epoch Step: 23 Loss: 0.257284 Tokens per Sec: 3705.768799\n",
      "Epoch Step: 23 Loss: 0.237031 Tokens per Sec: 3594.496582\n",
      "Epoch Step: 23 Loss: 0.259072 Tokens per Sec: 3563.610596\n",
      "Epoch Step: 23 Loss: 0.366575 Tokens per Sec: 3653.512939\n",
      "Epoch Step: 23 Loss: 0.141267 Tokens per Sec: 3623.650391\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 23 Loss: 0.072172 Tokens per Sec: 5007.608398\n",
      "<<<<< Evaluate loss: 0.124021\n",
      "Epoch Step: 24 Loss: 0.309896 Tokens per Sec: 3704.363770\n",
      "Epoch Step: 24 Loss: 0.279109 Tokens per Sec: 3756.364502\n",
      "Epoch Step: 24 Loss: 0.248188 Tokens per Sec: 3637.076416\n",
      "Epoch Step: 24 Loss: 0.135702 Tokens per Sec: 3735.730225\n",
      "Epoch Step: 24 Loss: 0.159047 Tokens per Sec: 3587.527100\n",
      "Epoch Step: 24 Loss: 0.290000 Tokens per Sec: 3533.736572\n",
      "Epoch Step: 24 Loss: 0.097044 Tokens per Sec: 3592.323486\n",
      "Epoch Step: 24 Loss: 0.236332 Tokens per Sec: 3715.219238\n",
      "Epoch Step: 24 Loss: 0.240038 Tokens per Sec: 3610.431641\n",
      "Epoch Step: 24 Loss: 0.252632 Tokens per Sec: 3568.535156\n",
      "Epoch Step: 24 Loss: 0.320986 Tokens per Sec: 3651.012939\n",
      "Epoch Step: 24 Loss: 0.117683 Tokens per Sec: 3638.892822\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 24 Loss: 0.091924 Tokens per Sec: 5028.004395\n",
      "<<<<< Evaluate loss: 0.114500\n",
      "Epoch Step: 25 Loss: 0.312030 Tokens per Sec: 3880.356689\n",
      "Epoch Step: 25 Loss: 0.260491 Tokens per Sec: 3760.404053\n",
      "Epoch Step: 25 Loss: 0.212896 Tokens per Sec: 3654.157959\n",
      "Epoch Step: 25 Loss: 0.235347 Tokens per Sec: 3739.707764\n",
      "Epoch Step: 25 Loss: 0.137601 Tokens per Sec: 3592.136230\n",
      "Epoch Step: 25 Loss: 0.237783 Tokens per Sec: 3525.253174\n",
      "Epoch Step: 25 Loss: 0.087804 Tokens per Sec: 3600.950684\n",
      "Epoch Step: 25 Loss: 0.222620 Tokens per Sec: 3708.926025\n",
      "Epoch Step: 25 Loss: 0.224339 Tokens per Sec: 3613.500000\n",
      "Epoch Step: 25 Loss: 0.179191 Tokens per Sec: 3566.341553\n",
      "Epoch Step: 25 Loss: 0.350931 Tokens per Sec: 3656.360352\n",
      "Epoch Step: 25 Loss: 0.187397 Tokens per Sec: 3641.630127\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 25 Loss: 0.086518 Tokens per Sec: 5015.031250\n",
      "<<<<< Evaluate loss: 0.108066\n",
      "Epoch Step: 26 Loss: 0.266936 Tokens per Sec: 3867.622314\n",
      "Epoch Step: 26 Loss: 0.250881 Tokens per Sec: 3760.038818\n",
      "Epoch Step: 26 Loss: 0.200778 Tokens per Sec: 3659.744873\n",
      "Epoch Step: 26 Loss: 0.155222 Tokens per Sec: 3743.183105\n",
      "Epoch Step: 26 Loss: 0.107912 Tokens per Sec: 3589.011475\n",
      "Epoch Step: 26 Loss: 0.160663 Tokens per Sec: 3532.898682\n",
      "Epoch Step: 26 Loss: 0.147476 Tokens per Sec: 3592.293457\n",
      "Epoch Step: 26 Loss: 0.249806 Tokens per Sec: 3709.092041\n",
      "Epoch Step: 26 Loss: 0.206644 Tokens per Sec: 3602.265869\n",
      "Epoch Step: 26 Loss: 0.200555 Tokens per Sec: 3571.860352\n",
      "Epoch Step: 26 Loss: 0.311004 Tokens per Sec: 3654.232910\n",
      "Epoch Step: 26 Loss: 0.148953 Tokens per Sec: 3639.365967\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 26 Loss: 0.101003 Tokens per Sec: 4995.175293\n",
      "<<<<< Evaluate loss: 0.097669\n",
      "Epoch Step: 27 Loss: 0.278151 Tokens per Sec: 3870.218018\n",
      "Epoch Step: 27 Loss: 0.154932 Tokens per Sec: 3757.748291\n",
      "Epoch Step: 27 Loss: 0.208186 Tokens per Sec: 3655.835449\n",
      "Epoch Step: 27 Loss: 0.155345 Tokens per Sec: 3738.997070\n",
      "Epoch Step: 27 Loss: 0.087338 Tokens per Sec: 3582.513672\n",
      "Epoch Step: 27 Loss: 0.173955 Tokens per Sec: 3533.379639\n",
      "Epoch Step: 27 Loss: 0.164979 Tokens per Sec: 3595.681641\n",
      "Epoch Step: 27 Loss: 0.198661 Tokens per Sec: 3701.356445\n",
      "Epoch Step: 27 Loss: 0.158122 Tokens per Sec: 3605.046387\n",
      "Epoch Step: 27 Loss: 0.183034 Tokens per Sec: 3565.723633\n",
      "Epoch Step: 27 Loss: 0.255584 Tokens per Sec: 3654.234375\n",
      "Epoch Step: 27 Loss: 0.138356 Tokens per Sec: 3638.297607\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 27 Loss: 0.105505 Tokens per Sec: 5034.145020\n",
      "<<<<< Evaluate loss: 0.101640\n",
      "Epoch Step: 28 Loss: 0.254179 Tokens per Sec: 3853.021484\n",
      "Epoch Step: 28 Loss: 0.190062 Tokens per Sec: 3761.080322\n",
      "Epoch Step: 28 Loss: 0.127678 Tokens per Sec: 3654.014648\n",
      "Epoch Step: 28 Loss: 0.156019 Tokens per Sec: 3744.723877\n",
      "Epoch Step: 28 Loss: 0.123606 Tokens per Sec: 3593.110596\n",
      "Epoch Step: 28 Loss: 0.172883 Tokens per Sec: 3529.966797\n",
      "Epoch Step: 28 Loss: 0.111423 Tokens per Sec: 3593.549316\n",
      "Epoch Step: 28 Loss: 0.260640 Tokens per Sec: 3711.776367\n",
      "Epoch Step: 28 Loss: 0.177889 Tokens per Sec: 3606.499756\n",
      "Epoch Step: 28 Loss: 0.179822 Tokens per Sec: 3565.748047\n",
      "Epoch Step: 28 Loss: 0.254978 Tokens per Sec: 3656.618652\n",
      "Epoch Step: 28 Loss: 0.090066 Tokens per Sec: 3644.895020\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 28 Loss: 0.087630 Tokens per Sec: 4986.085449\n",
      "<<<<< Evaluate loss: 0.090701\n",
      "Epoch Step: 29 Loss: 0.259900 Tokens per Sec: 3830.515137\n",
      "Epoch Step: 29 Loss: 0.204408 Tokens per Sec: 3761.067871\n",
      "Epoch Step: 29 Loss: 0.163633 Tokens per Sec: 3656.053467\n",
      "Epoch Step: 29 Loss: 0.146339 Tokens per Sec: 3739.343262\n",
      "Epoch Step: 29 Loss: 0.126978 Tokens per Sec: 3592.072510\n",
      "Epoch Step: 29 Loss: 0.180977 Tokens per Sec: 3508.135010\n",
      "Epoch Step: 29 Loss: 0.066689 Tokens per Sec: 3597.093018\n",
      "Epoch Step: 29 Loss: 0.131761 Tokens per Sec: 3712.077637\n",
      "Epoch Step: 29 Loss: 0.177288 Tokens per Sec: 3608.344238\n",
      "Epoch Step: 29 Loss: 0.158162 Tokens per Sec: 3566.055420\n",
      "Epoch Step: 29 Loss: 0.234900 Tokens per Sec: 3658.029297\n",
      "Epoch Step: 29 Loss: 0.120255 Tokens per Sec: 3631.984131\n",
      ">>>>> Evaluate\n",
      "Epoch Step: 29 Loss: 0.067237 Tokens per Sec: 5027.775391\n",
      "<<<<< Evaluate loss: 0.086276\n",
      "<<<<<<< finished train, cost 1633.7692 seconds\n"
     ]
    }
   ],
   "source": [
    "# 训练\n",
    "print(\">>>>>>> start train\")\n",
    "train_start = time.time()\n",
    "criterion = LabelSmoothing(tgt_vocab, padding_idx = 0, smoothing= 0.0)\n",
    "optimizer = NoamOpt(D_MODEL, 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9,0.98), eps=1e-9))\n",
    "\n",
    "train(data, model.cuda(), criterion.cuda(), optimizer)\n",
    "print(f\"<<<<<<< finished train, cost {time.time()-train_start:.4f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def greedy_decode(model, src, src_mask, max_len, start_symbol):\n",
    "    \"\"\"\n",
    "    传入一个训练好的模型，对指定数据进行预测\n",
    "    \"\"\"\n",
    "    # 先用encoder进行encode\n",
    "    memory = model.encode(src, src_mask)\n",
    "    # 初始化预测内容为1×1的tensor，填入开始符('BOS')的id，并将type设置为输入数据类型(LongTensor)\n",
    "    ys = torch.ones(1, 1).fill_(start_symbol).type_as(src.data)\n",
    "    # 遍历输出的长度下标\n",
    "    for i in range(max_len-1):\n",
    "        # decode得到隐层表示\n",
    "        out = model.decode(memory,\n",
    "                           src_mask,\n",
    "                           Variable(ys),\n",
    "                           Variable(subsequent_mask(ys.size(1)).type_as(src.data)))\n",
    "        # 将隐藏表示转为对词典各词的log_softmax概率分布表示\n",
    "        prob = model.generator(out[:, -1])\n",
    "        # 获取当前位置最大概率的预测词id\n",
    "        _, next_word = torch.max(prob, dim = 1)\n",
    "        next_word = next_word.data[0]\n",
    "        # 将当前位置预测的字符id与之前的预测内容拼接起来\n",
    "        ys = torch.cat([ys,\n",
    "                        torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)\n",
    "    return ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(data, model):\n",
    "    \"\"\"\n",
    "    在data上用训练好的模型进行预测，打印模型翻译结果\n",
    "    \"\"\"\n",
    "    # 梯度清零\n",
    "    with torch.no_grad():\n",
    "        # 在data的英文数据长度上遍历下标\n",
    "        for i in range(len(data.dev_en)):\n",
    "            # TODO: 打印待翻译的英文句子\n",
    "            en_sent = \" \".join([data.en_index_dict[w] for w in  data.dev_en[i]])\n",
    "            print(\"\\n\" + en_sent)\n",
    "\n",
    "            # TODO: 打印对应的中文句子答案\n",
    "            cn_sent = \" \".join([data.cn_index_dict[w] for w in  data.dev_cn[i]])\n",
    "            print(\"\".join(cn_sent))\n",
    "\n",
    "            # 将当前以单词id表示的英文句子数据转为tensor，并放如DEVICE中\n",
    "            src = torch.from_numpy(np.array(data.dev_en[i])).long().to(DEVICE)\n",
    "            # 增加一维\n",
    "            src = src.unsqueeze(0)\n",
    "            # 设置attention mask\n",
    "            src_mask = (src != 0).unsqueeze(-2)\n",
    "            # 用训练好的模型进行decode预测\n",
    "            out = greedy_decode(model, src, src_mask, max_len=MAX_LENGTH, start_symbol=data.cn_word_dict[\"BOS\"])\n",
    "            # 初始化一个用于存放模型翻译结果句子单词的列表\n",
    "            translation = []\n",
    "            # 遍历翻译输出字符的下标（注意：开始符\"BOS\"的索引0不遍历）\n",
    "            for j in range(1, out.size(1)):\n",
    "                # 获取当前下标的输出字符\n",
    "                sym = data.cn_index_dict[out[0, j].item()]\n",
    "                # 如果输出字符不为'EOS'终止符，则添加到当前句子的翻译结果列表\n",
    "                if sym != 'EOS':\n",
    "                    translation.append(sym)\n",
    "                # 否则终止遍历\n",
    "                else:\n",
    "                    break\n",
    "            # 打印模型翻译输出的中文句子结果\n",
    "            print(\"translation: %s\" % \" \".join(translation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>> start evaluate\n",
      "\n",
      "BOS see above . EOS\n",
      "BOS 参 见 上 文 。 EOS\n",
      "translation: 参 见 上 文 。\n",
      "\n",
      "BOS join us . EOS\n",
      "BOS 来 加 入 我 们 吧 。 EOS\n",
      "translation: 来 加 入 我 们 吧 。\n",
      "\n",
      "BOS she disappeared . EOS\n",
      "BOS 她 消 失 了 。 EOS\n",
      "translation: 她 消 失 了 。\n",
      "\n",
      "BOS wood burns . EOS\n",
      "BOS 木 材 燃 烧 。 EOS\n",
      "translation: 木 材 燃 烧 。\n",
      "\n",
      "BOS we know . EOS\n",
      "BOS 我 们 知 道 。 EOS\n",
      "translation: 我 们 知 道 。\n",
      "\n",
      "BOS tom UNK . EOS\n",
      "BOS 汤 姆 皱 著 UNK 头 。 EOS\n",
      "translation: 汤 姆 皱 著 急 。\n",
      "\n",
      "BOS he is tall . EOS\n",
      "BOS 他 高 。 EOS\n",
      "translation: 他 高 。\n",
      "\n",
      "BOS boil the water . EOS\n",
      "BOS 把 水 烧 开 。 EOS\n",
      "translation: 把 水 烧 开 。\n",
      "\n",
      "BOS call the police ! EOS\n",
      "BOS 叫 警 察 ！ EOS\n",
      "translation: 报 警 ！\n",
      "\n",
      "BOS i began running . EOS\n",
      "BOS 我 开 始 跑 。 EOS\n",
      "translation: 我 开 始 跑 。\n",
      "\n",
      "BOS do n't worry . EOS\n",
      "BOS 别 担 心 。 EOS\n",
      "translation: 别 为 担 心 。\n",
      "\n",
      "BOS take me home . EOS\n",
      "BOS 带 我 回 家 。 EOS\n",
      "translation: 带 我 回 家 。\n",
      "\n",
      "BOS i admire you . EOS\n",
      "BOS 我 欣 赏 你 。 EOS\n",
      "translation: 我 欣 赏 你 。\n",
      "\n",
      "BOS tom loves flowers . EOS\n",
      "BOS 汤 姆 热 爱 花 UNK 。 EOS\n",
      "translation: 汤 姆 热 爱 花 。\n",
      "\n",
      "BOS i am coming . EOS\n",
      "BOS 我 来 了 。 EOS\n",
      "translation: 我 来 了 。\n",
      "\n",
      "BOS i like potatoes . EOS\n",
      "BOS 我 喜 欢 土 豆 。 EOS\n",
      "translation: 我 喜 欢 土 豆 。\n",
      "\n",
      "BOS break it up ! EOS\n",
      "BOS 停 手 ！ EOS\n",
      "translation: 停 手 ！\n",
      "\n",
      "BOS listen to this . EOS\n",
      "BOS 听 听 这 个 。 EOS\n",
      "translation: 听 听 听 听 听 这 个 。\n",
      "\n",
      "BOS i hate chemistry . EOS\n",
      "BOS 我 讨 厌 化 学 。 EOS\n",
      "translation: 我 讨 厌 化 学 。\n",
      "\n",
      "BOS tom will go . EOS\n",
      "BOS 汤 姆 要 走 。 EOS\n",
      "translation: 汤 姆 要 走 。\n",
      "\n",
      "BOS it makes sense . EOS\n",
      "BOS 那 样 说 得 通 。 EOS\n",
      "translation: 那 是 有 道 理 的 。\n",
      "\n",
      "BOS they sell candy . EOS\n",
      "BOS 他 们 卖 糖 果 。 EOS\n",
      "translation: 他 们 卖 糖 果 。\n",
      "\n",
      "BOS breakfast is ready . EOS\n",
      "BOS 早 饭 准 备 好 了 。 EOS\n",
      "translation: 早 饭 在 变 准 备 好 了 。\n",
      "\n",
      "BOS i 'm old . EOS\n",
      "BOS 我 老 了 。 EOS\n",
      "translation: 我 老 了 。\n",
      "\n",
      "BOS UNK loves you . EOS\n",
      "BOS 耶 UNK 爱 你 。 EOS\n",
      "translation: 耶 爱 你 。\n",
      "\n",
      "BOS tom UNK from UNK . EOS\n",
      "BOS 汤 姆 投 票 弃 权 。 EOS\n",
      "translation: 汤 姆 投 票 弃 权 。\n",
      "\n",
      "BOS you 're making progress . EOS\n",
      "BOS 你 在 进 步 。 EOS\n",
      "translation: 你 在 进 步 。\n",
      "\n",
      "BOS tom waved his hand . EOS\n",
      "BOS 汤 姆 挥 手 了 。 EOS\n",
      "translation: 汤 姆 挥 手 了 。\n",
      "\n",
      "BOS is this your car ? EOS\n",
      "BOS 这 是 你 的 车 吗 ? EOS\n",
      "translation: 这 是 你 的 车 吗 ?\n",
      "\n",
      "BOS i support political UNK . EOS\n",
      "BOS 我 支 持 政 治 改 革 。 EOS\n",
      "translation: 我 支 持 政 治 改 革 。\n",
      "\n",
      "BOS this UNK smells nice . EOS\n",
      "BOS 这 只 梨 闻 上 去 很 香 。 EOS\n",
      "translation: 这 只 梨 闻 上 去 很 香 。\n",
      "\n",
      "BOS do you need help ? EOS\n",
      "BOS 你 需 要 帮 助 吗 ？ EOS\n",
      "translation: 你 需 要 帮 助 吗 ？\n",
      "\n",
      "BOS fresh food is wonderful . EOS\n",
      "BOS 新 鲜 的 食 物 最 棒 了 。 EOS\n",
      "translation: 新 鲜 的 食 物 最 棒 了 。\n",
      "\n",
      "BOS what did you answer ? EOS\n",
      "BOS 你 回 答 了 什 么 ？ EOS\n",
      "translation: 你 回 答 了 什 么 ？\n",
      "\n",
      "BOS orange juice , please . EOS\n",
      "BOS 柳 橙 汁 ， 麻 烦 你 。 EOS\n",
      "translation: 柳 橙 汁 ， 麻 烦 你 。\n",
      "\n",
      "BOS my head is UNK . EOS\n",
      "BOS 我 的 头 要 炸 了 。 EOS\n",
      "translation: 我 的 头 要 炸 了 。\n",
      "\n",
      "BOS he cured my illness . EOS\n",
      "BOS 我 的 病 给 他 治 好 了 。 EOS\n",
      "translation: 他 的 病 给 我 的 风 险 。\n",
      "\n",
      "BOS the man finally confessed . EOS\n",
      "BOS 那 个 男 人 最 终 供 认 了 他 的 罪 行 。 EOS\n",
      "translation: 那 个 男 人 最 终 供 认 了 他 的 罪 行 。\n",
      "\n",
      "BOS do n't UNK me . EOS\n",
      "BOS 别 来 烦 我 。 EOS\n",
      "translation: 别 来 烦 我 。\n",
      "\n",
      "BOS what 's with you ? EOS\n",
      "BOS 你 怎 么 了 ？ EOS\n",
      "translation: 你 怎 么 了 ？\n",
      "\n",
      "BOS playing tennis is fun . EOS\n",
      "BOS 打 网 球 很 有 趣 。 EOS\n",
      "translation: 打 网 球 很 有 趣 。\n",
      "\n",
      "BOS i want your opinion . EOS\n",
      "BOS 我 想 要 听 听 你 的 意 见 。 EOS\n",
      "translation: 我 想 要 听 你 的 意 见 。\n",
      "\n",
      "BOS have you all eaten ? EOS\n",
      "BOS 你 们 吃 饭 了 吗 ？ EOS\n",
      "translation: 你 们 吃 饭 了 吗 ？\n",
      "\n",
      "BOS i often eat chicken . EOS\n",
      "BOS 我 经 常 吃 鸡 。 EOS\n",
      "translation: 我 经 常 吃 鸡 。\n",
      "\n",
      "BOS i like your car . EOS\n",
      "BOS 我 喜 欢 您 的 车 。 EOS\n",
      "translation: 我 喜 欢 您 的 车 。\n",
      "\n",
      "BOS tom is really handsome . EOS\n",
      "BOS 汤 姆 真 帅 。 EOS\n",
      "translation: 汤 姆 真 帅 。\n",
      "\n",
      "BOS the picture is nice . EOS\n",
      "BOS 这 画 不 错 。 EOS\n",
      "translation: 这 画 不 错 。\n",
      "\n",
      "BOS take care of yourself . EOS\n",
      "BOS 照 顾 好 自 己 。 EOS\n",
      "translation: 照 顾 好 自 己 。\n",
      "\n",
      "BOS can you freeze it ? EOS\n",
      "BOS 你 能 冷 冻 它 吗 ？ EOS\n",
      "translation: 你 能 冷 冻 它 吗 ？\n",
      "\n",
      "BOS i 've seen it . EOS\n",
      "BOS 我 见 过 。 EOS\n",
      "translation: 我 见 过 。\n",
      "\n",
      "BOS he plays baseball tomorrow . EOS\n",
      "BOS 他 明 天 将 打 棒 球 。 EOS\n",
      "translation: 他 明 天 将 打 棒 球 。\n",
      "\n",
      "BOS do you love music ? EOS\n",
      "BOS 你 爱 音 乐 吗 ？ EOS\n",
      "translation: 你 爱 音 乐 吗 ？\n",
      "\n",
      "BOS he greeted the lady . EOS\n",
      "BOS 他 向 那 位 女 士 问 好 。 EOS\n",
      "translation: 他 向 那 位 女 士 问 好 。\n",
      "\n",
      "BOS i know her address . EOS\n",
      "BOS 我 知 道 她 的 地 址 。 EOS\n",
      "translation: 我 知 道 她 的 地 址 。\n",
      "\n",
      "BOS three UNK is UNK . EOS\n",
      "BOS 三 的 立 方 是 二 十 七 。 EOS\n",
      "translation: 三 的 立 方 是 二 十 七 。\n",
      "\n",
      "BOS this is quite good . EOS\n",
      "BOS 这 可 真 好 。 EOS\n",
      "translation: 这 可 真 好 。\n",
      "\n",
      "BOS i have two cars . EOS\n",
      "BOS 我 有 两 辆 车 。 EOS\n",
      "translation: 我 有 两 辆 车 。\n",
      "\n",
      "BOS sorry to be late . EOS\n",
      "BOS 对 不 起 我 来 晚 了 。 EOS\n",
      "translation: 对 不 起 我 来 晚 了 。\n",
      "\n",
      "BOS the boy eats bread . EOS\n",
      "BOS 这 个 男 孩 吃 面 包 。 EOS\n",
      "translation: 这 个 男 孩 吃 了 面 包 。\n",
      "\n",
      "BOS we UNK the enemy . EOS\n",
      "BOS 我 们 战 胜 了 敌 人 。 EOS\n",
      "translation: 我 们 战 胜 了 敌 人 。\n",
      "\n",
      "BOS where 's my father ? EOS\n",
      "BOS 我 父 亲 在 哪 里 ？ EOS\n",
      "translation: 我 父 亲 在 哪 里 ？\n",
      "\n",
      "BOS it 's so hard . EOS\n",
      "BOS 太 难 了 。 EOS\n",
      "translation: 太 难 了 。\n",
      "\n",
      "BOS i 'm free tonight . EOS\n",
      "BOS 我 今 晚 有 空 。 EOS\n",
      "translation: 我 今 晚 有 空 。\n",
      "\n",
      "BOS be nice to others . EOS\n",
      "BOS 对 他 人 要 友 善 。 EOS\n",
      "translation: 对 他 人 要 友 好 。\n",
      "\n",
      "BOS i like your room . EOS\n",
      "BOS 我 喜 欢 你 的 房 间 。 EOS\n",
      "translation: 我 喜 欢 你 的 房 间 。\n",
      "\n",
      "BOS do you live here ? EOS\n",
      "BOS 你 住 这 里 吗 ? EOS\n",
      "translation: 你 住 这 里 吗 ?\n",
      "\n",
      "BOS i made it myself . EOS\n",
      "BOS 我 自 己 做 的 。 EOS\n",
      "translation: 我 自 己 做 的 。\n",
      "\n",
      "BOS it will rain soon . EOS\n",
      "BOS 天 快 要 下 雨 了 。 EOS\n",
      "translation: 天 快 要 下 雨 了 。\n",
      "\n",
      "BOS do n't talk UNK . EOS\n",
      "BOS 别 胡 说 。 EOS\n",
      "translation: 别 胡 说 。\n",
      "\n",
      "BOS training will be provided . EOS\n",
      "BOS 会 有 训 练 。 EOS\n",
      "translation: 会 有 训 练 。\n",
      "\n",
      "BOS tom did it again . EOS\n",
      "BOS 汤 姆 又 做 了 一 次 。 EOS\n",
      "translation: 汤 姆 又 做 了 一 次 。\n",
      "\n",
      "BOS i 'm really busy . EOS\n",
      "BOS 我 真 的 好 忙 。 EOS\n",
      "translation: 我 真 的 好 忙 。\n",
      "\n",
      "BOS i might say yes . EOS\n",
      "BOS 我 可 能 会 说 是 。 EOS\n",
      "translation: 我 可 能 会 说 是 。\n",
      "\n",
      "BOS i do n't lie . EOS\n",
      "BOS 我 不 说 谎 。 EOS\n",
      "translation: 我 不 说 谎 。\n",
      "\n",
      "BOS i do forgive tom . EOS\n",
      "BOS 我 真 的 原 谅 汤 姆 。 EOS\n",
      "translation: 我 真 的 原 谅 汤 姆 。\n",
      "\n",
      "BOS mary spoke japanese slowly . EOS\n",
      "BOS 玛 丽 日 语 说 得 很 慢 。 EOS\n",
      "translation: 玛 丽 日 语 说 得 很 慢 。\n",
      "\n",
      "BOS which cap is yours ? EOS\n",
      "BOS 哪 顶 帽 子 是 你 的 ？ EOS\n",
      "translation: 哪 顶 帽 子 是 你 的 ？\n",
      "\n",
      "BOS is she all right ? EOS\n",
      "BOS 她 是 正 确 的 吗 ？ EOS\n",
      "translation: 她 是 正 确 的 吗 ？\n",
      "\n",
      "BOS i noticed a UNK . EOS\n",
      "BOS 我 注 意 到 一 个 图 案 。 EOS\n",
      "translation: 我 注 意 到 一 个 图 案 。\n",
      "\n",
      "BOS it 's too large . EOS\n",
      "BOS 它 太 大 了 。 EOS\n",
      "translation: 它 太 大 了 。\n",
      "\n",
      "BOS everyone knows the law . EOS\n",
      "BOS 大 家 都 知 道 这 个 法 律 。 EOS\n",
      "translation: 大 家 都 知 道 这 个 法 律 。\n",
      "\n",
      "BOS you may take this . EOS\n",
      "BOS 您 可 以 拿 着 这 个 . EOS\n",
      "translation: 你 可 以 拿 这 个 .\n",
      "\n",
      "BOS i have three cameras . EOS\n",
      "BOS 我 有 三 台 摄 影 机 。 EOS\n",
      "translation: 我 有 三 台 摄 影 机 。\n",
      "\n",
      "BOS i can play tennis . EOS\n",
      "BOS 我 会 打 网 球 。 EOS\n",
      "translation: 我 会 打 网 球 。\n",
      "\n",
      "BOS was his story true ? EOS\n",
      "BOS 他 的 故 事 是 真 的 吗 ？ EOS\n",
      "translation: 他 的 故 事 是 真 的 吗 ？\n",
      "\n",
      "BOS tom is a UNK . EOS\n",
      "BOS 汤 姆 是 种 族 主 义 者 。 EOS\n",
      "translation: 汤 姆 是 个 骗 子 。\n",
      "\n",
      "BOS i sometimes watch tv . EOS\n",
      "BOS 我 有 时 看 电 视 。 EOS\n",
      "translation: 有 时 我 看 电 视 。\n",
      "\n",
      "BOS i usually eat out . EOS\n",
      "BOS 我 通 常 在 外 面 吃 饭 。 EOS\n",
      "translation: 我 通 常 在 外 面 吃 饭 。\n",
      "\n",
      "BOS i still love her . EOS\n",
      "BOS 我 仍 然 爱 着 她 。 EOS\n",
      "translation: 我 依 然 爱 着 她 。\n",
      "\n",
      "BOS it does n't matter . EOS\n",
      "BOS 没 关 系 。 EOS\n",
      "translation: 没 关 系 。\n",
      "\n",
      "BOS he took off his overcoat . EOS\n",
      "BOS 他 脱 掉 了 大 衣 。 EOS\n",
      "translation: 他 脱 下 了 大 衣 。\n",
      "\n",
      "BOS i do n't accept it . EOS\n",
      "BOS 我 不 接 受 。 EOS\n",
      "translation: 我 不 接 受 。\n",
      "\n",
      "BOS i 'm taller than you . EOS\n",
      "BOS 我 比 你 高 。 EOS\n",
      "translation: 我 比 你 高 。\n",
      "\n",
      "BOS i have to go home . EOS\n",
      "BOS 我 该 回 家 了 。 EOS\n",
      "translation: 我 该 回 家 了 。\n",
      "\n",
      "BOS she turned down every proposal . EOS\n",
      "BOS 她 拒 绝 了 每 一 项 提 议 。 EOS\n",
      "translation: 她 拒 绝 了 每 一 项 提 议 。\n",
      "\n",
      "BOS there is a spoon missing . EOS\n",
      "BOS 少 了 一 把 勺 子 。 EOS\n",
      "translation: 缺 一 把 勺 子 。\n",
      "\n",
      "BOS someone ate all my UNK . EOS\n",
      "BOS 某 人 把 我 的 杯 形 蛋 糕 吃 完 了 。 EOS\n",
      "translation: 某 人 把 我 的 杯 形 蛋 糕 吃 完 了 。\n",
      "\n",
      "BOS does it hurt a lot ? EOS\n",
      "BOS 很 痛 吗 ? EOS\n",
      "translation: 很 痛 吗 ?\n",
      "\n",
      "BOS he has just left home . EOS\n",
      "BOS 他 刚 离 开 家 。 EOS\n",
      "translation: 他 刚 离 开 家 。\n",
      "\n",
      "BOS she 's on a diet . EOS\n",
      "BOS 她 在 节 食 。 EOS\n",
      "translation: 她 在 节 食 。\n",
      "\n",
      "BOS he knows who they are . EOS\n",
      "BOS 他 知 道 他 们 是 谁 。 EOS\n",
      "translation: 他 认 识 他 们 是 谁 。\n",
      "\n",
      "BOS tom is always watching television . EOS\n",
      "BOS 汤 姆 总 是 在 看 电 视 。 EOS\n",
      "translation: 汤 姆 总 是 在 看 电 视 。\n",
      "\n",
      "BOS tom got ready for bed . EOS\n",
      "BOS 汤 姆 准 备 睡 觉 。 EOS\n",
      "translation: 汤 姆 准 备 睡 觉 。\n",
      "\n",
      "BOS the weather is good today . EOS\n",
      "BOS 今 天 天 气 很 好 。 EOS\n",
      "translation: 今 天 天 天 气 很 好 。\n",
      "\n",
      "BOS cooking takes too much time . EOS\n",
      "BOS 做 饭 太 费 时 间 了 。 EOS\n",
      "translation: 做 饭 太 费 了 太 多 时 间 。\n",
      "\n",
      "BOS we should 've studied harder . EOS\n",
      "BOS 我 们 本 该 更 加 努 力 学 习 。 EOS\n",
      "translation: 我 们 本 该 更 努 力 学 习 。\n",
      "\n",
      "BOS i prefer to remain seated . EOS\n",
      "BOS 我 更 想 继 续 坐 著 。 EOS\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translation: 我 更 想 继 续 坐 著 。\n",
      "\n",
      "BOS at last , he came . EOS\n",
      "BOS 最 终 ， 他 来 了 。 EOS\n",
      "translation: 最 终 ， 他 来 了 。\n",
      "\n",
      "BOS the soccer game is tomorrow . EOS\n",
      "BOS 明 天 有 足 球 比 赛 。 EOS\n",
      "translation: 明 天 有 足 球 比 赛 。\n",
      "\n",
      "BOS they have the same habits . EOS\n",
      "BOS 他 们 有 相 同 的 嗜 好 。 EOS\n",
      "translation: 他 们 有 相 同 的 嗜 好 。\n",
      "\n",
      "BOS tom knows mary was here . EOS\n",
      "BOS 汤 姆 知 道 玛 丽 在 这 里 。 EOS\n",
      "translation: 汤 姆 知 道 玛 丽 在 这 里 。\n",
      "\n",
      "BOS tom has a big appetite . EOS\n",
      "BOS 汤 姆 胃 口 很 大 。 EOS\n",
      "translation: 汤 姆 胃 口 很 大 。\n",
      "\n",
      "BOS it 's my birthday soon . EOS\n",
      "BOS 我 的 生 日 就 快 到 了 。 EOS\n",
      "translation: 我 的 生 日 就 快 到 了 。\n",
      "\n",
      "BOS are you coming next week ? EOS\n",
      "BOS 你 下 周 来 吗 ？ EOS\n",
      "translation: 你 下 周 来 吗 ？\n",
      "\n",
      "BOS what 's the weather like ? EOS\n",
      "BOS 天 气 怎 么 样 ？ EOS\n",
      "translation: 天 气 怎 么 样 ？\n",
      "\n",
      "BOS she is about my age . EOS\n",
      "BOS 她 大 约 跟 我 同 年 纪 。 EOS\n",
      "translation: 她 大 约 跟 我 同 年 纪 。\n",
      "\n",
      "BOS do n't let tom die . EOS\n",
      "BOS 别 让 汤 姆 死 了 。 EOS\n",
      "translation: 别 让 汤 姆 死 了 。\n",
      "\n",
      "BOS the soldiers UNK the building . EOS\n",
      "BOS 士 兵 们 佔 领 了 这 个 建 筑 。 EOS\n",
      "translation: 士 兵 们 佔 领 了 这 个 建 筑 。\n",
      "\n",
      "BOS i solved the problem easily . EOS\n",
      "BOS 我 很 容 易 地 解 决 了 这 个 问 题 。 EOS\n",
      "translation: 我 很 容 易 地 解 决 了 这 个 问 题 。\n",
      "\n",
      "BOS he UNK as he walked . EOS\n",
      "BOS 他 边 走 边 吹 口 哨 。 EOS\n",
      "translation: 他 边 走 边 吹 口 哨 。\n",
      "\n",
      "BOS tom is a good person . EOS\n",
      "BOS 汤 姆 是 个 好 人 。 EOS\n",
      "translation: 汤 姆 是 个 好 人 。\n",
      "\n",
      "BOS i want tom to win . EOS\n",
      "BOS 我 想 让 汤 姆 赢 。 EOS\n",
      "translation: 我 想 让 汤 姆 赢 。\n",
      "\n",
      "BOS can i have a bite ? EOS\n",
      "BOS 我 可 以 吃 一 口 吗 ？ EOS\n",
      "translation: 我 可 以 吃 一 口 吗 ？\n",
      "\n",
      "BOS do you know the answer ? EOS\n",
      "BOS 你 知 道 答 案 吗 ? EOS\n",
      "translation: 你 知 道 答 案 吗 ?\n",
      "\n",
      "BOS they 're afraid of us . EOS\n",
      "BOS 他 们 害 怕 我 们 。 EOS\n",
      "translation: 他 们 害 怕 我 们 。\n",
      "\n",
      "BOS i 'm tired of translating . EOS\n",
      "BOS 我 厌 倦 了 翻 译 。 EOS\n",
      "translation: 我 厌 倦 了 翻 译 。\n",
      "\n",
      "BOS how did you meet tom ? EOS\n",
      "BOS 你 是 怎 么 见 到 汤 姆 的 ？ EOS\n",
      "translation: 你 是 怎 么 见 到 汤 姆 的 ？\n",
      "\n",
      "BOS i 'm a good guy . EOS\n",
      "BOS 我 是 一 个 好 人 。 EOS\n",
      "translation: 我 是 一 个 好 人 。\n",
      "\n",
      "BOS red is out of fashion . EOS\n",
      "BOS 红 色 不 流 行 了 。 EOS\n",
      "translation: 红 色 不 流 行 了 。\n",
      "\n",
      "BOS the japanese have dark eyes . EOS\n",
      "BOS 日 本 人 有 黑 色 的 眼 睛 。 EOS\n",
      "translation: 日 本 人 有 黑 色 的 眼 睛 。\n",
      "\n",
      "BOS should n't you be studying ? EOS\n",
      "BOS 你 不 应 该 在 学 习 吗 ？ EOS\n",
      "translation: 你 不 应 该 在 学 习 吗 ？\n",
      "\n",
      "BOS may i borrow your knife ? EOS\n",
      "BOS 我 能 借 一 下 你 的 刀 吗 ？ EOS\n",
      "translation: 我 能 借 一 下 你 的 刀 吗 ？\n",
      "\n",
      "BOS he got off the bus . EOS\n",
      "BOS 他 下 了 公 车 。 EOS\n",
      "translation: 他 下 了 公 车 。\n",
      "\n",
      "BOS UNK loved playing the violin . EOS\n",
      "BOS 爱 因 斯 坦 喜 欢 拉 小 提 琴 。 EOS\n",
      "translation: 爱 因 斯 坦 喜 欢 拉 小 提 琴 。\n",
      "\n",
      "BOS they did n't act quickly . EOS\n",
      "BOS 他 们 没 有 立 刻 行 动 。 EOS\n",
      "translation: 他 们 没 有 立 刻 行 动 。\n",
      "\n",
      "BOS what 's tom given us ? EOS\n",
      "BOS 汤 姆 给 了 我 们 什 么 ？ EOS\n",
      "translation: 汤 姆 给 了 我 们 什 么 ？\n",
      "\n",
      "BOS he participated in the UNK . EOS\n",
      "BOS 他 参 加 辩 论 EOS\n",
      "translation: 他 参 加 辩 论\n",
      "\n",
      "BOS i think i will stay . EOS\n",
      "BOS 我 认 我 会 留 下 。 EOS\n",
      "translation: 我 认 我 会 留 下 。\n",
      "\n",
      "BOS what was that meeting about ? EOS\n",
      "BOS 那 场 会 议 主 题 是 什 么 ？ EOS\n",
      "translation: 那 场 会 议 主 题 是 什 么 ？\n",
      "\n",
      "BOS i have nothing to say . EOS\n",
      "BOS 我 没 什 么 可 说 的 。 EOS\n",
      "translation: 我 没 什 么 可 说 的 。\n",
      "\n",
      "BOS this really is great weather . EOS\n",
      "BOS 这 真 的 是 个 好 天 气 。 EOS\n",
      "translation: 这 真 的 是 个 好 天 气 。\n",
      "\n",
      "BOS this book is quite difficult . EOS\n",
      "BOS 这 本 书 很 难 读 。 EOS\n",
      "translation: 这 本 书 很 难 读 。\n",
      "\n",
      "BOS how long did that take ? EOS\n",
      "BOS 那 持 续 了 多 久 了 ？ EOS\n",
      "translation: 那 持 续 了 多 久 了 ？\n",
      "\n",
      "BOS where can i find UNK ? EOS\n",
      "BOS 在 哪 儿 我 能 找 到 牙 UNK ？ EOS\n",
      "translation: 在 哪 儿 我 能 找 到 牙 ？\n",
      "\n",
      "BOS i really have to go . EOS\n",
      "BOS 我 真 的 得 走 了 。 EOS\n",
      "translation: 我 真 的 得 走 了 。\n",
      "\n",
      "BOS his words gave me hope . EOS\n",
      "BOS 他 的 话 给 了 我 希 望 。 EOS\n",
      "translation: 他 的 话 让 我 希 望 。\n",
      "\n",
      "BOS the news made her sad . EOS\n",
      "BOS 这 个 消 息 让 她 很 伤 心 。 EOS\n",
      "translation: 这 个 消 息 让 她 很 伤 心 。\n",
      "\n",
      "BOS she came into the room . EOS\n",
      "BOS 她 进 了 房 间 。 EOS\n",
      "translation: 她 进 了 房 间 。\n",
      "\n",
      "BOS he seems to be ill . EOS\n",
      "BOS 他 好 像 病 了 。 EOS\n",
      "translation: 他 看 起 来 有 病 了 。\n",
      "\n",
      "BOS i 'm not too smart . EOS\n",
      "BOS 我 不 太 聪 明 。 EOS\n",
      "translation: 我 不 太 聪 明 。\n",
      "\n",
      "BOS tell tom what you heard . EOS\n",
      "BOS 把 你 听 到 的 告 诉 汤 姆 。 EOS\n",
      "translation: 告 诉 汤 姆 你 在 听 什 么 。\n",
      "\n",
      "BOS where 's the UNK pharmacy ? EOS\n",
      "BOS 最 近 的 药 房 在 哪 里 ? EOS\n",
      "translation: 最 近 的 药 房 在 哪 里 ?\n",
      "\n",
      "BOS he 's a bit UNK . EOS\n",
      "BOS 他 有 点 活 泼 。 EOS\n",
      "translation: 他 有 点 活 泼 。\n",
      "\n",
      "BOS it 's cold this morning . EOS\n",
      "BOS 今 天 早 上 冷 。 EOS\n",
      "translation: 今 天 早 上 冷 。\n",
      "\n",
      "BOS it 's a clear day . EOS\n",
      "BOS 它 是 一 个 晴 朗 的 日 子 。 EOS\n",
      "translation: 天 空 很 晴 朗 。\n",
      "\n",
      "BOS you should come back immediately . EOS\n",
      "BOS 你 应 该 马 上 回 来 。 EOS\n",
      "translation: 你 应 该 马 上 回 来 。\n",
      "\n",
      "BOS i was struck by lightning . EOS\n",
      "BOS 我 被 雷 UNK 了 。 EOS\n",
      "translation: 我 被 雷 击 了 。\n",
      "\n",
      "BOS it 's hot out here . EOS\n",
      "BOS 这 里 够 热 。 EOS\n",
      "translation: 这 里 够 热 。\n",
      "\n",
      "BOS tom is a boston native . EOS\n",
      "BOS 汤 姆 是 波 士 顿 本 地 人 。 EOS\n",
      "translation: 汤 姆 是 波 士 顿 本 地 人 。\n",
      "\n",
      "BOS could you drive tom home ? EOS\n",
      "BOS 你 能 载 汤 姆 回 家 吗 ？ EOS\n",
      "translation: 你 能 载 汤 姆 回 家 吗 ？\n",
      "\n",
      "BOS he seldom goes to church . EOS\n",
      "BOS 他 很 少 去 教 堂 。 EOS\n",
      "translation: 他 很 少 去 教 堂 。\n",
      "\n",
      "BOS tom gave mary a UNK . EOS\n",
      "BOS 汤 姆 给 了 玛 丽 一 把 手 电 UNK 。 EOS\n",
      "translation: 汤 姆 给 玛 丽 一 把 手 电 。\n",
      "\n",
      "BOS he 's a good UNK . EOS\n",
      "BOS 他 是 个 好 人 。 EOS\n",
      "translation: 他 是 个 好 的 。\n",
      "\n",
      "BOS she came across the street . EOS\n",
      "BOS 她 过 了 马 路 。 EOS\n",
      "translation: 她 过 了 马 路 。\n",
      "\n",
      "BOS tom has written to me . EOS\n",
      "BOS 汤 姆 写 给 我 了 。 EOS\n",
      "translation: 汤 姆 写 给 我 。\n",
      "\n",
      "BOS let 's get off here . EOS\n",
      "BOS 我 们 在 这 里 下 车 吧 。 EOS\n",
      "translation: 我 们 在 这 里 下 车 吧 。\n",
      "\n",
      "BOS i 'm tom 's assistant . EOS\n",
      "BOS 我 是 汤 姆 的 助 手 。 EOS\n",
      "translation: 我 是 汤 姆 的 助 手 。\n",
      "\n",
      "BOS he will be back tomorrow . EOS\n",
      "BOS 明 天 他 会 回 来 。 EOS\n",
      "translation: 他 明 天 会 回 来 。\n",
      "\n",
      "BOS she folded her handkerchief UNK . EOS\n",
      "BOS 她 整 齐 地 把 她 的 手 帕 折 好 。 EOS\n",
      "translation: 她 整 齐 地 把 她 的 手 帕 折 好 。\n",
      "\n",
      "BOS you should take up golf . EOS\n",
      "BOS 你 应 该 开 始 从 事 高 尔 夫 。 EOS\n",
      "translation: 你 应 该 开 始 从 事 。\n",
      "\n",
      "BOS tom 's comment was inappropriate . EOS\n",
      "BOS 汤 姆 的 评 论 不 合 适 。 EOS\n",
      "translation: 汤 姆 的 评 论 不 合 适 。\n",
      "\n",
      "BOS she gave me a present . EOS\n",
      "BOS 她 给 了 我 一 件 礼 物 。 EOS\n",
      "translation: 她 给 了 我 一 个 礼 物 。\n",
      "\n",
      "BOS did he go to see mary ? EOS\n",
      "BOS 他 去 看 玛 丽 了 吗 ？ EOS\n",
      "translation: 他 去 看 玛 丽 了 吗 ？\n",
      "\n",
      "BOS my father works at a factory . EOS\n",
      "BOS 我 父 亲 在 工 厂 工 作 。 EOS\n",
      "translation: 我 爸 爸 爸 在 工 厂 工 作 。\n",
      "\n",
      "BOS we are going to leave tomorrow . EOS\n",
      "BOS 我 们 明 天 要 离 开 。 EOS\n",
      "translation: 我 们 明 天 要 离 开 。\n",
      "\n",
      "BOS i suggested that she go alone . EOS\n",
      "BOS 我 建 议 她 一 个 人 去 。 EOS\n",
      "translation: 我 建 议 她 一 个 人 去 。\n",
      "\n",
      "BOS tom is now living in boston . EOS\n",
      "BOS 汤 姆 现 在 住 在 波 士 顿 。 EOS\n",
      "translation: 汤 姆 现 在 住 在 波 士 顿 。\n",
      "\n",
      "BOS she was hit by a car . EOS\n",
      "BOS 她 被 车 撞 了 。 EOS\n",
      "translation: 她 被 车 撞 了 。\n",
      "\n",
      "BOS she was busy with her UNK . EOS\n",
      "BOS 她 忙 着 编 织 。 EOS\n",
      "translation: 她 忙 着 编 织 。\n",
      "\n",
      "BOS were your mother and father home ? EOS\n",
      "BOS 你 父 母 亲 在 家 吗 ？ EOS\n",
      "translation: 你 父 亲 在 家 吗 ？\n",
      "\n",
      "BOS he does n't speak our language . EOS\n",
      "BOS 他 不 会 说 我 们 的 语 言 。 EOS\n",
      "translation: 他 不 会 说 我 们 的 语 言 。\n",
      "\n",
      "BOS i owe ten dollars to her . EOS\n",
      "BOS 我 欠 她 1 0 美 元 。 EOS\n",
      "translation: 我 欠 她 1 0 美 元 。\n",
      "\n",
      "BOS wet clothes stick to your skin . EOS\n",
      "BOS 湿 衣 服 贴 著 你 的 皮 肤 。 EOS\n",
      "translation: 湿 衣 服 贴 著 你 的 皮 肤 。\n",
      "\n",
      "BOS they must be waiting for you . EOS\n",
      "BOS 他 们 一 定 在 等 你 。 EOS\n",
      "translation: 他 们 一 定 在 等 你 。\n",
      "\n",
      "BOS i will lend you my textbook . EOS\n",
      "BOS 我 会 借 给 你 我 的 课 本 。 EOS\n",
      "translation: 我 会 借 给 你 我 的 课 本 。\n",
      "\n",
      "BOS how about going to the movies ? EOS\n",
      "BOS 去 看 电 影 怎 样 ？ EOS\n",
      "translation: 去 看 电 影 怎 么 样 ？\n",
      "\n",
      "BOS a good idea came to me . EOS\n",
      "BOS 我 有 一 个 好 主 意 。 EOS\n",
      "translation: 我 有 一 个 好 主 意 。\n",
      "\n",
      "BOS what were you doing this morning ? EOS\n",
      "BOS 今 天 早 上 你 在 做 什 么 ？ EOS\n",
      "translation: 今 天 早 上 你 在 做 什 么 ？\n",
      "\n",
      "BOS you must pay attention to him . EOS\n",
      "BOS 你 必 须 注 意 他 。 EOS\n",
      "translation: 你 必 须 注 意 他 。\n",
      "\n",
      "BOS i will attend the next meeting . EOS\n",
      "BOS 我 会 参 加 下 次 的 会 议 。 EOS\n",
      "translation: 我 会 在 下 次 会 议 的 会 出 席 。\n",
      "\n",
      "BOS it was very hot this afternoon . EOS\n",
      "BOS 今 天 下 午 非 常 炎 热 。 EOS\n",
      "translation: 今 天 下 午 非 常 炎 热 。\n",
      "\n",
      "BOS how about some more UNK beef ? EOS\n",
      "BOS 再 多 一 些 烤 牛 肉 怎 么 样 ? EOS\n",
      "translation: 再 来 一 些 烤 牛 肉 怎 么 样 ?\n",
      "\n",
      "BOS they made us work all night . EOS\n",
      "BOS 他 们 要 我 们 整 夜 工 作 。 EOS\n",
      "translation: 他 们 要 我 们 整 夜 工 作 。\n",
      "\n",
      "BOS i ran away in a hurry . EOS\n",
      "BOS 我 赶 快 跑 走 了 。 EOS\n",
      "translation: 我 赶 快 !\n",
      "\n",
      "BOS i want him to read this . EOS\n",
      "BOS 想 让 他 读 这 个 。 EOS\n",
      "translation: 我 想 让 他 读 这 个 。\n",
      "\n",
      "BOS i need to take a shower . EOS\n",
      "BOS 我 需 要 洗 个 澡 。 EOS\n",
      "translation: 我 需 要 洗 个 澡 。\n",
      "\n",
      "BOS i 'm buying a new car . EOS\n",
      "BOS 我 在 买 一 辆 新 车 。 EOS\n",
      "translation: 我 在 买 一 辆 新 车 。\n",
      "\n",
      "BOS i stood waiting for a bus . EOS\n",
      "BOS 我 站 著 等 公 车 。 EOS\n",
      "translation: 我 站 著 等 公 车 。\n",
      "\n",
      "BOS i will show you my room . EOS\n",
      "BOS 我 会 带 你 看 看 我 的 房 间 。 EOS\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translation: 我 会 带 你 看 我 的 房 间 。\n",
      "\n",
      "BOS she made fun of her husband . EOS\n",
      "BOS 她 取 笑 了 她 的 丈 夫 。 EOS\n",
      "translation: 她 取 笑 了 她 的 丈 夫 。\n",
      "\n",
      "BOS chocolate has more iron than spinach . EOS\n",
      "BOS 巧 克 力 含 有 比 菠 菜 更 多 的 铁 质 。 EOS\n",
      "translation: 巧 克 力 含 有 比 菠 菜 更 多 的 铁 质 。\n",
      "\n",
      "BOS i have something to give you . EOS\n",
      "BOS 我 有 些 东 西 要 给 你 。 EOS\n",
      "translation: 我 有 些 东 西 要 给 你 。\n",
      "\n",
      "BOS tom is n't a good worker . EOS\n",
      "BOS 汤 姆 不 是 个 好 工 人 。 EOS\n",
      "translation: 汤 姆 不 是 个 好 工 人 。\n",
      "\n",
      "BOS please wait till he comes back . EOS\n",
      "BOS 请 等 到 他 回 来 。 EOS\n",
      "translation: 请 等 到 他 回 来 。\n",
      "\n",
      "BOS do you want anything to eat ? EOS\n",
      "BOS 你 想 要 吃 任 何 东 西 吗 ? EOS\n",
      "translation: 你 想 要 吃 任 何 东 西 吗 ?\n",
      "\n",
      "BOS i left earlier than my sister . EOS\n",
      "BOS 我 比 我 的 妹 妹 早 离 开 。 EOS\n",
      "translation: 我 比 我 的 妹 妹 早 离 开 。\n",
      "\n",
      "BOS is everything going ok at work ? EOS\n",
      "BOS 工 作 一 切 顺 利 吗 ？ EOS\n",
      "translation: 工 作 一 切 顺 利 吗 ？\n",
      "\n",
      "BOS we 've already won this battle . EOS\n",
      "BOS 我 们 已 经 赢 得 了 这 场 战 斗 。 EOS\n",
      "translation: 我 们 已 经 赢 得 了 这 场 战 斗 。\n",
      "\n",
      "BOS can i have something to eat ? EOS\n",
      "BOS 我 能 些 吃 的 东 西 吗 ？ EOS\n",
      "translation: 我 能 些 吃 的 东 西 吗 ？\n",
      "\n",
      "BOS she has already left the office . EOS\n",
      "BOS 她 已 经 离 开 了 办 公 室 。 EOS\n",
      "translation: 她 已 经 离 开 了 办 公 室 。\n",
      "\n",
      "BOS none of those books are useful . EOS\n",
      "BOS 这 些 书 里 没 有 一 本 是 有 用 的 。 EOS\n",
      "translation: 这 些 书 都 没 有 用 。\n",
      "\n",
      "BOS it 's going to rain soon . EOS\n",
      "BOS 天 快 要 下 雨 了 。 EOS\n",
      "translation: 天 快 要 下 雨 了 。\n",
      "\n",
      "BOS she climbed down from the roof . EOS\n",
      "BOS 她 从 屋 顶 上 下 来 了 。 EOS\n",
      "translation: 她 从 屋 顶 上 下 来 了 。\n",
      "\n",
      "BOS taking off is easier than landing . EOS\n",
      "BOS 起 飞 比 降 落 容 易 。 EOS\n",
      "translation: 起 飞 比 降 落 容 易 。\n",
      "\n",
      "BOS have you ever seen him swimming ? EOS\n",
      "BOS 你 曾 看 过 他 游 泳 吗 ？ EOS\n",
      "translation: 你 曾 看 过 他 游 泳 吗 ？\n",
      "\n",
      "BOS tom can not tie a tie . EOS\n",
      "BOS 汤 姆 不 会 系 领 带 。 EOS\n",
      "translation: 汤 姆 不 会 系 领 带 。\n",
      "\n",
      "BOS this vending machine is n't working . EOS\n",
      "BOS 这 台 自 动 UNK 卖 机 故 障 了 。 EOS\n",
      "translation: 这 台 自 动 机 不 运 转 。\n",
      "\n",
      "BOS not everybody can be an artist . EOS\n",
      "BOS 不 是 所 有 人 都 能 当 艺 术 家 的 。 EOS\n",
      "translation: 不 是 所 有 人 都 能 当 艺 术 家 的 。\n",
      "\n",
      "BOS tom did n't buy any bread . EOS\n",
      "BOS 汤 姆 没 有 买 甚 么 面 包 。 EOS\n",
      "translation: 汤 姆 没 有 买 甚 么 面 包 。\n",
      "\n",
      "BOS please tell me everything 's ok . EOS\n",
      "BOS 请 告 诉 我 一 切 都 好 。 EOS\n",
      "translation: 请 告 诉 我 一 切 都 好 。\n",
      "\n",
      "BOS they do n't have to know . EOS\n",
      "BOS 他 们 不 需 要 知 道 。 EOS\n",
      "translation: 他 们 不 需 要 知 道 。\n",
      "\n",
      "BOS do n't try to stop me . EOS\n",
      "BOS 别 想 阻 止 我 。 EOS\n",
      "translation: 别 想 阻 止 我 。\n",
      "\n",
      "BOS she did n't have any pencils . EOS\n",
      "BOS 她 一 支 铅 笔 也 没 有 。 EOS\n",
      "translation: 她 一 支 铅 笔 也 没 有 。\n",
      "\n",
      "BOS our school is fifty years old . EOS\n",
      "BOS 我 们 学 校 有 五 十 年 的 历 史 了 。 EOS\n",
      "translation: 我 们 学 校 有 五 十 年 的 历 史 了 。\n",
      "\n",
      "BOS i 'm very tired right now . EOS\n",
      "BOS 我 现 在 很 累 。 EOS\n",
      "translation: 我 现 在 很 累 。\n",
      "\n",
      "BOS i can help you , too . EOS\n",
      "BOS 我 也 能 帮 你 。 EOS\n",
      "translation: 我 也 能 帮 你 。\n",
      "\n",
      "BOS could you please autograph this book ? EOS\n",
      "BOS 请 你 在 这 本 书 上 签 名 好 吗 ？ EOS\n",
      "translation: 请 你 在 这 本 书 上 签 名 好 吗 ?\n",
      "\n",
      "BOS has n't tom told you anything ? EOS\n",
      "BOS 汤 姆 什 么 都 没 告 诉 你 吗 ？ EOS\n",
      "translation: 汤 姆 什 么 都 没 告 诉 你 吗 ？\n",
      "\n",
      "BOS they 're trying to control you . EOS\n",
      "BOS 他 们 设 法 控 制 你 。 EOS\n",
      "translation: 他 们 设 法 控 制 你 。\n",
      "\n",
      "BOS my father lives in the country . EOS\n",
      "BOS 我 父 亲 住 在 乡 下 。 EOS\n",
      "translation: 我 父 亲 住 在 乡 下 。\n",
      "\n",
      "BOS would you like tea or coffee ? EOS\n",
      "BOS 您 想 要 茶 还 是 咖 啡 ？ EOS\n",
      "translation: 你 想 要 茶 还 是 咖 啡 ？\n",
      "\n",
      "BOS let 's do it another time . EOS\n",
      "BOS 改 天 再 说 吧 。 EOS\n",
      "translation: 改 天 再 说 吧 。\n",
      "\n",
      "BOS would you like to go abroad ? EOS\n",
      "BOS 你 想 出 国 吗 ? EOS\n",
      "translation: 你 想 出 国 吗 ?\n",
      "\n",
      "BOS tom sat at his desk working . EOS\n",
      "BOS 汤 姆 在 他 的 桌 边 工 作 。 EOS\n",
      "translation: 汤 姆 在 他 的 桌 边 工 作 。\n",
      "\n",
      "BOS do you have any japanese magazines ? EOS\n",
      "BOS 你 有 任 何 日 本 杂 志 吗 ？ EOS\n",
      "translation: 你 有 任 何 日 本 杂 志 吗 ？\n",
      "\n",
      "BOS he came to see you yesterday . EOS\n",
      "BOS 他 昨 天 来 看 你 。 EOS\n",
      "translation: 他 昨 天 来 看 你 。\n",
      "\n",
      "BOS i should be ready by 2:30 . EOS\n",
      "BOS 我 应 该 在 2 ： 3 0 准 备 好 。 EOS\n",
      "translation: 我 应 该 在 2 ： 3 0 准 备 好 。\n",
      "\n",
      "BOS the boy enjoyed painting a picture . EOS\n",
      "BOS 这 个 男 孩 喜 欢 UNK 画 。 EOS\n",
      "translation: 这 个 男 孩 喜 欢 画 。\n",
      "\n",
      "BOS what 's the weather like there ? EOS\n",
      "BOS 那 儿 是 什 么 天 气 ？ EOS\n",
      "translation: 那 里 的 气 候 怎 么 样 ?\n",
      "\n",
      "BOS wake me up early tomorrow morning . EOS\n",
      "BOS 明 天 早 上 早 点 叫 醒 我 。 EOS\n",
      "translation: 明 天 早 上 早 点 叫 醒 我 。\n",
      "\n",
      "BOS she married him for his money . EOS\n",
      "BOS 她 为 了 他 的 钱 嫁 给 了 他 。 EOS\n",
      "translation: 她 为 了 他 的 钱 嫁 给 了 他 。\n",
      "\n",
      "BOS money makes the world go round . EOS\n",
      "BOS 金 钱 万 能 。 EOS\n",
      "translation: 金 钱 万 能 。\n",
      "\n",
      "BOS what time do you start work ? EOS\n",
      "BOS 你 什 么 时 候 开 始 工 作 ? EOS\n",
      "translation: 你 什 么 时 候 开 始 工 作 ?\n",
      "\n",
      "BOS she must have studied very hard . EOS\n",
      "BOS 她 一 定 很 用 功 读 书 。 EOS\n",
      "translation: 她 一 定 很 用 功 读 书 。\n",
      "\n",
      "BOS tom gave mary the cold shoulder . EOS\n",
      "BOS 汤 姆 给 玛 丽 冷 的 肩 膀 。 EOS\n",
      "translation: 汤 姆 给 玛 丽 冷 的 肩 膀 。\n",
      "\n",
      "BOS the man is wanted for murder . EOS\n",
      "BOS 这 人 因 谋 杀 被 通 缉 。 EOS\n",
      "translation: 这 人 因 谋 杀 被 通 缉 。\n",
      "\n",
      "BOS turn the key to the right . EOS\n",
      "BOS 往 右 边 转 动 钥 匙 。 EOS\n",
      "translation: 往 右 边 转 动 钥 匙 。\n",
      "\n",
      "BOS where did you go that night ? EOS\n",
      "BOS 那 天 晚 上 你 在 哪 里 ？ EOS\n",
      "translation: 那 天 晚 上 你 在 哪 里 ？\n",
      "\n",
      "BOS i belong to the swimming club . EOS\n",
      "BOS 我 参 加 游 泳 社 。 EOS\n",
      "translation: 我 参 加 游 泳 社 。\n",
      "\n",
      "BOS i can not shut it down . EOS\n",
      "BOS 我 不 能 关 掉 它 。 EOS\n",
      "translation: 我 不 能 关 掉 它 。\n",
      "\n",
      "BOS i want to reserve a room . EOS\n",
      "BOS 我 想 预 定 一 个 房 间 。 EOS\n",
      "translation: 我 想 预 定 一 个 房 间 。\n",
      "\n",
      "BOS hi ! do you work here ? EOS\n",
      "BOS 嗨 ！ 你 在 这 儿 工 作 吗 ？ EOS\n",
      "translation: 嗨 ！ 你 在 这 儿 工 作 吗 ？\n",
      "\n",
      "BOS how many people know about us ? EOS\n",
      "BOS 有 多 少 人 知 道 我 们 ？ EOS\n",
      "translation: 有 多 少 人 知 道 我 们 ？\n",
      "\n",
      "BOS tom helped mary move the furniture . EOS\n",
      "BOS 汤 姆 帮 玛 丽 移 动 家 俱 。 EOS\n",
      "translation: 汤 姆 帮 玛 丽 移 动 家 俱 。\n",
      "\n",
      "BOS i do n't like to UNK . EOS\n",
      "BOS 我 不 喜 欢 跟 人 打 交 道 。 EOS\n",
      "translation: 我 不 喜 欢 跟 人 打 交 道 。\n",
      "\n",
      "BOS have you ever had food UNK ? EOS\n",
      "BOS 您 曾 经 食 物 中 毒 过 吗 ？ EOS\n",
      "translation: 你 曾 经 食 物 中 毒 过 吗 ？\n",
      "\n",
      "BOS how did you find my house ? EOS\n",
      "BOS 你 怎 么 找 到 我 的 房 子 的 ？ EOS\n",
      "translation: 你 怎 么 找 到 我 的 房 子 的 ？\n",
      "\n",
      "BOS i want to buy this dictionary . EOS\n",
      "BOS 我 想 买 这 本 字 典 。 EOS\n",
      "translation: 我 想 买 这 本 字 典 。\n",
      "\n",
      "BOS water becomes UNK when it freezes . EOS\n",
      "BOS 水 结 冰 后 成 为 固 体 。 EOS\n",
      "translation: 水 结 冰 后 成 为 固 体 。\n",
      "\n",
      "BOS tom wants to go to japan . EOS\n",
      "BOS 汤 姆 想 去 日 本 。 EOS\n",
      "translation: 汤 姆 想 去 日 本 。\n",
      "\n",
      "BOS my french is n't good enough . EOS\n",
      "BOS 我 的 法 语 说 得 不 够 好 。 EOS\n",
      "translation: 我 的 法 语 说 得 不 够 好 。\n",
      "\n",
      "BOS i should not have said that . EOS\n",
      "BOS 我 不 该 说 那 个 。 EOS\n",
      "translation: 我 不 应 该 说 那 个 。\n",
      "\n",
      "BOS school UNK before noon on saturdays . EOS\n",
      "BOS 学 校 在 星 期 六 中 午 之 前 放 学 。 EOS\n",
      "translation: 学 校 在 周 六 前 到 学 校 。\n",
      "\n",
      "BOS will the train leave on time ? EOS\n",
      "BOS 这 班 火 车 会 准 时 出 发 吗 ？ EOS\n",
      "translation: 这 班 火 车 会 准 时 出 发 吗 ？\n",
      "\n",
      "BOS she UNK herself in a blanket . EOS\n",
      "BOS 她 用 一 条 毯 子 把 自 己 裹 起 来 。 EOS\n",
      "translation: 她 用 一 条 毯 子 把 自 己 裹 起 来 。\n",
      "\n",
      "BOS your tripod is in my office . EOS\n",
      "BOS 你 的 三 脚 架 在 我 的 办 公 室 里 。 EOS\n",
      "translation: 你 的 三 脚 架 在 我 的 办 公 室 里 。\n",
      "\n",
      "BOS i think maybe tom was right . EOS\n",
      "BOS 我 认 为 汤 姆 可 能 是 对 的 。 EOS\n",
      "translation: 我 认 为 汤 姆 可 能 是 对 的 。\n",
      "\n",
      "BOS he painted the door over white . EOS\n",
      "BOS 他 把 门 漆 成 了 白 色 。 EOS\n",
      "translation: 他 把 门 漆 成 了 白 色 。\n",
      "\n",
      "BOS how long will the meeting last ? EOS\n",
      "BOS 会 议 持 续 多 久 ？ EOS\n",
      "translation: 会 议 持 续 多 久 ？\n",
      "\n",
      "BOS i have a brother in boston . EOS\n",
      "BOS 我 有 个 在 波 士 顿 的 兄 弟 。 EOS\n",
      "translation: 我 有 个 在 波 士 顿 的 兄 弟 。\n",
      "\n",
      "BOS we 're still a little confused . EOS\n",
      "BOS 我 们 还 是 有 点 疑 惑 。 EOS\n",
      "translation: 我 们 还 是 有 点 疑 惑 。\n",
      "\n",
      "BOS he 's always at home on sundays . EOS\n",
      "BOS 他 星 期 日 总 是 在 家 里 。 EOS\n",
      "translation: 他 总 是 在 家 里 。\n",
      "\n",
      "BOS there 's nothing good on television now . EOS\n",
      "BOS 现 在 电 视 上 没 有 甚 么 好 看 的 。 EOS\n",
      "translation: 现 在 电 视 上 没 有 什 么 好 事 。\n",
      "\n",
      "BOS do you really want tom to UNK ? EOS\n",
      "BOS 你 真 想 让 汤 姆 受 苦 吗 ？ EOS\n",
      "translation: 你 真 想 让 汤 姆 受 苦 吗 ？\n",
      "\n",
      "BOS japan UNK a lot of good cameras . EOS\n",
      "BOS 日 本 生 产 很 多 好 相 机 。 EOS\n",
      "translation: 日 本 生 产 很 多 好 相 机 。\n",
      "\n",
      "BOS i met him on my way home . EOS\n",
      "BOS 我 在 回 家 的 路 上 遇 见 了 他 。 EOS\n",
      "translation: 我 在 回 家 的 路 上 遇 见 了 他 。\n",
      "\n",
      "BOS i lost my way in the woods . EOS\n",
      "BOS 我 在 树 林 里 迷 路 了 。 EOS\n",
      "translation: 我 在 树 林 里 迷 路 了 。\n",
      "\n",
      "BOS i know how to set a trap . EOS\n",
      "BOS 我 知 道 怎 么 设 陷 阱 。 EOS\n",
      "translation: 我 知 道 怎 么 设 陷 阱 。\n",
      "\n",
      "BOS he stuck his pencil behind his ear . EOS\n",
      "BOS 他 把 他 的 铅 笔 放 在 他 的 耳 朵 后 面 。 EOS\n",
      "translation: 他 把 他 的 铅 笔 放 在 他 的 耳 朵 后 面 。\n",
      "\n",
      "BOS that 's the part i liked best . EOS\n",
      "BOS 那 是 我 最 喜 欢 的 部 分 。 EOS\n",
      "translation: 这 是 我 最 喜 欢 的 部 分 。\n",
      "\n",
      "BOS pass me the salt , will you ? EOS\n",
      "BOS 请 把 盐 递 给 我 ， 好 吗 ？ EOS\n",
      "translation: 请 给 我 盐 ， 好 吗 ？\n",
      "\n",
      "BOS he is getting along with his neighborhood . EOS\n",
      "BOS 他 与 他 的 邻 居 相 处 。 EOS\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translation: 他 和 他 的 邻 居 相 处 。\n",
      "\n",
      "BOS i do n't see much of him . EOS\n",
      "BOS 我 不 常 见 到 他 。 EOS\n",
      "translation: 我 不 常 见 到 他 。\n",
      "\n",
      "BOS she was very proud of her father . EOS\n",
      "BOS 她 为 她 的 父 亲 感 到 非 常 骄 傲 。 EOS\n",
      "translation: 她 为 她 的 父 亲 感 到 非 常 骄 傲 。\n",
      "\n",
      "BOS i would like to be a UNK . EOS\n",
      "BOS 我 想 要 成 为 吉 他 手 。 EOS\n",
      "translation: 我 想 要 成 为 吉 他 手 。\n",
      "\n",
      "BOS my leg was bitten by that dog . EOS\n",
      "BOS 我 的 腿 被 那 只 狗 咬 了 。 EOS\n",
      "translation: 我 的 腿 被 那 只 狗 咬 了 。\n",
      "\n",
      "BOS i 've found a place to live . EOS\n",
      "BOS 我 找 到 了 个 住 处 。 EOS\n",
      "translation: 我 找 到 了 个 住 处 。\n",
      "\n",
      "BOS i think i know who did it . EOS\n",
      "BOS 我 认 为 我 知 道 是 谁 做 的 。 EOS\n",
      "translation: 我 认 为 我 知 道 是 谁 做 的 。\n",
      "\n",
      "BOS i got up early in the morning . EOS\n",
      "BOS 我 早 上 很 早 起 牀 。 EOS\n",
      "translation: 我 早 上 很 早 起 牀 。\n",
      "\n",
      "BOS what did you major in at college ? EOS\n",
      "BOS 你 大 学 时 主 修 什 么 ？ EOS\n",
      "translation: 你 大 学 时 主 修 什 么 ？\n",
      "\n",
      "BOS tom does n't share mary 's UNK . EOS\n",
      "BOS 汤 姆 没 有 感 受 到 玛 丽 的 热 情 。 EOS\n",
      "translation: 汤 姆 没 有 感 受 到 玛 丽 的 热 情 。\n",
      "\n",
      "BOS i 've been careful my whole life . EOS\n",
      "BOS 我 一 生 都 谨 慎 行 事 。 EOS\n",
      "translation: 我 一 生 都 谨 慎 行 事 。\n",
      "\n",
      "BOS UNK is no excuse for being late . EOS\n",
      "BOS 睡 过 头 不 是 迟 到 的 理 由 。 EOS\n",
      "translation: 睡 过 头 不 是 迟 到 的 理 由 。\n",
      "\n",
      "BOS we receive many telephone calls from abroad . EOS\n",
      "BOS 我 们 接 到 许 多 来 自 国 外 的 电 话 。 EOS\n",
      "translation: 我 们 接 到 许 多 来 自 国 外 的 电 话 。\n",
      "\n",
      "BOS we UNK a path through the forest . EOS\n",
      "BOS 我 们 开 出 了 一 条 穿 过 森 林 的 小 路 。 EOS\n",
      "translation: 我 们 开 出 了 一 条 穿 过 森 林 的 小 路 。\n",
      "\n",
      "BOS he was in america at that time . EOS\n",
      "BOS 他 当 时 在 美 国 。 EOS\n",
      "translation: 他 当 时 在 美 国 。\n",
      "\n",
      "BOS i bought it at a department store . EOS\n",
      "BOS 我 在 百 货 公 司 买 了 它 。 EOS\n",
      "translation: 我 在 百 货 公 司 买 了 它 。\n",
      "\n",
      "BOS it is going to be cold tonight . EOS\n",
      "BOS 今 晚 会 很 冷 。 EOS\n",
      "translation: 今 晚 会 很 冷 。\n",
      "\n",
      "BOS he asked me why i was laughing . EOS\n",
      "BOS 他 问 我 为 什 么 在 笑 。 EOS\n",
      "translation: 他 问 我 为 什 么 在 笑 。\n",
      "\n",
      "BOS there was no furniture in that room . EOS\n",
      "BOS 那 个 房 间 里 没 有 家 俱 。 EOS\n",
      "translation: 那 个 房 间 里 没 有 家 俱 。\n",
      "\n",
      "BOS he made himself heard across the room . EOS\n",
      "BOS 他 让 对 面 的 房 间 都 能 听 到 他 的 声 音 。 EOS\n",
      "translation: 他 让 对 面 的 房 间 都 能 听 到 他 的 声 音 。\n",
      "\n",
      "BOS she left her umbrella in the train . EOS\n",
      "BOS 她 把 她 的 雨 伞 留 在 火 车 上 了 。 EOS\n",
      "translation: 她 把 她 的 雨 伞 留 在 火 车 上 了 。\n",
      "\n",
      "BOS i do n't feel much like talking . EOS\n",
      "BOS 我 不 太 想 说 话 。 EOS\n",
      "translation: 我 不 太 想 说 话 。\n",
      "\n",
      "BOS i go to bed after i study . EOS\n",
      "BOS 我 读 完 书 之 后 就 去 睡 觉 。 EOS\n",
      "translation: 我 读 完 书 之 后 就 去 睡 觉 。\n",
      "\n",
      "BOS my mother made me a white dress . EOS\n",
      "BOS 我 妈 妈 为 我 做 了 一 件 白 色 的 洋 装 。 EOS\n",
      "translation: 我 妈 妈 为 我 做 了 一 件 白 色 的 洋 装 。\n",
      "\n",
      "BOS it 's likely to snow this evening . EOS\n",
      "BOS 今 天 晚 上 可 能 会 下 雪 。 EOS\n",
      "translation: 今 天 晚 上 可 能 会 下 雪 。\n",
      "\n",
      "BOS he 's started looking for a job . EOS\n",
      "BOS 他 开 始 找 工 作 了 。 EOS\n",
      "translation: 他 开 始 找 工 作 了 。\n",
      "\n",
      "BOS i went without food for a week . EOS\n",
      "BOS 我 一 星 期 没 带 食 物 去 。 EOS\n",
      "translation: 我 一 星 期 没 带 食 物 去 。\n",
      "\n",
      "BOS you seem like a very smart person . EOS\n",
      "BOS 你 看 起 来 像 一 个 聪 明 人 。 EOS\n",
      "translation: 你 看 起 来 像 一 个 聪 明 人 。\n",
      "\n",
      "BOS we got to the station at six . EOS\n",
      "BOS 我 们 六 点 钟 到 了 车 站 。 EOS\n",
      "translation: 我 们 六 点 钟 到 了 车 站 。\n",
      "\n",
      "BOS did tom use to be a teacher ? EOS\n",
      "BOS 汤 姆 以 前 是 教 师 吗 ？ EOS\n",
      "translation: 汤 姆 以 前 是 教 师 吗 ？\n",
      "\n",
      "BOS you need to let me handle this . EOS\n",
      "BOS 你 要 让 我 来 解 决 。 EOS\n",
      "translation: 你 要 让 我 来 解 决 。\n",
      "\n",
      "BOS i get depressed by the slightest things . EOS\n",
      "BOS 我 为 小 事 情 觉 得 沮 丧 。 EOS\n",
      "translation: 我 为 小 事 情 觉 得 沮 丧 。\n",
      "\n",
      "BOS i can not do without her help . EOS\n",
      "BOS 没 有 她 的 帮 助 我 做 不 到 。 EOS\n",
      "translation: 没 有 她 的 帮 助 我 做 不 了 。\n",
      "\n",
      "BOS i did n't know she was married . EOS\n",
      "BOS 我 不 知 道 她 结 婚 了 。 EOS\n",
      "translation: 我 不 知 道 她 结 婚 了 。\n",
      "\n",
      "BOS tom will not know what to do . EOS\n",
      "BOS 汤 姆 不 会 知 道 要 做 甚 么 。 EOS\n",
      "translation: 汤 姆 不 会 知 道 要 做 什 么 。\n",
      "\n",
      "BOS i will show you to the station . EOS\n",
      "BOS 我 会 告 诉 你 去 车 站 的 路 。 EOS\n",
      "translation: 我 会 带 你 去 车 站 的 。\n",
      "\n",
      "BOS i 've heard you 've been sick . EOS\n",
      "BOS 我 听 说 你 病 了 。 EOS\n",
      "translation: 我 听 说 你 病 了 。\n",
      "\n",
      "BOS his parents took him for a walk . EOS\n",
      "BOS 他 的 父 母 带 他 去 散 步 。 EOS\n",
      "translation: 他 的 父 母 为 他 散 步 。\n",
      "\n",
      "BOS he is old enough to understand it . EOS\n",
      "BOS 他 已 经 到 了 能 了 解 它 的 年 纪 。 EOS\n",
      "translation: 他 年 纪 够 大 可 以 了 解 它 。\n",
      "\n",
      "BOS he always mistakes me for my sister . EOS\n",
      "BOS 他 老 是 把 我 和 我 姐 姐 搞 错 。 EOS\n",
      "translation: 他 老 是 把 我 和 我 姐 姐 姐 姐 。\n",
      "\n",
      "BOS tom came to boston three years ago . EOS\n",
      "BOS 汤 姆 三 年 前 去 了 波 士 顿 。 EOS\n",
      "translation: 汤 姆 三 年 前 去 了 波 士 顿 。\n",
      "\n",
      "BOS there is an orange on the table . EOS\n",
      "BOS 桌 上 有 一 只 橙 子 。 EOS\n",
      "translation: 桌 上 有 一 只 橙 子 。\n",
      "\n",
      "BOS i did n't mean to be UNK . EOS\n",
      "BOS 自 私 不 是 我 的 本 意 。 EOS\n",
      "translation: 我 没 有 意 思 。\n",
      "\n",
      "BOS my grandmother can not see very well . EOS\n",
      "BOS 我 的 祖 母 无 法 看 得 很 清 楚 。 EOS\n",
      "translation: 我 的 祖 母 无 法 看 得 很 清 楚 。\n",
      "\n",
      "BOS this textbook is written in simple english . EOS\n",
      "BOS 这 本 教 科 书 是 用 简 单 的 英 语 写 的 。 EOS\n",
      "translation: 这 本 教 科 书 是 用 简 单 的 英 语 写 的 。\n",
      "\n",
      "BOS there is little chance of his winning . EOS\n",
      "BOS 他 赢 的 机 会 极 小 。 EOS\n",
      "translation: 他 赢 的 机 会 极 小 。\n",
      "\n",
      "BOS i 'm not very particular about food . EOS\n",
      "BOS 我 对 食 物 不 是 很 讲 究 。 EOS\n",
      "translation: 我 对 食 物 不 是 很 讲 究 。\n",
      "\n",
      "BOS tom is perfect , is n't he ? EOS\n",
      "BOS 汤 姆 是 完 美 的 ， 不 是 么 ？ EOS\n",
      "translation: 汤 姆 是 完 美 的 ， 不 是 么 ？\n",
      "\n",
      "BOS i guess most of them went home . EOS\n",
      "BOS 我 猜 他 们 大 多 数 回 家 了 。 EOS\n",
      "translation: 我 猜 他 们 大 多 数 回 家 了 。\n",
      "\n",
      "BOS she will UNK me on the piano . EOS\n",
      "BOS 她 会 弹 钢 琴 为 我 伴 奏 。 EOS\n",
      "translation: 她 会 弹 钢 琴 为 我 伴 奏 。\n",
      "\n",
      "BOS a UNK test showed he was innocent . EOS\n",
      "BOS D N A 检 测 表 明 ， 他 是 无 辜 的 。 EOS\n",
      "translation: D N A 检 测 表 明 ， 他 是 无 辜 的 。\n",
      "\n",
      "BOS do you need to work on sunday ? EOS\n",
      "BOS 你 周 日 需 要 去 工 作 吗 ？ EOS\n",
      "translation: 你 周 日 需 要 做 工 作 吗 ？\n",
      "\n",
      "BOS he is glad to hear the news . EOS\n",
      "BOS 他 很 高 兴 听 到 这 个 消 息 。 EOS\n",
      "translation: 他 很 高 兴 听 到 这 个 消 息 。\n",
      "\n",
      "BOS tom did n't want to do that . EOS\n",
      "BOS 汤 姆 不 想 去 做 那 件 事 。 EOS\n",
      "translation: 汤 姆 不 想 去 做 那 件 事 。\n",
      "\n",
      "BOS i had a pleasant dream last night . EOS\n",
      "BOS 我 昨 晚 作 了 一 个 好 梦 。 EOS\n",
      "translation: 昨 晚 ， 我 作 了 一 个 好 梦 。\n",
      "\n",
      "BOS she still has n't heard this news . EOS\n",
      "BOS 她 还 没 听 到 这 个 消 息 。 EOS\n",
      "translation: 她 还 没 听 到 这 个 消 息 。\n",
      "\n",
      "BOS mary is going to help us tomorrow . EOS\n",
      "BOS 玛 丽 明 天 会 帮 我 们 。 EOS\n",
      "translation: 玛 丽 明 天 会 帮 我 们 。\n",
      "\n",
      "BOS my father got married in his twenties . EOS\n",
      "BOS 我 父 亲 在 他 二 十 多 岁 时 结 婚 了 。 EOS\n",
      "translation: 我 爸 爸 每 次 去 世 了 。\n",
      "\n",
      "BOS could you call a doctor , please ? EOS\n",
      "BOS 你 能 请 个 医 生 来 吗 ？ EOS\n",
      "translation: 请 你 给 个 医 生 来 好 吗 ？\n",
      "\n",
      "BOS what do you usually do after dinner ? EOS\n",
      "BOS 你 通 常 晚 饭 后 做 什 么 ? EOS\n",
      "translation: 你 通 常 晚 饭 后 做 什 么 ?\n",
      "\n",
      "BOS do you have a driver 's license ? EOS\n",
      "BOS 你 有 驾 驶 执 照 吗 ？ EOS\n",
      "translation: 你 有 驾 驶 执 照 吗 ？\n",
      "\n",
      "BOS she was amazed to hear the news . EOS\n",
      "BOS 她 听 到 那 个 消 息 后 惊 呆 了 。 EOS\n",
      "translation: 她 听 到 这 个 消 息 后 惊 呆 了 。\n",
      "\n",
      "BOS i did n't study at all yesterday . EOS\n",
      "BOS 我 昨 天 都 没 读 书 。 EOS\n",
      "translation: 我 昨 天 都 没 读 书 。\n",
      "\n",
      "BOS fish can not live out of water . EOS\n",
      "BOS 鱼 离 开 水 就 无 法 生 存 。 EOS\n",
      "translation: 鱼 离 开 水 就 无 法 生 存 。\n",
      "\n",
      "BOS i have an urgent message from tom . EOS\n",
      "BOS 我 收 到 汤 姆 的 一 条 紧 急 消 息 。 EOS\n",
      "translation: 我 收 到 汤 姆 的 一 条 紧 急 消 息 。\n",
      "\n",
      "BOS he 's gone to nagoya on business . EOS\n",
      "BOS 他 因 公 出 差 到 名 古 屋 。 EOS\n",
      "translation: 他 去 名 古 屋 出 差 。\n",
      "\n",
      "BOS that 's an interesting piece of information . EOS\n",
      "BOS 这 是 条 有 趣 的 信 息 。 EOS\n",
      "translation: 那 是 条 有 趣 的 信 息 。\n",
      "\n",
      "BOS i do n't want to go alone . EOS\n",
      "BOS 我 不 想 独 自 前 往 。 EOS\n",
      "translation: 我 不 想 独 自 去 。\n",
      "\n",
      "BOS there 's nothing else we can do . EOS\n",
      "BOS 没 有 我 们 能 还 做 的 事 。 EOS\n",
      "translation: 没 有 我 们 能 无 法 做 的 事 。\n",
      "\n",
      "BOS he 's staying at his aunt 's . EOS\n",
      "BOS 他 呆 在 他 阿 姨 家 。 EOS\n",
      "translation: 他 呆 在 他 阿 姨 家 。\n",
      "\n",
      "BOS you should be able to UNK it . EOS\n",
      "BOS 你 应 该 能 解 决 。 EOS\n",
      "translation: 你 应 该 能 解 决 。\n",
      "\n",
      "BOS the couple is walking hand in hand . EOS\n",
      "BOS 这 对 夫 妻 手 牵 手 走 路 。 EOS\n",
      "translation: 这 对 夫 妻 手 牵 手 走 路 。\n",
      "\n",
      "BOS we had a heavy frost this morning . EOS\n",
      "BOS 今 天 早 上 天 气 严 寒 。 EOS\n",
      "translation: 今 天 早 上 下 厚 霜 了 。\n",
      "\n",
      "BOS rome was not built in a day . EOS\n",
      "BOS 罗 马 不 是 一 天 建 成 的 。 EOS\n",
      "translation: 罗 马 不 是 一 天 建 成 的 。\n",
      "\n",
      "BOS the boy made fun of the girl . EOS\n",
      "BOS 这 个 男 孩 取 笑 了 这 个 女 孩 。 EOS\n",
      "translation: 这 个 男 孩 取 笑 了 这 个 女 孩 。\n",
      "\n",
      "BOS he lives in that house over there . EOS\n",
      "BOS 他 住 在 那 边 的 那 间 房 子 。 EOS\n",
      "translation: 他 住 在 那 边 的 那 间 房 子 。\n",
      "\n",
      "BOS he 's always worrying about his daughter . EOS\n",
      "BOS 他 一 直 很 担 心 他 的 女 儿 。 EOS\n",
      "translation: 他 一 直 很 担 心 他 的 女 儿 。\n",
      "\n",
      "BOS he retired at the age of UNK . EOS\n",
      "BOS 他 6 5 岁 退 了 休 。 EOS\n",
      "translation: 他 6 5 岁 退 了 休 。\n",
      "\n",
      "BOS tom should n't have been here today . EOS\n",
      "BOS T o m 今 天 不 该 在 这 。 EOS\n",
      "translation: 汤 姆 今 天 不 应 该 在 这 里 。\n",
      "\n",
      "BOS help yourself to a piece of cake . EOS\n",
      "BOS 你 们 自 己 吃 蛋 糕 。 EOS\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translation: 随 意 吃 点 蛋 糕 。\n",
      "\n",
      "BOS i helped my father wash his car . EOS\n",
      "BOS 我 帮 我 父 亲 洗 他 的 车 。 EOS\n",
      "translation: 我 帮 我 父 亲 洗 他 的 车 。\n",
      "\n",
      "BOS the bus stop is near our school . EOS\n",
      "BOS 巴 士 站 离 我 们 的 学 校 近 。 EOS\n",
      "translation: 巴 士 站 离 我 们 的 学 校 近 。\n",
      "\n",
      "BOS i was afraid i would be late . EOS\n",
      "BOS 我 怕 我 会 迟 到 。 EOS\n",
      "translation: 我 怕 我 会 迟 到 。\n",
      "\n",
      "BOS i plan to buy him a pen . EOS\n",
      "BOS 我 打 算 给 他 买 一 支 钢 笔 。 EOS\n",
      "translation: 我 打 算 给 他 买 一 支 钢 笔 。\n",
      "\n",
      "BOS my mother is a very good cook . EOS\n",
      "BOS 我 妈 妈 的 厨 艺 很 好 。 EOS\n",
      "translation: 我 妈 妈 是 个 很 棒 的 厨 师 。\n",
      "\n",
      "BOS she is busy preparing for the trip . EOS\n",
      "BOS 她 正 忙 著 准 备 这 次 旅 行 。 EOS\n",
      "translation: 她 正 忙 著 准 备 这 次 旅 游 。\n",
      "\n",
      "BOS tom is unlikely to do that again . EOS\n",
      "BOS 汤 姆 不 太 可 能 再 那 么 做 了 。 EOS\n",
      "translation: 汤 姆 不 太 可 能 再 那 么 做 了 。\n",
      "\n",
      "BOS the flood caused a lot of damage . EOS\n",
      "BOS 洪 水 造 成 了 很 大 的 损 害 。 EOS\n",
      "translation: 洪 水 造 成 了 很 大 的 损 害 。\n",
      "\n",
      "BOS you do n't need to study today . EOS\n",
      "BOS 你 今 天 不 必 读 书 。 EOS\n",
      "translation: 你 今 天 不 必 读 书 。\n",
      "\n",
      "BOS how many UNK are there in japan ? EOS\n",
      "BOS 日 本 有 多 少 个 县 ? EOS\n",
      "translation: 日 本 有 多 少 个 县 ?\n",
      "\n",
      "BOS he works hard all the year round . EOS\n",
      "BOS 他 一 年 到 头 努 力 工 作 。 EOS\n",
      "translation: 他 一 年 纪 , 工 作 都 很 努 力 。\n",
      "\n",
      "BOS he did n't agree to my proposal . EOS\n",
      "BOS 他 不 同 意 我 的 建 议 。 EOS\n",
      "translation: 他 不 同 意 我 的 建 议 。\n",
      "\n",
      "BOS i grew up in this small town . EOS\n",
      "BOS 我 在 这 个 小 镇 上 长 大 的 。 EOS\n",
      "translation: 我 在 这 个 小 镇 上 长 大 的 。\n",
      "\n",
      "BOS attack is the best form of UNK . EOS\n",
      "BOS 进 攻 是 最 好 的 防 UNK 。 EOS\n",
      "translation: 进 攻 是 最 好 的 防 造 的 。\n",
      "\n",
      "BOS when can we eat , i wonder . EOS\n",
      "BOS 我 不 知 道 我 们 什 么 时 候 可 以 吃 。 EOS\n",
      "translation: 当 我 也 能 吃 饭 时 ， 我 可 以 吃 。\n",
      "\n",
      "BOS what fruit do you like the best ? EOS\n",
      "BOS 你 最 喜 欢 什 么 水 果 ？ EOS\n",
      "translation: 你 们 最 喜 欢 什 么 水 果 吗 ?\n",
      "\n",
      "BOS i 'm afraid it will not work . EOS\n",
      "BOS 我 怕 它 是 行 不 通 的 。 EOS\n",
      "translation: 我 怕 它 是 行 不 通 的 。\n",
      "\n",
      "BOS i am willing to attend the meeting . EOS\n",
      "BOS 我 愿 意 参 加 这 场 会 议 。 EOS\n",
      "translation: 我 愿 意 参 加 这 场 会 议 。\n",
      "\n",
      "BOS who 's the man playing the piano ? EOS\n",
      "BOS 那 个 正 在 弹 钢 琴 的 男 人 是 谁 ? EOS\n",
      "translation: 谁 在 弹 钢 琴 ？\n",
      "\n",
      "BOS many kinds of birds live in japan . EOS\n",
      "BOS 许 多 种 鸟 类 住 在 日 本 。 EOS\n",
      "translation: 许 多 种 鸟 类 住 在 日 本 。\n",
      "\n",
      "BOS it seems less crowded during the week . EOS\n",
      "BOS 平 日 似 乎 比 较 不 拥 挤 。 EOS\n",
      "translation: 平 日 似 乎 比 较 不 拥 挤 。\n",
      "\n",
      "BOS here is a photograph of my family . EOS\n",
      "BOS 这 里 有 一 张 我 全 家 的 照 片 。 EOS\n",
      "translation: 这 里 有 一 张 我 全 家 的 照 片 。\n",
      "\n",
      "BOS what were tom and mary doing here ? EOS\n",
      "BOS 汤 姆 和 玛 丽 在 这 里 做 了 甚 么 ？ EOS\n",
      "translation: 汤 姆 和 玛 丽 在 这 里 做 甚 么 ？\n",
      "\n",
      "BOS visiting a foreign country must be expensive . EOS\n",
      "BOS 到 外 国 一 定 很 贵 。 EOS\n",
      "translation: 到 外 国 一 定 很 贵 。\n",
      "\n",
      "BOS i do n't know either of them . EOS\n",
      "BOS 他 们 中 的 任 何 一 位 我 都 不 认 识 。 EOS\n",
      "translation: 我 不 认 识 他 们 。\n",
      "\n",
      "BOS what did you do with my luggage ? EOS\n",
      "BOS 你 把 我 的 行 李 放 到 哪 里 去 了 ？ EOS\n",
      "translation: 你 把 我 的 行 李 放 到 哪 里 去 了 ？\n",
      "\n",
      "BOS do you think this thing is cute ? EOS\n",
      "BOS 你 觉 得 这 个 东 西 UNK 吗 ？ EOS\n",
      "translation: 你 觉 得 这 个 东 西 是 有 什 么 吗 ？\n",
      "\n",
      "BOS she was very proud of her father . EOS\n",
      "BOS 她 非 常 地 以 她 父 亲 为 荣 。 EOS\n",
      "translation: 她 为 她 的 父 亲 感 到 非 常 骄 傲 。\n",
      "\n",
      "BOS he came from a tiny mountain town . EOS\n",
      "BOS 他 来 自 一 个 小 山 城 。 EOS\n",
      "translation: 他 来 自 一 个 小 山 城 。\n",
      "\n",
      "BOS you 've got a lot of UNK . EOS\n",
      "BOS 你 胆 子 很 大 。 EOS\n",
      "translation: 你 胆 子 很 大 。\n",
      "\n",
      "BOS tom is living with his uncle now . EOS\n",
      "BOS 汤 姆 现 在 跟 他 叔 叔 住 在 一 起 。 EOS\n",
      "translation: 汤 姆 现 在 跟 他 叔 住 在 一 起 。\n",
      "\n",
      "BOS i worked three years as tom 's assistant . EOS\n",
      "BOS 我 给 汤 姆 当 了 三 年 助 手 。 EOS\n",
      "translation: 我 给 汤 姆 当 了 三 年 助 手 。\n",
      "\n",
      "BOS nobody knows what will happen in the future . EOS\n",
      "BOS 人 不 知 道 未 来 可 能 发 生 什 么 事 。 EOS\n",
      "translation: 没 有 人 知 道 未 来 会 发 生 什 么 事 。\n",
      "\n",
      "BOS tom fell into a UNK of UNK acid . EOS\n",
      "BOS T o m 摔 倒 进 一 桶 硫 酸 中 EOS\n",
      "translation: T o m 摔 倒 进 一 桶 硫 酸 中\n",
      "\n",
      "BOS he likes mathematics , but i do n't . EOS\n",
      "BOS 他 喜 欢 数 学 ， 但 我 不 喜 欢 。 EOS\n",
      "translation: 他 喜 欢 数 学 ， 但 我 不 喜 欢 。\n",
      "\n",
      "BOS tom does everything he can to save money . EOS\n",
      "BOS 汤 姆 尽 全 力 省 钱 。 EOS\n",
      "translation: 汤 姆 尽 全 力 省 钱 。\n",
      "\n",
      "BOS you may depend on him to help you . EOS\n",
      "BOS 你 或 许 可 以 依 靠 他 来 帮 助 你 。 EOS\n",
      "translation: 你 或 许 可 以 依 靠 他 来 帮 助 你 。\n",
      "\n",
      "BOS UNK ! it 's been a long time . EOS\n",
      "BOS UNK ! 好 久 不 见 了 。 EOS\n",
      "translation: 已 经 很 长 一 段 时 间 了 。\n",
      "\n",
      "BOS these tools are used for building a house . EOS\n",
      "BOS 这 些 工 具 是 用 来 造 房 子 的 。 EOS\n",
      "translation: 这 些 工 具 是 用 来 造 房 子 的 。\n",
      "\n",
      "BOS i would rather walk than take a bus . EOS\n",
      "BOS 我 宁 愿 走 路 胜 过 搭 公 车 。 EOS\n",
      "translation: 我 宁 愿 走 路 胜 过 搭 公 车 。\n",
      "\n",
      "BOS it 's on the tip of my tongue . EOS\n",
      "BOS 它 在 我 的 舌 尖 上 。 EOS\n",
      "translation: 它 在 我 的 舌 尖 上 。\n",
      "\n",
      "BOS our music teacher advised me to visit UNK . EOS\n",
      "BOS 我 们 的 音 乐 老 师 建 议 我 去 维 也 纳 看 看 。 EOS\n",
      "translation: 我 们 的 音 乐 老 师 建 议 我 去 维 也 纳 看 看 看 看 。\n",
      "\n",
      "BOS the man you saw yesterday was my uncle . EOS\n",
      "BOS 你 昨 天 看 到 的 那 个 男 人 是 我 叔 叔 。 EOS\n",
      "translation: 你 昨 天 看 到 的 那 个 男 人 是 我 叔 。\n",
      "\n",
      "BOS tom 's eyes were UNK to the UNK . EOS\n",
      "BOS T o m 的 眼 睛 被 UNK 幕 吸 引 住 了 。 EOS\n",
      "translation: T o m 的 眼 睛 被 命 合 。\n",
      "\n",
      "BOS i would like to go abroad one day . EOS\n",
      "BOS 我 想 有 一 天 去 国 外 。 EOS\n",
      "translation: 我 想 有 一 天 去 国 外 。\n",
      "\n",
      "BOS i 've got to try to find tom . EOS\n",
      "BOS 我 试 图 找 到 T o m 。 EOS\n",
      "translation: 我 试 著 试 图 找 到 汤 姆 。\n",
      "\n",
      "BOS i consider you one of my UNK friends . EOS\n",
      "BOS 我 把 你 视 为 我 最 要 好 的 朋 友 之 一 。 EOS\n",
      "translation: 我 把 你 视 为 我 最 要 好 的 朋 友 之 一 。\n",
      "\n",
      "BOS my aunt made a new skirt for me . EOS\n",
      "BOS 我 UNK UNK 做 了 一 条 新 裙 子 给 我 。 EOS\n",
      "translation: 我 姑 姑 做 了 一 条 新 裙 子 给 我 。\n",
      "\n",
      "BOS why do you want to become a nurse ? EOS\n",
      "BOS 你 为 什 么 想 成 为 护 士 呢 ？ EOS\n",
      "translation: 你 为 什 么 想 成 为 护 士 呢 ？\n",
      "\n",
      "BOS could you help me translate this into french ? EOS\n",
      "BOS 你 能 帮 我 把 这 译 成 法 语 吗 ？ EOS\n",
      "translation: 你 能 帮 我 把 这 译 成 法 语 吗 ？\n",
      "\n",
      "BOS the tiger UNK looked like a large kitten . EOS\n",
      "BOS 小 老 虎 看 起 来 像 只 大 猫 。 EOS\n",
      "translation: 小 老 虎 看 起 来 像 只 大 猫 。\n",
      "\n",
      "BOS she 's really smart , is n't she ? EOS\n",
      "BOS 她 真 的 很 聪 明 ， 不 是 吗 ？ EOS\n",
      "translation: 她 真 的 很 聪 明 ， 不 是 吗 ？\n",
      "\n",
      "BOS that 's a nice tie you 're wearing . EOS\n",
      "BOS 你 打 了 一 条 不 错 的 领 带 。 EOS\n",
      "translation: 你 打 这 条 领 带 很 不 错 。\n",
      "\n",
      "BOS it 's impossible to get there by noon . EOS\n",
      "BOS 中 午 到 达 那 里 是 不 可 能 的 。 EOS\n",
      "translation: 中 午 到 达 那 里 是 不 可 能 的 。\n",
      "\n",
      "BOS she got first prize in the eating contest . EOS\n",
      "BOS 她 在 吃 东 西 比 赛 里 得 了 一 等 奖 。 EOS\n",
      "translation: 她 在 吃 东 西 比 赛 里 得 了 一 等 奖 。\n",
      "\n",
      "BOS i 'm not sure what i was thinking . EOS\n",
      "BOS 我 不 确 定 当 时 我 正 在 想 什 么 。 EOS\n",
      "translation: 我 不 确 定 我 在 想 什 么 。\n",
      "\n",
      "BOS why did you not tell me the truth ? EOS\n",
      "BOS 为 什 么 你 不 告 诉 我 真 相 ？ EOS\n",
      "translation: 你 为 甚 么 不 告 诉 我 真 相 ？\n",
      "\n",
      "BOS tom is the only boy in our class . EOS\n",
      "BOS 汤 姆 是 我 们 班 里 唯 一 的 男 孩 。 EOS\n",
      "translation: 汤 姆 是 我 们 班 里 唯 一 的 男 孩 。\n",
      "\n",
      "BOS keep your mouth shut and your eyes open . EOS\n",
      "BOS 闭 嘴 看 着 。 EOS\n",
      "translation: 闭 嘴 看 着 。\n",
      "\n",
      "BOS there 's a big fly on the ceiling . EOS\n",
      "BOS 有 一 只 大 苍 蝇 在 天 花 板 上 。 EOS\n",
      "translation: 有 一 只 大 苍 蝇 在 天 花 板 上 。\n",
      "\n",
      "BOS he UNK me to leave the room immediately . EOS\n",
      "BOS 他 命 令 我 立 刻 离 开 这 个 房 间 。 EOS\n",
      "translation: 他 命 令 我 立 刻 离 开 这 个 房 间 。\n",
      "\n",
      "BOS he kept his promise and helped his brothers . EOS\n",
      "BOS 他 履 行 了 他 的 承 诺 ， 并 且 帮 助 了 他 的 兄 弟 。 EOS\n",
      "translation: 他 履 行 了 他 的 承 诺 ， 并 且 帮 助 了 他 的 兄 弟 。\n",
      "\n",
      "BOS it 's UNK to park your car here . EOS\n",
      "BOS 把 车 停 在 这 里 是 违 法 的 。 EOS\n",
      "translation: 在 这 是 你 的 车 站 。\n",
      "\n",
      "BOS would you care for another glass of beer ? EOS\n",
      "BOS 再 来 一 杯 啤 酒 怎 么 样 ？ EOS\n",
      "translation: 再 来 一 杯 啤 酒 怎 么 样 ？\n",
      "\n",
      "BOS he put his hand gently on her shoulder . EOS\n",
      "BOS 他 把 他 的 手 温 柔 地 放 在 她 的 肩 上 。 EOS\n",
      "translation: 他 把 他 的 手 温 柔 地 放 在 她 的 肩 上 。\n",
      "\n",
      "BOS i still can not believe you 're married . EOS\n",
      "BOS 我 仍 然 无 法 相 信 你 结 婚 了 。 EOS\n",
      "translation: 我 仍 然 无 法 相 信 你 结 婚 了 。\n",
      "\n",
      "BOS he has ambition , so he works hard . EOS\n",
      "BOS 他 有 抱 负 ， 所 以 他 很 努 力 工 作 。 EOS\n",
      "translation: 他 有 抱 负 ， 所 以 他 很 努 力 工 作 。\n",
      "\n",
      "BOS i do not know how to use it . EOS\n",
      "BOS 我 不 知 道 如 何 使 用 它 。 EOS\n",
      "translation: 我 不 知 道 怎 么 使 用 它 。\n",
      "\n",
      "BOS the man and his wife helped each other . EOS\n",
      "BOS 这 男 人 和 他 妻 子 互 相 帮 助 。 EOS\n",
      "translation: 这 男 人 和 他 妻 子 互 相 帮 助 。\n",
      "\n",
      "BOS i 've never met such a kind man . EOS\n",
      "BOS 我 从 没 遇 到 过 那 么 好 心 的 男 人 。 EOS\n",
      "translation: 我 从 没 遇 到 过 那 么 好 心 的 男 人 。\n",
      "\n",
      "BOS i paid two thousand yen for the book . EOS\n",
      "BOS 我 用 两 千 日 元 买 了 这 本 书 。 EOS\n",
      "translation: 我 付 了 两 万 日 元 买 这 本 书 。\n",
      "\n",
      "BOS i 'm currently a teacher at this school . EOS\n",
      "BOS 我 现 在 在 这 所 学 校 任 教 。 EOS\n",
      "translation: 我 现 在 这 所 学 校 有 学 校 。\n",
      "\n",
      "BOS i do n't want to have an operation . EOS\n",
      "BOS 我 不 想 接 受 手 术 。 EOS\n",
      "translation: 我 不 想 接 受 手 术 。\n",
      "\n",
      "BOS did you watch the soccer game on television ? EOS\n",
      "BOS 你 在 电 视 上 看 足 球 赛 了 吗 ？ EOS\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translation: 你 在 电 视 上 看 足 球 赛 了 吗 ？\n",
      "\n",
      "BOS we did n't know what to do next . EOS\n",
      "BOS 我 们 不 知 道 下 一 步 要 做 什 么 。 EOS\n",
      "translation: 我 们 不 知 道 下 一 步 要 做 什 么 。\n",
      "\n",
      "BOS she rested her head on her mother 's shoulder . EOS\n",
      "BOS 她 把 头 UNK 在 她 母 亲 的 肩 上 。 EOS\n",
      "translation: 她 把 头 发 的 头 去 了 她 母 亲 的 肩 上 。\n",
      "\n",
      "BOS i do n't know if i have the time . EOS\n",
      "BOS 我 不 知 道 我 有 没 有 时 间 。 EOS\n",
      "translation: 我 不 知 道 我 有 没 有 时 间 。\n",
      "\n",
      "BOS are you planning on eating that all by yourself ? EOS\n",
      "BOS 你 打 算 自 己 一 人 把 它 全 部 吃 了 吗 ？ EOS\n",
      "translation: 你 打 算 自 己 一 人 把 它 全 部 吃 了 吗 ？\n",
      "\n",
      "BOS i will be absent from home in the afternoon . EOS\n",
      "BOS 下 午 我 不 在 家 。 EOS\n",
      "translation: 下 午 我 不 会 呆 在 家 里 。\n",
      "\n",
      "BOS no , thank you . i 've had enough . EOS\n",
      "BOS 不 ， 谢 谢 你 。 我 已 经 吃 饱 了 。 EOS\n",
      "translation: 不 ， 谢 谢 你 。 我 已 经 吃 饱 了 。\n",
      "\n",
      "BOS without the sun , there would be no life . EOS\n",
      "BOS 没 有 太 阳 就 没 有 生 命 。 EOS\n",
      "translation: 没 有 太 阳 就 没 有 生 命 。\n",
      "\n",
      "BOS she stayed in that area for a short while . EOS\n",
      "BOS 她 在 那 地 方 待 了 片 刻 。 EOS\n",
      "translation: 她 在 那 地 方 待 了 片 刻 。\n",
      "\n",
      "BOS we will ask tom and see what he thinks . EOS\n",
      "BOS 我 们 要 问 问 汤 姆 ， 看 看 他 怎 么 想 。 EOS\n",
      "translation: 我 们 会 问 汤 姆 ， 看 看 看 看 看 看 看 见 他 。\n",
      "\n",
      "BOS she seems to be in love with my brother . EOS\n",
      "BOS 她 似 乎 喜 欢 上 了 我 哥 哥 。 EOS\n",
      "translation: 她 似 乎 喜 欢 上 了 我 哥 哥 。\n",
      "\n",
      "BOS there are more girls than boys at our school . EOS\n",
      "BOS 在 我 们 学 校 里 女 孩 比 男 孩 多 。 EOS\n",
      "translation: 我 们 的 大 女 孩 比 较 多 共 用 。\n",
      "\n",
      "BOS come on . i want to show you something . EOS\n",
      "BOS 来 吧 。 我 想 让 你 看 点 东 西 。 EOS\n",
      "translation: 来 吧 。 我 想 让 你 看 点 东 西 。\n",
      "\n",
      "BOS i 've been spending too much time with tom . EOS\n",
      "BOS 我 在 汤 姆 身 上 花 的 时 间 太 多 了 。 EOS\n",
      "translation: 我 在 汤 姆 身 上 花 的 时 间 太 多 了 。\n",
      "\n",
      "BOS do n't laugh at him for making a mistake . EOS\n",
      "BOS 不 要 笑 他 犯 了 错 误 。 EOS\n",
      "translation: 不 要 笑 他 犯 了 错 误 。\n",
      "\n",
      "BOS are you a student of a private high school ? EOS\n",
      "BOS 你 在 私 立 高 中 读 书 吗 ？ EOS\n",
      "translation: 你 在 私 立 高 中 读 书 吗 ？\n",
      "\n",
      "BOS you will soon get accustomed to this cold weather . EOS\n",
      "BOS 你 很 快 就 会 习 惯 这 种 冷 天 气 。 EOS\n",
      "translation: 你 很 快 就 会 习 惯 这 种 冷 天 气 。\n",
      "\n",
      "BOS this is the lady who wants to see you . EOS\n",
      "BOS 这 位 就 是 谁 想 见 你 的 夫 人 。 EOS\n",
      "translation: 这 位 就 是 谁 想 见 你 的 夫 人 。\n",
      "\n",
      "BOS UNK surgery is not covered by most insurance plans . EOS\n",
      "BOS 大 部 份 的 保 险 计 划 不 包 含 整 容 手 术 。 EOS\n",
      "translation: 大 部 份 的 保 险 计 划 不 包 含 整 容 易 。\n",
      "\n",
      "BOS he is UNK than anyone else in this town . EOS\n",
      "BOS 他 比 镇 里 的 其 他 人 都 有 钱 。 EOS\n",
      "translation: 他 比 镇 里 的 其 他 人 都 有 钱 。\n",
      "\n",
      "BOS i am a student , but he is n't . EOS\n",
      "BOS 我 是 个 学 生 ， 但 他 不 是 。 EOS\n",
      "translation: 我 是 个 学 生 ， 但 他 不 是 。\n",
      "\n",
      "BOS it is difficult for me to answer the question . EOS\n",
      "BOS 这 个 问 题 对 我 来 讲 很 难 回 答 。 EOS\n",
      "translation: 这 个 问 题 对 我 来 讲 很 难 回 答 。\n",
      "\n",
      "BOS believe it or not , she has three children . EOS\n",
      "BOS 她 有 三 个 孩 子 ， 信 不 信 由 你 。 EOS\n",
      "translation: 她 不 相 信 汤 姆 有 三 个 孩 子 。\n",
      "\n",
      "BOS my room is three times as large as yours . EOS\n",
      "BOS 我 的 房 间 是 你 的 三 倍 大 。 EOS\n",
      "translation: 我 的 房 间 是 你 的 三 倍 大 。\n",
      "\n",
      "BOS if i were healthy , i would be happy . EOS\n",
      "BOS 如 果 我 很 健 康 ， 我 会 很 快 乐 。 EOS\n",
      "translation: 如 果 我 很 健 康 ， 我 会 很 快 乐 。\n",
      "\n",
      "BOS i do n't want to work under these conditions . EOS\n",
      "BOS 我 不 想 在 这 些 条 件 下 工 作 。 EOS\n",
      "translation: 我 不 想 在 这 些 条 件 下 工 作 。\n",
      "\n",
      "BOS after the accident , tom decided to stop skating . EOS\n",
      "BOS 事 故 过 后 ， 汤 姆 觉 得 停 止 滑 冰 了 。 EOS\n",
      "translation: 事 故 过 后 ， 汤 姆 觉 得 停 止 滑 冰 了 。\n",
      "\n",
      "BOS UNK is a popular game for UNK to play . EOS\n",
      "BOS 大 富 翁 是 一 个 家 庭 玩 的 热 门 游 戏 。 EOS\n",
      "translation: 大 富 翁 是 一 个 家 庭 玩 的 热 门 游 戏 。\n",
      "\n",
      "BOS i wish i could buy you everything you wanted . EOS\n",
      "BOS 我 希 望 我 能 买 下 你 想 要 的 所 有 东 西 。 EOS\n",
      "translation: 我 希 望 我 能 买 下 你 想 要 的 东 西 。\n",
      "\n",
      "BOS almost everyone in our village is related to one another . EOS\n",
      "BOS 我 们 村 里 所 有 的 村 民 几 乎 彼 此 都 是 亲 戚 。 EOS\n",
      "translation: 我 们 村 里 所 有 的 村 民 几 乎 彼 此 都 是 亲 戚 。\n",
      "\n",
      "BOS i know that you still want to be with me . EOS\n",
      "BOS 我 知 道 你 还 是 想 跟 着 我 。 EOS\n",
      "translation: 我 知 道 你 还 是 想 跟 着 我 。\n",
      "\n",
      "BOS he crashed his car because someone UNK with the brakes . EOS\n",
      "BOS 他 撞 车 是 因 为 有 人 在 刹 车 上 做 了 手 脚 。 EOS\n",
      "translation: 他 撞 车 是 因 为 有 人 在 刹 车 上 做 了 手 脚 。\n",
      "\n",
      "BOS i would like to get married to someone like you . EOS\n",
      "BOS 我 想 跟 你 这 样 的 人 结 婚 。 EOS\n",
      "translation: 我 想 跟 你 这 样 的 人 结 婚 。\n",
      "\n",
      "BOS she was advised by him to go to the police . EOS\n",
      "BOS 他 劝 她 去 报 警 。 EOS\n",
      "translation: 她 建 议 他 去 。\n",
      "\n",
      "BOS you 've been late for school more often than before . EOS\n",
      "BOS 你 比 以 前 更 容 易 上 课 迟 到 了 。 EOS\n",
      "translation: 你 比 以 前 一 直 很 晚 还 要 上 学 。\n",
      "\n",
      "BOS you can see a lot of stars in the sky . EOS\n",
      "BOS 你 能 看 到 天 空 中 的 繁 星 。 EOS\n",
      "translation: 你 能 看 到 天 空 中 的 繁 星 。\n",
      "\n",
      "BOS that child may have been UNK on his way home . EOS\n",
      "BOS 那 个 孩 子 可 能 在 回 家 的 路 上 被 绑 架 了 。 EOS\n",
      "translation: 那 孩 子 可 能 在 家 里 被 路 取 消 到 他 的 。\n",
      "\n",
      "BOS i have no idea what i 'm going to wear . EOS\n",
      "BOS 我 不 知 道 穿 甚 么 好 。 EOS\n",
      "translation: 我 不 知 道 穿 什 么 好 意 。\n",
      "\n",
      "BOS he is n't my brother . he 's my cousin . EOS\n",
      "BOS 他 不 是 我 的 兄 弟 。 他 是 我 的 表 弟 。 EOS\n",
      "translation: 他 不 是 我 弟 的 表 弟 。\n",
      "\n",
      "BOS i liked tom 's first book more than the second . EOS\n",
      "BOS 相 比 第 二 本 ， 我 更 喜 欢 汤 姆 的 第 一 本 书 。 EOS\n",
      "translation: 我 喜 欢 汤 姆 的 第 一 本 书 比 第 二 名 。\n",
      "\n",
      "BOS this car was so cheap that he could afford it . EOS\n",
      "BOS 这 车 便 宜 得 让 他 能 买 得 起 。 EOS\n",
      "translation: 这 车 便 宜 得 让 他 能 买 得 很 苍 白 。\n",
      "\n",
      "BOS i can not focus on two things at the same time . EOS\n",
      "BOS 我 不 能 同 时 注 意 两 件 事 。 EOS\n",
      "translation: 我 不 能 同 时 注 意 两 件 事 。\n",
      "\n",
      "BOS this guidebook might be of use to you on your trip . EOS\n",
      "BOS 这 本 导 游 册 子 或 许 会 对 你 的 旅 行 有 帮 助 。 EOS\n",
      "translation: 这 本 导 游 册 子 或 许 会 对 你 的 旅 游 用 。\n",
      "\n",
      "BOS i can hear you , but i can not see you . EOS\n",
      "BOS 我 听 得 见 你 ， 但 我 看 不 见 你 。 EOS\n",
      "translation: 我 听 得 见 你 ， 但 我 看 不 见 你 。\n",
      "\n",
      "BOS all horses are animals , but not all animals are horses . EOS\n",
      "BOS 所 有 的 马 都 是 动 物 ， 但 并 非 所 有 的 动 物 都 是 马 。 EOS\n",
      "translation: 所 有 的 马 都 是 动 物 ， 但 并 非 所 有 的 动 物 都 是 马 。\n",
      "\n",
      "BOS tom is n't nearly as smart as he thinks he is . EOS\n",
      "BOS 汤 姆 并 不 像 他 自 认 为 的 那 么 聪 明 。 EOS\n",
      "translation: 汤 姆 并 不 像 他 自 认 为 的 那 么 聪 明 。\n",
      "\n",
      "BOS how long did it take you to finish reading that book ? EOS\n",
      "BOS 你 看 完 这 本 书 要 多 少 时 间 ？ EOS\n",
      "translation: 你 看 完 这 本 书 要 多 少 时 间 ？\n",
      "\n",
      "BOS i can read german , but i can not speak it . EOS\n",
      "BOS 我 能 看 德 语 ， 但 是 不 能 说 。 EOS\n",
      "translation: 我 能 看 德 语 ， 但 是 不 能 说 。\n",
      "\n",
      "BOS he said that he gets up at 6 o'clock every day . EOS\n",
      "BOS 他 说 他 每 天 早 上 六 点 起 牀 。 EOS\n",
      "translation: 他 说 他 每 天 早 上 六 点 起 牀 。\n",
      "\n",
      "BOS thanks very much for having me to dinner the other night . EOS\n",
      "BOS 谢 谢 那 天 晚 上 请 我 吃 了 饭 。 EOS\n",
      "translation: 谢 谢 那 天 晚 上 我 吃 饭 。\n",
      "\n",
      "BOS hurry up , and you will be in time for the bus . EOS\n",
      "BOS 快 点 ， 你 就 能 准 时 搭 上 公 交 车 了 。 EOS\n",
      "translation: 快 点 ， 你 就 能 准 时 搭 公 交 车 了 。\n",
      "\n",
      "BOS i 'm disappointed that i was n't able to go with her . EOS\n",
      "BOS 我 很 失 望 我 没 能 和 她 一 起 去 。 EOS\n",
      "translation: 我 很 失 望 我 没 能 和 她 一 起 去 。\n",
      "\n",
      "BOS tom said he was n't ready , but mary said she was . EOS\n",
      "BOS 汤 姆 说 他 没 准 备 好 ， 但 玛 丽 说 她 准 备 好 了 。 EOS\n",
      "translation: 汤 姆 说 他 没 准 备 好 ， 但 玛 丽 说 她 准 备 好 了 。\n",
      "\n",
      "BOS i 'm at work now , so i will call you later . EOS\n",
      "BOS 我 现 在 在 上 班 ， 所 以 晚 点 打 给 你 。 EOS\n",
      "translation: 我 现 在 在 在 就 能 打 电 话 给 你 。\n",
      "\n",
      "BOS i had no idea it would put you to so much trouble . EOS\n",
      "BOS 我 一 点 都 不 知 道 这 会 给 你 带 来 那 么 多 问 题 。 EOS\n",
      "translation: 我 一 点 都 不 知 道 这 会 给 你 带 来 那 么 多 问 题 。\n",
      "\n",
      "BOS tom told mary he could n't do what she asked him to do . EOS\n",
      "BOS 汤 姆 告 诉 玛 丽 他 不 能 做 她 要 他 做 的 事 。 EOS\n",
      "translation: 汤 姆 告 诉 玛 丽 他 不 能 做 她 要 他 做 的 事 。\n",
      "\n",
      "BOS it 's a good thing to read good books when you are young . EOS\n",
      "BOS 年 轻 的 时 候 多 看 点 好 书 是 件 好 事 。 EOS\n",
      "translation: 年 轻 的 时 候 多 看 点 好 书 是 件 好 事 。\n",
      "\n",
      "BOS she lost her way and on top of that it began to rain . EOS\n",
      "BOS 她 迷 路 了 ， 紧 接 着 天 开 始 下 雨 了 。 EOS\n",
      "translation: 她 迷 路 了 ， 紧 接 着 天 开 始 下 雨 了 。\n",
      "\n",
      "BOS the best way to lose weight is to eat less and exercise more . EOS\n",
      "BOS 最 好 的 减 肥 方 法 就 是 少 吃 多 运 动 。 EOS\n",
      "translation: 最 好 的 减 肥 方 法 就 是 少 吃 多 运 动 。\n",
      "\n",
      "BOS you 've got to set the alarm clock before you go to bed . EOS\n",
      "BOS 在 你 上 牀 睡 觉 之 前 ， 你 必 须 把 闹 钟 设 定 好 。 EOS\n",
      "translation: 在 你 上 牀 睡 觉 之 前 ， 你 必 须 把 闹 钟 设 定 好 。\n",
      "\n",
      "BOS `` the phone is ringing . `` `` i will get it . `` EOS\n",
      "BOS “ 电 话 响 了 。 ” “ 我 去 接 。 ” EOS\n",
      "translation: “ 电 话 响 了 。 ” “ 我 去 接 。 ”\n",
      "\n",
      "BOS the teacher UNK her finger at me and asked me to come with her . EOS\n",
      "BOS 教 师 用 手 指 指 着 我 ， 要 我 跟 她 走 。 EOS\n",
      "translation: 老 师 用 手 指 指 指 指 指 指 指 指 指 给 我 ， 要 我 。\n",
      "\n",
      "BOS tom was admiring my new car at the time the truck crashed into it . EOS\n",
      "BOS 在 汤 姆 欣 赏 我 的 新 车 的 同 时 ， 一 辆 卡 车 撞 了 过 来 。 EOS\n",
      "translation: 在 汤 姆 欣 赏 我 的 新 车 的 同 意 了 。\n",
      "\n",
      "BOS i do n't think that it 's going to be easy to find tom . EOS\n",
      "BOS 我 认 为 ， 要 找 到 汤 姆 可 不 容 易 。 EOS\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translation: 我 认 为 ， 要 找 到 汤 姆 可 不 容 易 。\n",
      "\n",
      "BOS we 're out of tissue paper , so i 've got to go buy some . EOS\n",
      "BOS 卫 生 纸 用 完 了 ， 我 必 须 去 买 。 EOS\n",
      "translation: 卫 生 纸 用 完 了 ， 我 必 须 去 买 。\n",
      "\n",
      "BOS if a person has not had a chance to UNK his target language by the time he 's an adult , he 's unlikely to be able to reach native speaker level in that language . EOS\n",
      "BOS 如 果 一 个 人 在 成 人 前 没 有 机 会 习 得 目 标 语 言 ， 他 对 该 语 言 的 认 识 达 到 母 语 者 程 度 的 机 会 是 相 当 小 的 。 EOS\n",
      "translation: 如 果 一 个 人 在 成 人 前 没 有 机 会 习 得 目 标 语 言 ， 他 对 该 语 言 的 认 识 达 到 母 语 者 程 度 的 语 言 的\n",
      "<<<<<<< finished evaluate, cost 177.0160 seconds\n"
     ]
    }
   ],
   "source": [
    "# 预测\n",
    "# 加载模型\n",
    "model.load_state_dict(torch.load(SAVE_FILE))\n",
    "# 开始预测\n",
    "print(\">>>>>>> start evaluate\")\n",
    "evaluate_start  = time.time()\n",
    "evaluate(data, model)\n",
    "print(f\"<<<<<<< finished evaluate, cost {time.time()-evaluate_start:.4f} seconds\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}