alignment_assessment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assessing the quality of the alignments\n",
    "\n",
    "This notebook looks at the quality of the alignments between the OCR text and the human-corrected text from the Trove dataset that takes place in the `aligning_trove.ipynb` notebook. It checks the ratio of characters that have been aligned in contrast with the total number of characters, and produces a CSV file for quick evaluation of the alignment of a sample of 50 articles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampledf = pd.read_pickle(\"trove_subsample_aligned.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filePath</th>\n",
       "      <th>articleId</th>\n",
       "      <th>articleType</th>\n",
       "      <th>year</th>\n",
       "      <th>ocrText</th>\n",
       "      <th>humanText</th>\n",
       "      <th>corrected</th>\n",
       "      <th>str_similarity</th>\n",
       "      <th>str_length</th>\n",
       "      <th>quality_band</th>\n",
       "      <th>use_corrected</th>\n",
       "      <th>alignment</th>\n",
       "      <th>processed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>26185</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14330048</td>\n",
       "      <td>Article</td>\n",
       "      <td>1900</td>\n",
       "      <td>WESTERN SUBURBS COTTAGE HOSPITAL. ,-«A-. Tim c...</td>\n",
       "      <td>WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...</td>\n",
       "      <td>WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...</td>\n",
       "      <td>0.920732</td>\n",
       "      <td>1144</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 7, 0, 7), (8, 15, 8, 15), (16, 23, 16, 23...</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13445</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>13166969</td>\n",
       "      <td>Article</td>\n",
       "      <td>1868</td>\n",
       "      <td>INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...</td>\n",
       "      <td>INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...</td>\n",
       "      <td>INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...</td>\n",
       "      <td>0.924826</td>\n",
       "      <td>2155</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 13, 0, 13), (14, 19, 14, 19), (20, 31, 20...</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13729</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14953246</td>\n",
       "      <td>Article</td>\n",
       "      <td>1908</td>\n",
       "      <td>IN DIVORCE. (Before Mr. Justlco Simpson.) O'BR...</td>\n",
       "      <td>IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...</td>\n",
       "      <td>IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...</td>\n",
       "      <td>0.929938</td>\n",
       "      <td>3240</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 2, 0, 2), (3, 11, 3, 11), (12, 19, 12, 19...</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13979</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14947605</td>\n",
       "      <td>Article</td>\n",
       "      <td>1908</td>\n",
       "      <td>I MILITAEY. MELBOURNE, Saturday. The following...</td>\n",
       "      <td>MILITARY. MELBOURNE, Saturday. The following n...</td>\n",
       "      <td>MILITARY. MELBOURNE, Saturday. The following n...</td>\n",
       "      <td>0.921942</td>\n",
       "      <td>2563</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(2, 11, 0, 9), (12, 22, 10, 20), (23, 32, 21,...</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22979</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>15708305</td>\n",
       "      <td>Article</td>\n",
       "      <td>1917</td>\n",
       "      <td>I WAR CASUALTIES. -» j KILLED. ! PRIVATE W. PR...</td>\n",
       "      <td>WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...</td>\n",
       "      <td>WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...</td>\n",
       "      <td>0.915966</td>\n",
       "      <td>578</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(6, 17, 4, 15), (23, 30, 19, 26), (44, 50, 38...</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                filePath articleId  \\\n",
       "26185  ./trove_overproof/datasets/dataset1/rawTextAnd...  14330048   \n",
       "13445  ./trove_overproof/datasets/dataset1/rawTextAnd...  13166969   \n",
       "13729  ./trove_overproof/datasets/dataset1/rawTextAnd...  14953246   \n",
       "13979  ./trove_overproof/datasets/dataset1/rawTextAnd...  14947605   \n",
       "22979  ./trove_overproof/datasets/dataset1/rawTextAnd...  15708305   \n",
       "\n",
       "      articleType  year                                            ocrText  \\\n",
       "26185     Article  1900  WESTERN SUBURBS COTTAGE HOSPITAL. ,-«A-. Tim c...   \n",
       "13445     Article  1868  INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...   \n",
       "13729     Article  1908  IN DIVORCE. (Before Mr. Justlco Simpson.) O'BR...   \n",
       "13979     Article  1908  I MILITAEY. MELBOURNE, Saturday. The following...   \n",
       "22979     Article  1917  I WAR CASUALTIES. -» j KILLED. ! PRIVATE W. PR...   \n",
       "\n",
       "                                               humanText  \\\n",
       "26185  WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...   \n",
       "13445  INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...   \n",
       "13729  IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...   \n",
       "13979  MILITARY. MELBOURNE, Saturday. The following n...   \n",
       "22979  WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...   \n",
       "\n",
       "                                               corrected  str_similarity  \\\n",
       "26185  WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...        0.920732   \n",
       "13445  INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...        0.924826   \n",
       "13729  IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...        0.929938   \n",
       "13979  MILITARY. MELBOURNE, Saturday. The following n...        0.921942   \n",
       "22979  WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...        0.915966   \n",
       "\n",
       "       str_length  quality_band  use_corrected  \\\n",
       "26185        1144             1              0   \n",
       "13445        2155             1              0   \n",
       "13729        3240             1              0   \n",
       "13979        2563             1              0   \n",
       "22979         578             1              0   \n",
       "\n",
       "                                               alignment processed  \n",
       "26185  [(0, 7, 0, 7), (8, 15, 8, 15), (16, 23, 16, 23...       yes  \n",
       "13445  [(0, 13, 0, 13), (14, 19, 14, 19), (20, 31, 20...       yes  \n",
       "13729  [(0, 2, 0, 2), (3, 11, 3, 11), (12, 19, 12, 19...       yes  \n",
       "13979  [(2, 11, 0, 9), (12, 22, 10, 20), (23, 32, 21,...       yes  \n",
       "22979  [(6, 17, 4, 15), (23, 30, 19, 26), (44, 50, 38...       yes  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampledf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare the amount of text that has been aligned vs uncertain\n",
    "\n",
    "Number of characters that have been aligned vs characters whose alignment has been left as uncertain. Measure is given for each quality band."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampledf['alignedchars_ocr'] = 0\n",
    "sampledf['alignedchars_hum'] = 0\n",
    "sampledf['uncertainchars_ocr'] = 0\n",
    "sampledf['uncertainchars_hum'] = 0\n",
    "\n",
    "sampledf = sampledf[(abs(sampledf['ocrText'].str.len() - sampledf['humanText'].str.len()) <= 100)]\n",
    "\n",
    "for index, row in sampledf.iterrows():\n",
    "    aligned_chars_ocr = 0\n",
    "    aligned_chars_hum = 0\n",
    "    uncertain_chars_ocr = 0\n",
    "    uncertain_chars_hum = 0\n",
    "    ocrText = row['ocrText']\n",
    "    humanText = row['humanText']\n",
    "    alignment = ast.literal_eval(row['alignment'])\n",
    "    ocr_index = 0\n",
    "    hum_index = 0\n",
    "    alignment.sort(key=lambda tup: tup[0])\n",
    "    for a in alignment:\n",
    "        if a[0] != 0 and a[2] != 0:\n",
    "            uncertain_chars_ocr += len(ocrText[ocr_index : a[0] - 1].strip())\n",
    "            uncertain_chars_hum += len(humanText[hum_index : a[2] - 1].strip())\n",
    "            aligned_chars_ocr += len(ocrText[a[0] : a[1]].strip())\n",
    "            aligned_chars_hum += len(humanText[a[2] : a[3]].strip())\n",
    "        ocr_index = a[1] + 1\n",
    "        hum_index = a[3] + 1\n",
    "    sampledf.loc[index, 'alignedchars_ocr'] = aligned_chars_ocr\n",
    "    sampledf.loc[index, 'alignedchars_hum'] = aligned_chars_hum\n",
    "    sampledf.loc[index, 'uncertainchars_ocr'] = uncertain_chars_ocr\n",
    "    sampledf.loc[index, 'uncertainchars_hum'] = uncertain_chars_hum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filePath</th>\n",
       "      <th>articleId</th>\n",
       "      <th>articleType</th>\n",
       "      <th>year</th>\n",
       "      <th>ocrText</th>\n",
       "      <th>humanText</th>\n",
       "      <th>corrected</th>\n",
       "      <th>str_similarity</th>\n",
       "      <th>str_length</th>\n",
       "      <th>quality_band</th>\n",
       "      <th>use_corrected</th>\n",
       "      <th>alignment</th>\n",
       "      <th>processed</th>\n",
       "      <th>alignedchars_ocr</th>\n",
       "      <th>alignedchars_hum</th>\n",
       "      <th>uncertainchars_ocr</th>\n",
       "      <th>uncertainchars_hum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>26185</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14330048</td>\n",
       "      <td>Article</td>\n",
       "      <td>1900</td>\n",
       "      <td>WESTERN SUBURBS COTTAGE HOSPITAL. ,-«A-. Tim c...</td>\n",
       "      <td>WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...</td>\n",
       "      <td>WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...</td>\n",
       "      <td>0.920732</td>\n",
       "      <td>1144</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 7, 0, 7), (8, 15, 8, 15), (16, 23, 16, 23...</td>\n",
       "      <td>yes</td>\n",
       "      <td>773</td>\n",
       "      <td>777</td>\n",
       "      <td>206</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13445</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>13166969</td>\n",
       "      <td>Article</td>\n",
       "      <td>1868</td>\n",
       "      <td>INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...</td>\n",
       "      <td>INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...</td>\n",
       "      <td>INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...</td>\n",
       "      <td>0.924826</td>\n",
       "      <td>2155</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 13, 0, 13), (14, 19, 14, 19), (20, 31, 20...</td>\n",
       "      <td>yes</td>\n",
       "      <td>1399</td>\n",
       "      <td>1416</td>\n",
       "      <td>411</td>\n",
       "      <td>409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13729</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14953246</td>\n",
       "      <td>Article</td>\n",
       "      <td>1908</td>\n",
       "      <td>IN DIVORCE. (Before Mr. Justlco Simpson.) O'BR...</td>\n",
       "      <td>IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...</td>\n",
       "      <td>IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...</td>\n",
       "      <td>0.929938</td>\n",
       "      <td>3240</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 2, 0, 2), (3, 11, 3, 11), (12, 19, 12, 19...</td>\n",
       "      <td>yes</td>\n",
       "      <td>2193</td>\n",
       "      <td>2212</td>\n",
       "      <td>566</td>\n",
       "      <td>565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13979</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14947605</td>\n",
       "      <td>Article</td>\n",
       "      <td>1908</td>\n",
       "      <td>I MILITAEY. MELBOURNE, Saturday. The following...</td>\n",
       "      <td>MILITARY. MELBOURNE, Saturday. The following n...</td>\n",
       "      <td>MILITARY. MELBOURNE, Saturday. The following n...</td>\n",
       "      <td>0.921942</td>\n",
       "      <td>2563</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(2, 11, 0, 9), (12, 22, 10, 20), (23, 32, 21,...</td>\n",
       "      <td>yes</td>\n",
       "      <td>1971</td>\n",
       "      <td>1971</td>\n",
       "      <td>273</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22979</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>15708305</td>\n",
       "      <td>Article</td>\n",
       "      <td>1917</td>\n",
       "      <td>I WAR CASUALTIES. -» j KILLED. ! PRIVATE W. PR...</td>\n",
       "      <td>WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...</td>\n",
       "      <td>WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...</td>\n",
       "      <td>0.915966</td>\n",
       "      <td>578</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(6, 17, 4, 15), (23, 30, 19, 26), (44, 50, 38...</td>\n",
       "      <td>yes</td>\n",
       "      <td>339</td>\n",
       "      <td>339</td>\n",
       "      <td>177</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                filePath articleId  \\\n",
       "26185  ./trove_overproof/datasets/dataset1/rawTextAnd...  14330048   \n",
       "13445  ./trove_overproof/datasets/dataset1/rawTextAnd...  13166969   \n",
       "13729  ./trove_overproof/datasets/dataset1/rawTextAnd...  14953246   \n",
       "13979  ./trove_overproof/datasets/dataset1/rawTextAnd...  14947605   \n",
       "22979  ./trove_overproof/datasets/dataset1/rawTextAnd...  15708305   \n",
       "\n",
       "      articleType  year                                            ocrText  \\\n",
       "26185     Article  1900  WESTERN SUBURBS COTTAGE HOSPITAL. ,-«A-. Tim c...   \n",
       "13445     Article  1868  INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...   \n",
       "13729     Article  1908  IN DIVORCE. (Before Mr. Justlco Simpson.) O'BR...   \n",
       "13979     Article  1908  I MILITAEY. MELBOURNE, Saturday. The following...   \n",
       "22979     Article  1917  I WAR CASUALTIES. -» j KILLED. ! PRIVATE W. PR...   \n",
       "\n",
       "                                               humanText  \\\n",
       "26185  WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...   \n",
       "13445  INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...   \n",
       "13729  IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...   \n",
       "13979  MILITARY. MELBOURNE, Saturday. The following n...   \n",
       "22979  WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...   \n",
       "\n",
       "                                               corrected  str_similarity  \\\n",
       "26185  WESTERN SUBURBS COTTAGE HOSPITAL. — The commit...        0.920732   \n",
       "13445  INTERCOLONIAL NEWS. QUEENSLAND. BRISBANE files...        0.924826   \n",
       "13729  IN DIVORCE. (Before Mr. Justice Simpson.) O'BR...        0.929938   \n",
       "13979  MILITARY. MELBOURNE, Saturday. The following n...        0.921942   \n",
       "22979  WAR CASUALTIES. -» KILLED. PRIVATE W. PRYOR. M...        0.915966   \n",
       "\n",
       "       str_length  quality_band  use_corrected  \\\n",
       "26185        1144             1              0   \n",
       "13445        2155             1              0   \n",
       "13729        3240             1              0   \n",
       "13979        2563             1              0   \n",
       "22979         578             1              0   \n",
       "\n",
       "                                               alignment processed  \\\n",
       "26185  [(0, 7, 0, 7), (8, 15, 8, 15), (16, 23, 16, 23...       yes   \n",
       "13445  [(0, 13, 0, 13), (14, 19, 14, 19), (20, 31, 20...       yes   \n",
       "13729  [(0, 2, 0, 2), (3, 11, 3, 11), (12, 19, 12, 19...       yes   \n",
       "13979  [(2, 11, 0, 9), (12, 22, 10, 20), (23, 32, 21,...       yes   \n",
       "22979  [(6, 17, 4, 15), (23, 30, 19, 26), (44, 50, 38...       yes   \n",
       "\n",
       "       alignedchars_ocr  alignedchars_hum  uncertainchars_ocr  \\\n",
       "26185               773               777                 206   \n",
       "13445              1399              1416                 411   \n",
       "13729              2193              2212                 566   \n",
       "13979              1971              1971                 273   \n",
       "22979               339               339                 177   \n",
       "\n",
       "       uncertainchars_hum  \n",
       "26185                 198  \n",
       "13445                 409  \n",
       "13729                 565  \n",
       "13979                 265  \n",
       "22979                 160  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampledf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampledf_band1 = sampledf[sampledf['quality_band'] == 1]\n",
    "sampledf_band2 = sampledf[sampledf['quality_band'] == 2]\n",
    "sampledf_band3 = sampledf[sampledf['quality_band'] == 3]\n",
    "sampledf_band4 = sampledf[sampledf['quality_band'] == 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Band: 3 0.3051644178703162\n",
      "Band: 2 0.468855100896861\n",
      "Band: 1 0.6407480665926935\n",
      "Band: 0 0.7756302856978652\n"
     ]
    }
   ],
   "source": [
    "dfbands = [sampledf_band1, sampledf_band2, sampledf_band3, sampledf_band4]\n",
    "\n",
    "for band in reversed(range(len(dfbands))):\n",
    "    ahum = dfbands[band]['alignedchars_hum'].sum(axis=0)\n",
    "    aocr = dfbands[band]['alignedchars_ocr'].sum(axis=0)\n",
    "    uhum = dfbands[band]['uncertainchars_ocr'].sum(axis=0)\n",
    "    uocr = dfbands[band]['uncertainchars_hum'].sum(axis=0)\n",
    "    \n",
    "    print(\"Band:\", band, ahum / (ahum + uhum))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create csv with aligned text of articles in the sample\n",
    "\n",
    "Format (uncertain text is surrounded by `(((` and `)))` to make it easier to see them when annotating):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11, 17)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_annotate = pd.DataFrame()\n",
    "to_annotate = pd.concat([to_annotate, sampledf_band1.sample(2)])\n",
    "to_annotate = pd.concat([to_annotate, sampledf_band2.sample(2)])\n",
    "to_annotate = pd.concat([to_annotate, sampledf_band3.sample(3)])\n",
    "to_annotate = pd.concat([to_annotate, sampledf_band4.sample(4)])\n",
    "to_annotate.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filePath</th>\n",
       "      <th>articleId</th>\n",
       "      <th>articleType</th>\n",
       "      <th>year</th>\n",
       "      <th>ocrText</th>\n",
       "      <th>humanText</th>\n",
       "      <th>corrected</th>\n",
       "      <th>str_similarity</th>\n",
       "      <th>str_length</th>\n",
       "      <th>quality_band</th>\n",
       "      <th>use_corrected</th>\n",
       "      <th>alignment</th>\n",
       "      <th>processed</th>\n",
       "      <th>alignedchars_ocr</th>\n",
       "      <th>alignedchars_hum</th>\n",
       "      <th>uncertainchars_ocr</th>\n",
       "      <th>uncertainchars_hum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>418</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>17948935</td>\n",
       "      <td>Article</td>\n",
       "      <td>1945</td>\n",
       "      <td>\"BRITAIN STILL UNITED\" Conservative M.P. Reass...</td>\n",
       "      <td>\"BRITAIN STILL UNITED\" Conservative M.P. Reass...</td>\n",
       "      <td>\"BRITAIN STILL UNITED\" Conservative M.P. Reass...</td>\n",
       "      <td>0.905191</td>\n",
       "      <td>2331</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 8, 0, 8), (9, 14, 9, 14), (15, 22, 15, 22...</td>\n",
       "      <td>yes</td>\n",
       "      <td>1378</td>\n",
       "      <td>1423</td>\n",
       "      <td>563</td>\n",
       "      <td>576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16951</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>13205688</td>\n",
       "      <td>Article</td>\n",
       "      <td>1870</td>\n",
       "      <td>THE DARLING. Tho correspondent of the Pastoral...</td>\n",
       "      <td>THE DARLING. The correspondent of the Pastoral...</td>\n",
       "      <td>THE DARLING. The correspondent of the Pastoral...</td>\n",
       "      <td>0.966443</td>\n",
       "      <td>1638</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 3, 0, 3), (4, 12, 4, 12), (17, 30, 17, 30...</td>\n",
       "      <td>yes</td>\n",
       "      <td>1164</td>\n",
       "      <td>1164</td>\n",
       "      <td>202</td>\n",
       "      <td>201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14637</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>17582605</td>\n",
       "      <td>Article</td>\n",
       "      <td>1939</td>\n",
       "      <td>LATE MR. JUSTICE ' STEPHEN. Supreme Court Trib...</td>\n",
       "      <td>LATE MR. JUSTICE STEPHEN. Supreme Court Tribut...</td>\n",
       "      <td>LATE MR. JUSTICE STEPHEN. Supreme Court Tribut...</td>\n",
       "      <td>0.872858</td>\n",
       "      <td>1809</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 4, 0, 4), (5, 8, 5, 8), (9, 16, 9, 16), (...</td>\n",
       "      <td>yes</td>\n",
       "      <td>999</td>\n",
       "      <td>1014</td>\n",
       "      <td>538</td>\n",
       "      <td>535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21641</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14847840</td>\n",
       "      <td>Article</td>\n",
       "      <td>1907</td>\n",
       "      <td>SHOCKING FATALITY. ALBURY, Thursday. Benjamin ...</td>\n",
       "      <td>SHOCKING FATALITY. ALBURY, Thursday. Benjamin ...</td>\n",
       "      <td>SHOCKING FATALITY. ALBURY, Thursday. Benjamin ...</td>\n",
       "      <td>0.892356</td>\n",
       "      <td>641</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 8, 0, 8), (9, 18, 9, 18), (19, 26, 19, 26...</td>\n",
       "      <td>yes</td>\n",
       "      <td>401</td>\n",
       "      <td>406</td>\n",
       "      <td>135</td>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16349</td>\n",
       "      <td>./trove_overproof/datasets/dataset1/rawTextAnd...</td>\n",
       "      <td>14739922</td>\n",
       "      <td>Article</td>\n",
       "      <td>1905</td>\n",
       "      <td>BRITISH íASTEONOMICAL ASSOO&amp;l ATION. » ---?--....</td>\n",
       "      <td>BRITISH ASTRONOMICAL ASSOCI- ATION. ----------...</td>\n",
       "      <td>BRITISH ASTRONOMICAL ASSOCI- ATION. ----------...</td>\n",
       "      <td>0.787679</td>\n",
       "      <td>909</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>[(0, 7, 0, 7), (8, 21, 8, 20), (30, 36, 29, 35...</td>\n",
       "      <td>yes</td>\n",
       "      <td>443</td>\n",
       "      <td>453</td>\n",
       "      <td>279</td>\n",
       "      <td>348</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                filePath articleId  \\\n",
       "418    ./trove_overproof/datasets/dataset1/rawTextAnd...  17948935   \n",
       "16951  ./trove_overproof/datasets/dataset1/rawTextAnd...  13205688   \n",
       "14637  ./trove_overproof/datasets/dataset1/rawTextAnd...  17582605   \n",
       "21641  ./trove_overproof/datasets/dataset1/rawTextAnd...  14847840   \n",
       "16349  ./trove_overproof/datasets/dataset1/rawTextAnd...  14739922   \n",
       "\n",
       "      articleType  year                                            ocrText  \\\n",
       "418       Article  1945  \"BRITAIN STILL UNITED\" Conservative M.P. Reass...   \n",
       "16951     Article  1870  THE DARLING. Tho correspondent of the Pastoral...   \n",
       "14637     Article  1939  LATE MR. JUSTICE ' STEPHEN. Supreme Court Trib...   \n",
       "21641     Article  1907  SHOCKING FATALITY. ALBURY, Thursday. Benjamin ...   \n",
       "16349     Article  1905  BRITISH íASTEONOMICAL ASSOO&l ATION. » ---?--....   \n",
       "\n",
       "                                               humanText  \\\n",
       "418    \"BRITAIN STILL UNITED\" Conservative M.P. Reass...   \n",
       "16951  THE DARLING. The correspondent of the Pastoral...   \n",
       "14637  LATE MR. JUSTICE STEPHEN. Supreme Court Tribut...   \n",
       "21641  SHOCKING FATALITY. ALBURY, Thursday. Benjamin ...   \n",
       "16349  BRITISH ASTRONOMICAL ASSOCI- ATION. ----------...   \n",
       "\n",
       "                                               corrected  str_similarity  \\\n",
       "418    \"BRITAIN STILL UNITED\" Conservative M.P. Reass...        0.905191   \n",
       "16951  THE DARLING. The correspondent of the Pastoral...        0.966443   \n",
       "14637  LATE MR. JUSTICE STEPHEN. Supreme Court Tribut...        0.872858   \n",
       "21641  SHOCKING FATALITY. ALBURY, Thursday. Benjamin ...        0.892356   \n",
       "16349  BRITISH ASTRONOMICAL ASSOCI- ATION. ----------...        0.787679   \n",
       "\n",
       "       str_length  quality_band  use_corrected  \\\n",
       "418          2331             1              0   \n",
       "16951        1638             1              0   \n",
       "14637        1809             2              0   \n",
       "21641         641             2              0   \n",
       "16349         909             3              0   \n",
       "\n",
       "                                               alignment processed  \\\n",
       "418    [(0, 8, 0, 8), (9, 14, 9, 14), (15, 22, 15, 22...       yes   \n",
       "16951  [(0, 3, 0, 3), (4, 12, 4, 12), (17, 30, 17, 30...       yes   \n",
       "14637  [(0, 4, 0, 4), (5, 8, 5, 8), (9, 16, 9, 16), (...       yes   \n",
       "21641  [(0, 8, 0, 8), (9, 18, 9, 18), (19, 26, 19, 26...       yes   \n",
       "16349  [(0, 7, 0, 7), (8, 21, 8, 20), (30, 36, 29, 35...       yes   \n",
       "\n",
       "       alignedchars_ocr  alignedchars_hum  uncertainchars_ocr  \\\n",
       "418                1378              1423                 563   \n",
       "16951              1164              1164                 202   \n",
       "14637               999              1014                 538   \n",
       "21641               401               406                 135   \n",
       "16349               443               453                 279   \n",
       "\n",
       "       uncertainchars_hum  \n",
       "418                   576  \n",
       "16951                 201  \n",
       "14637                 535  \n",
       "21641                 134  \n",
       "16349                 348  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_annotate.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "dAlignmentsToEvaluate = dict()\n",
    "for index, row in to_annotate.iterrows():\n",
    "    newArticle = []\n",
    "    ocrText = row['ocrText']\n",
    "    humanText = row['corrected']\n",
    "    alignment = ast.literal_eval(row['alignment'])\n",
    "    ocr_index = 0\n",
    "    hum_index = 0\n",
    "    alignment.sort(key=lambda tup: tup[0])\n",
    "    for a in alignment:\n",
    "        if a[0] != 0 and a[2] != 0:\n",
    "            uncertain_match = ocrText[ocr_index : a[0] - 1].strip() + \"\\t\" + humanText[hum_index : a[2] - 1].strip()\n",
    "            uncertain_match = uncertain_match.strip()\n",
    "            if uncertain_match:\n",
    "                uncertain = str(row['articleId']) + \"\\t\" + str(row['str_similarity']) + \"\\tUNCERTAIN\\t\" + \"(((\" + uncertain_match + \")))\\t\" + str(ocr_index) + \"\\t \" + str(a[0] - 1)\n",
    "                newArticle.append(uncertain)\n",
    "        aligned_match = ocrText[a[0] : a[1]].strip() + \"\\t\" + humanText[a[2] : a[3]].strip()\n",
    "        aligned = str(row['articleId']) + \"\\t\" + str(row['str_similarity']) + \"\\t\" + \"ALIGNED\\t\" + aligned_match + \"\\t\" + str(a[2]) + \"\\t \" + str(a[3])\n",
    "        newArticle.append(aligned)\n",
    "        ocr_index = a[1] + 1\n",
    "        hum_index = a[3] + 1\n",
    "\n",
    "    dAlignmentsToEvaluate[(row['articleId'], row['str_similarity'])] = newArticle\n",
    "\n",
    "with open(\"alignments.tsv\", \"w\") as csvfile:\n",
    "    writer = csv.writer(csvfile, delimiter='\\t')\n",
    "    for article in dAlignmentsToEvaluate:\n",
    "        for textsnippet in dAlignmentsToEvaluate[article]:\n",
    "            textsnippet = textsnippet.split(\"\\t\")\n",
    "            writer.writerow(textsnippet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assessment of the quality of the evaluation here: https://docs.google.com/spreadsheets/d/1HugVFVYPjtz9rBPNfJwsmL9WXVDCUVw_IuFiCYSHR88/edit#gid=1936002185"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Alignment of just one article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trovedf = pd.read_pickle(\"trove_artidigh_aligned_human.pkl\")\n",
    "trovedf['quality_band'] = trovedf[\"str_similarity\"].apply(quality)\n",
    "artidigh_example = trovedf[trovedf['articleId'] == '13031691']\n",
    "\n",
    "for index, row in artidigh_example.iterrows():\n",
    "    aligned_chars_ocr = 0\n",
    "    aligned_chars_hum = 0\n",
    "    uncertain_chars_ocr = 0\n",
    "    uncertain_chars_hum = 0\n",
    "    ocrText = row['ocrText']\n",
    "    humanText = row['humanText']\n",
    "    alignment = ast.literal_eval(row['alignment'])\n",
    "    ocr_index = 0\n",
    "    hum_index = 0\n",
    "    alignment.sort(key=lambda tup: tup[0])\n",
    "    for a in alignment:\n",
    "        if a[0] != 0 and a[2] != 0:\n",
    "            uncertain_chars_ocr += len(ocrText[ocr_index : a[0] - 1].strip())\n",
    "            uncertain_chars_hum += len(humanText[hum_index : a[2] - 1].strip())\n",
    "            aligned_chars_ocr += len(ocrText[a[0] : a[1]].strip())\n",
    "            aligned_chars_hum += len(humanText[a[2] : a[3]].strip())\n",
    "        ocr_index = a[1] + 1\n",
    "        hum_index = a[3] + 1\n",
    "    artidigh_example.loc[index, 'alignedchars_ocr'] = aligned_chars_ocr\n",
    "    artidigh_example.loc[index, 'alignedchars_hum'] = aligned_chars_hum\n",
    "    artidigh_example.loc[index, 'uncertainchars_ocr'] = uncertain_chars_ocr\n",
    "    artidigh_example.loc[index, 'uncertainchars_hum'] = uncertain_chars_hum\n",
    "\n",
    "dAlignmentsToEvaluate = dict()\n",
    "for index, row in artidigh_example.iterrows():\n",
    "    newArticle = []\n",
    "    ocrText = row['ocrText']\n",
    "    humanText = row['humanText']\n",
    "    alignment = ast.literal_eval(row['alignment'])\n",
    "    ocr_index = 0\n",
    "    hum_index = 0\n",
    "    alignment.sort(key=lambda tup: tup[0])\n",
    "    for a in alignment:\n",
    "        if a[0] != 0 and a[2] != 0:\n",
    "            uncertain_match = ocrText[ocr_index : a[0] - 1].strip() + \"\\t\" + humanText[hum_index : a[2] - 1].strip()\n",
    "            uncertain_match = uncertain_match.strip()\n",
    "            if uncertain_match:\n",
    "                uncertain = str(row['articleId']) + \"\\t\" + str(row['str_similarity']) + \"\\tUNCERTAIN\\t\" + \"(((\" + uncertain_match + \")))\\t\" + str(ocr_index) + \"\\t \" + str(a[0] - 1)\n",
    "                newArticle.append(uncertain)\n",
    "        aligned_match = ocrText[a[0] : a[1]].strip() + \"\\t\" + humanText[a[2] : a[3]].strip()\n",
    "        aligned = str(row['articleId']) + \"\\t\" + str(row['str_similarity']) + \"\\t\" + \"ALIGNED\\t\" + aligned_match + \"\\t\" + str(a[2]) + \"\\t \" + str(a[3])\n",
    "        newArticle.append(aligned)\n",
    "        ocr_index = a[1] + 1\n",
    "        hum_index = a[3] + 1\n",
    "\n",
    "    dAlignmentsToEvaluate[(row['articleId'], row['str_similarity'])] = newArticle\n",
    "\n",
    "for i in dAlignmentsToEvaluate:\n",
    "    print(i[0])\n",
    "    print(i[1])\n",
    "    for e in dAlignmentsToEvaluate[i]:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import ast\n",
    "\n",
    "dfexample = pd.read_pickle(\"trove_subsample_aligned.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in dfexample.iloc[:1].iterrows():\n",
    "    ocrText = row['ocrText']\n",
    "    humanText = row['corrected']\n",
    "    print(ocrText)\n",
    "    print(humanText)\n",
    "    alignments = ast.literal_eval(row['alignment'])\n",
    "    for a in alignments:\n",
    "        print(a, ocrText[a[0]:a[1]], \"---\", humanText[a[2]:a[3]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:allennlp] *",
   "language": "python",
   "name": "conda-env-allennlp-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}