Migrate from MeSH to CUIs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GENERATE NEW CUIS INSTEAD OF MESH TERMS\n",
    "\n",
    "# This code will redo the concept extraction of the full database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from trialstreamer import dbutil\n",
    "import psycopg2\n",
    "import tqdm\n",
    "import json\n",
    "from robotreviewer.textprocessing import minimap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first transfer pre-done\n",
    "\n",
    "import pandas\n",
    "import json\n",
    "import tqdm\n",
    "\n",
    "pmids_done = set()\n",
    "\n",
    "df.read_csv('~/ijmarshall/cuis.csv')\n",
    "\n",
    "for i, r in tqdm.tqdm(df.iterrows()):\n",
    "    cur.execute('update pubmed_annotations set population_mesh=(%s), interventions_mesh=(%s), outcomes_mesh=(%s) where pmid=(%s);',\n",
    "                (json.dumps(r['population_mesh']), json.dumps(r['interventions_mesh']), json.dumps(r['outcomes_mesh']), r['pmid']))\n",
    "    pmids_done.add(r['pmid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the pico span data\n",
    "\n",
    "cur = dbutil.db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)\n",
    "cur.execute('select pmid, population, interventions, outcomes from pubmed_annotations;')\n",
    "results_in = [dict(r) for r in cur if r['pmid'] not in pmids_done]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/44 [00:00<?, ?it/s]\u001b[A\n",
      "  5%|▍         | 2/44 [00:00<00:02, 19.06it/s]\u001b[A\n",
      " 14%|█▎        | 6/44 [00:00<00:01, 21.08it/s]\u001b[A\n",
      " 23%|██▎       | 10/44 [00:00<00:01, 24.54it/s]\u001b[A\n",
      " 34%|███▍      | 15/44 [00:00<00:01, 26.78it/s]\u001b[A\n",
      " 45%|████▌     | 20/44 [00:00<00:00, 30.89it/s]\u001b[A\n",
      " 55%|█████▍    | 24/44 [00:00<00:00, 28.33it/s]\u001b[A\n",
      " 64%|██████▎   | 28/44 [00:00<00:00, 29.66it/s]\u001b[A\n",
      " 73%|███████▎  | 32/44 [00:01<00:00, 30.80it/s]\u001b[A\n",
      " 82%|████████▏ | 36/44 [00:01<00:00, 29.64it/s]\u001b[A\n",
      "100%|██████████| 44/44 [00:01<00:00, 32.24it/s]\u001b[A\n"
     ]
    }
   ],
   "source": [
    "# this should fix up PubMed\n",
    "fields = ['population', 'interventions', 'outcomes']\n",
    "\n",
    "cur = dbutil.db.cursor()\n",
    "\n",
    "for in_r in tqdm.tqdm(results_in):\n",
    "    out_r = {}\n",
    "    for f in fields:\n",
    "        out_r[f] = minimap.get_unique_terms(in_r[f])\n",
    "    cur.execute('update pubmed_annotations set population_mesh=(%s), interventions_mesh=(%s), outcomes_mesh=(%s) where pmid =(%s)',\n",
    "                   (json.dumps(out_r['population']), json.dumps(out_r['interventions']), json.dumps(out_r['outcomes']), in_r['pmid']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbutil.db.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the pico span data\n",
    "\n",
    "cur = dbutil.db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)\n",
    "cur.execute('select id, population, interventions, outcomes from medrxiv_covid19;')\n",
    "results_in = [dict(r) for r in cur]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 44/44 [00:01<00:00, 25.13it/s]\n"
     ]
    }
   ],
   "source": [
    "# this should fix up PubMed\n",
    "fields = ['population', 'interventions', 'outcomes']\n",
    "\n",
    "cur = dbutil.db.cursor()\n",
    "\n",
    "for in_r in tqdm.tqdm(results_in):\n",
    "    out_r = {}\n",
    "    for f in fields:\n",
    "        out_r[f] = minimap.get_unique_terms(in_r[f])\n",
    "    cur.execute('update medrxiv_covid19 set population_mesh=(%s), interventions_mesh=(%s), outcomes_mesh=(%s) where id =(%s)',\n",
    "                   (json.dumps(out_r['population']), json.dumps(out_r['interventions']), json.dumps(out_r['outcomes']), in_r['id']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the pico span data\n",
    "\n",
    "cur = dbutil.db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)\n",
    "cur.execute('select id, population, interventions, outcomes from ictrp;')\n",
    "results_in = [dict(r) for r in cur]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 241378/241378 [7:32:49<00:00,  8.88it/s]      \n"
     ]
    }
   ],
   "source": [
    "# this should fix up ICTRP\n",
    "fields = ['population', 'interventions', 'outcomes']\n",
    "\n",
    "cur = dbutil.db.cursor()\n",
    "\n",
    "\n",
    "with tqdm.tqdm(total = len(results_in)) as pbar:\n",
    "\n",
    "    while results_in:\n",
    "        in_r = results_in.pop()\n",
    "        out_r = {}\n",
    "        for f in fields:\n",
    "            out_r[f] = minimap.get_unique_terms(in_r[f])\n",
    "        cur.execute('update ictrp set population_mesh=(%s), interventions_mesh=(%s), outcomes_mesh=(%s) where id =(%s)',\n",
    "                       (json.dumps(out_r['population']), json.dumps(out_r['interventions']), json.dumps(out_r['outcomes']), in_r['id']))\n",
    "        pbar.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbutil.db.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}