GRPM_02_ref-mesh_builder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ref MeSH Builder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " This notebook allows you to create MeSH lists to inquire the GRPM Dataset starting with a biomedical topic of choice.\n",
    " The system is based on the complete MESH datasheet.\n",
    " The system uses ChatGPT API to supply the initial biomedical terms for constructing the Ref MeSH set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Only for Google Colab\n",
    "import os\n",
    "import sys\n",
    "import subprocess\n",
    "\n",
    "# @markdown Run in Colab virtual machine by default\n",
    "\n",
    "# @markdown to run in google drive set:\n",
    "import_mydrive = False #@param {type:\"boolean\"}\n",
    "\n",
    "\n",
    "if 'google.colab' in sys.modules:\n",
    "    subprocess.run([\"pip\", \"install\", \"nbib\"])\n",
    "    subprocess.run([\"pip\", \"install\", \"biopython\"])\n",
    "\n",
    "    if import_mydrive:\n",
    "        from google.colab import drive\n",
    "        drive.mount('/content/drive')\n",
    "\n",
    "        if not os.path.exists('/content/drive/MyDrive/grpm_system/'):\n",
    "            subprocess.run(['mkdir', '/content/drive/MyDrive/grpm_system/'])\n",
    "        subprocess.run(['cd', '/content/drive/MyDrive/grpm_system/'])\n",
    "    else:\n",
    "        if not os.path.exists('/content/grpm_system/'):\n",
    "            subprocess.run(['mkdir', '/content/grpm_system/'])\n",
    "        subprocess.run(['cd', '/content/grpm_system/'])\n",
    "\n",
    "current_directory = os.getcwd()\n",
    "print(\"Current working directory:\", current_directory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-11T17:00:23.965062Z",
     "start_time": "2024-04-11T17:00:22.929991100Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "\n",
    "def simple_bool(message):\n",
    "    choose = input(message+\" (y/n): \").lower()\n",
    "    your_bool = choose in [\"y\", \"yes\",\"yea\",\"sure\"]\n",
    "    return your_bool\n",
    "\n",
    "def get_file(url, file_name, dir = os.getcwd()):\n",
    "    url = url\n",
    "    file_name = file_name\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        content = response.content\n",
    "        file_path = os.path.join(dir, file_name)\n",
    "        with open(file_path, 'wb') as file:\n",
    "            file.write(content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get requirements"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get MESH.csv from 'bioportal.bioontology.org'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gzip\n",
    "import shutil\n",
    "# Get MESH.csv from https://bioportal.bioontology.org/ontologies/MESH?p=summary\n",
    "\n",
    "if not os.path.exists('ref-mesh-archive'):\n",
    "    os.makedirs('ref-mesh-archive')\n",
    "\n",
    "if not os.path.exists('ref-mesh-archive/MESH.csv'):\n",
    "    get_file( url='https://data.bioontology.org/ontologies/MESH/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb&download_format=csv', file_name='MESH.gz', dir = 'ref-mesh-archive')\n",
    "\n",
    "    # Open the gzipped input file and the output file\n",
    "    file_path = 'ref-mesh-archive/'\n",
    "    with gzip.open(file_path+'MESH.gz', 'rb') as input_file, open(file_path+'MESH.csv', 'wb') as output_file:\n",
    "        # Copy the content from the input gzipped file to the output file\n",
    "        shutil.copyfileobj(input_file, output_file)\n",
    "    \n",
    "    os.remove(file_path+'MESH.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get pre-made datasets from Zenodo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get pre-made datasets from Zenodo Repository\n",
    "#https://zenodo.org/record/8205724  DOI: 10.5281/zenodo.8205724\n",
    "\n",
    "import io\n",
    "import zipfile\n",
    "\n",
    "def get_and_extract(file, dir = os.getcwd(), ext = '.zip'):\n",
    "    url='https://zenodo.org/record/8205724/files/'+file+'.zip?download=1'\n",
    "    zip_file_name = file+ext\n",
    "    extracted_folder_name = dir\n",
    "\n",
    "    # Download the ZIP file\n",
    "    response = requests.get(url)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        # Extract the ZIP contents\n",
    "        with io.BytesIO(response.content) as zip_buffer:\n",
    "            with zipfile.ZipFile(zip_buffer, 'r') as zip_ref:\n",
    "                zip_ref.extractall(extracted_folder_name)\n",
    "        print(f\"ZIP file '{zip_file_name}' extracted to '{extracted_folder_name}' successfully.\")\n",
    "    else:\n",
    "        print(\"Failed to download the ZIP file.\")\n",
    "\n",
    "if not os.path.exists('ref-mesh-archive/MESH_STY_LITVAR1.csv'):\n",
    "    get_and_extract('ref-mesh-archive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load Mesh full dataset\n",
    "(skip this if MESH.csv is not provided)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#load MESH dataframe:\n",
    "if os.path.exists('ref-mesh-archive/MESH.csv'):\n",
    "    df = pd.read_csv('ref-mesh-archive/MESH.csv', low_memory=False)\n",
    "else:\n",
    "    print(\"no MESH.csv available.\\n => skip to Section 2\")\n",
    "\n",
    "#info on Proprieties: https://bioportal.bioontology.org/ontologies/MESH?p=properties\n",
    "#Mesh Browser: https://meshb.nlm.nih.gov/\n",
    "\n",
    "# AQL: Allowable Qualifiers\n",
    "AQL = [\"blood (BL)\",\"cerebrospinal fluid (CF)\",\"chemically induced (CI)\",\"classification (CL)\",\"complications (CO)\",\"congenital (CN)\",\"diagnosis (DI)\",\"diagnostic imaging (DG)\",\"diet therapy (DH)\",\"drug therapy (DT)\",\"economics (EC)\",\"embryology (EM)\",\"enzymology (EN)\",\"epidemiology (EP)\",\"ethnology (EH)\",\"etiology (ET)\",\"genetics (GE)\",\"history (HI)\",\"immunology (IM)\",\"metabolism (ME)\",\"microbiology (MI)\",\"mortality (MO)\",\"nursing (NU)\",\"parasitology (PS)\",\"pathology (PA)\",\"physiopathology (PP)\",\"prevention & control (PC)\",\"psychology (PX)\",\"radiotherapy (RT)\",\"rehabilitation (RH)\",\"surgery (SU)\",\"therapy (TH)\",\"urine (UR)\",\"veterinary (VE)\",\"virology (VI)\"]\n",
    "aql=pd.DataFrame(AQL)\n",
    "aql"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('display MeSH data:\\n')\n",
    "print(df.columns)\n",
    "df.iloc[:3]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#How many MeSH has mapping qualifier (subheadings)?\n",
    "all_mesh = df['Preferred Label'].drop_duplicates()\n",
    "mapping_qf = df[['Mapping qualifier of','Preferred Label']].drop_duplicates().dropna()\n",
    "has = df[['Has mapping qualifier','Preferred Label']].drop_duplicates().dropna()\n",
    "has_len = has['Preferred Label'].nunique()\n",
    "\n",
    "print('Total number of MeSH terms:\\n', df['Preferred Label'].nunique())\n",
    "print('\\nHow many Mesh has mapping qualifier (subheadings)?\\n',has_len)\n",
    "print(' %', round((has_len/len(all_mesh)*100),3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# search for single exact value\n",
    "mesh = 'Disease or Syndrome'\n",
    "mesh = 'Heart Failure'\n",
    "print('search for single exact value:')\n",
    "df[df['Preferred Label']==mesh].T.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Extract subset of Mesh\n",
    "str_1 = 'Fabry'\n",
    "str_2 = 'Lysosomal Sto'\n",
    "str_3 = ''\n",
    "str_full = str_2 or str_1 or str_3\n",
    "\n",
    "ref_search = df[df['Preferred Label'].str.contains(str_full).fillna(False)]\n",
    "print('look for mesh containing \"',str_1,',',str_2,',',str_3,  '\":')\n",
    "ref_search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Semantic Types Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Extract subset of all semantic types\n",
    "df_sem = df[df['Class ID'].str.contains('STY').fillna(False)]\n",
    "df_sem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Characterize a Semantic Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#how many mesh for semantic type?\n",
    "semantictype = 'T192'\n",
    "sam_label = df[df['Class ID'].str.contains(semantictype).fillna(False)].reset_index().at[0,'Preferred Label']\n",
    "# Get Semantic Type name for STY ID\n",
    "print('semantic type:', sam_label)\n",
    "\n",
    "mask2 = df['Semantic Types'].str.contains(semantictype).fillna(False)\n",
    "mesh_sem = df[mask2][['Class ID','Preferred Label','Definitions','Semantic Types']]\n",
    "print('number of mesh:', len(mesh_sem))\n",
    "print('\\nMesh for \"',semantictype,sam_label,'\" semantic type:')\n",
    "\n",
    "mesh_sem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Characterize a single mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# search for single Mesh\n",
    "mesh_classid ='D004048'\n",
    "maskmesh = df['Class ID'].str.contains(mesh_classid).fillna(False)\n",
    "\n",
    "sty_classid = df[maskmesh]['Semantic Types'].reset_index().iat[0, 1][-4:]\n",
    "\n",
    "masksty = df['Class ID'].str.contains(sty_classid).fillna(False)\n",
    "\n",
    "masked = df[maskmesh][['Semantic Types','Preferred Label','Definitions']].reset_index()\n",
    "\n",
    "#get mesh definition:\n",
    "print('mesh:', masked.at[0,'Preferred Label'],\n",
    "      '\\nsemantic type:',df[masksty].reset_index().at[0,'Preferred Label'],sty_classid,\n",
    "      '\\ndescrition:',masked.at[0,'Definitions'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Row aggregation\n",
    "data_mesh = []\n",
    "data_sty = ['T021','T116']\n",
    "dfd = pd.DataFrame()\n",
    "for i in data_sty:\n",
    "    mask = df['Semantic Types'].str.contains(i).fillna(False) # aggregate mesh for sty\n",
    "    dfd = pd.concat([dfd, df[mask]])\n",
    "dfd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Load/create MESH-LITVAR dataset (required)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-11T17:01:34.346758Z",
     "start_time": "2024-04-11T17:01:21.454612200Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MESH_STY sty imported from csv\n",
      "MESH_STY_LITVAR1 imported from csv\n",
      "\n",
      " => For Reference MeSH list creation skip to Section 3\n"
     ]
    }
   ],
   "source": [
    "# Subset MeSH dataset\n",
    "\n",
    "#set options:\n",
    "import_mesh_sty = True\n",
    "\n",
    "# concat the exploded mesh table or import prearranged csv\n",
    "if os.path.isfile('ref-mesh-archive/MESH_STY.csv') and import_mesh_sty:\n",
    "    mesh_large_df_sty = pd.read_csv('ref-mesh-archive/MESH_STY.csv', index_col=0)\n",
    "    print('MESH_STY sty imported from csv')\n",
    "elif import_mesh_sty:\n",
    "        if os.path.exists('ref-mesh-archive/MESH.csv'):\n",
    "            # correct list format\n",
    "            df['STY_ID'] = df['Semantic Types'].str.replace(r'http://purl.bioontology.org/ontology/STY/','', regex = False)\n",
    "            start_word = '[\\\"'\n",
    "            end_word = '\\\"]'\n",
    "            df['STY_ID'] = f'{start_word} ' + df['STY_ID'] + f' {end_word}'\n",
    "            df['STY_ID'] = df['STY_ID'].str.replace(' ','', regex = False)\n",
    "            df['STY_ID'] = df['STY_ID'].str.replace('|','\\\",\\\"', regex = False)\n",
    "            #print(df['STY_ID'].isna().sum())\n",
    "            df.dropna(subset=['STY_ID'], inplace=True)\n",
    "            df.reset_index(drop=True,inplace=True)\n",
    "\n",
    "            from ast import literal_eval\n",
    "            df['STY_ID'] = df['STY_ID'].apply(literal_eval)\n",
    "\n",
    "            # Import exhcange table sty-code\n",
    "            sty = pd.read_csv('ref-mesh-archive/MeshSTY-code.csv',sep=';')\n",
    "            sty = sty.rename(columns={'ID':'Semantic Types'})\n",
    "            sty = sty.rename(columns={'ID':'Semantic Types'})\n",
    "            #print(sty['Semantic Types'].nunique())\n",
    "\n",
    "            #mesh_large = pd.DataFrame()\n",
    "            mesh_large = []\n",
    "            for i in range(len(df)):\n",
    "                for sem in df['STY_ID'][i]: #dfrspost = mother table\n",
    "                    out = df['Preferred Label'][i],df['Class ID'][i],sem,df['Synonyms'][i],df['Parents'][i],df['CUI'][i],df['AQL'][i],df['TERMUI'][i]\n",
    "                    mesh_large.append(out)\n",
    "\n",
    "            mesh_large_df = pd.DataFrame(mesh_large)\n",
    "            new_col_names = ['Preferred Label','Class ID','Semantic Types','Synonyms','Parents','CUI','AQL','TERMUI']\n",
    "            mesh_large_df.columns = new_col_names\n",
    "\n",
    "            ## Add STY Labels\n",
    "            mesh_large_df_sty = pd.merge(mesh_large_df, sty, on='Semantic Types', how='inner').reset_index(drop=True)\n",
    "            #Add rsid coulmn con merge\n",
    "            mesh_large_df_sty = mesh_large_df_sty.rename(columns={'Preferred Label_y':'Semantic Types Label','Preferred Label_x':'Preferred Label'})\n",
    "\n",
    "            mesh_large_df_sty = mesh_large_df_sty[['Preferred Label', 'Semantic Types Label', 'Class ID', 'Semantic Types', 'Synonyms', 'Parents', 'CUI', 'AQL', 'TERMUI']]\n",
    "            mesh_large_df_sty.to_csv('ref-mesh-archive/MESH_STY.csv')\n",
    "            print('MESH_STY created')\n",
    "        else:\n",
    "            print('MESH.csv not available')\n",
    "\n",
    "#---------------------------------------------------\n",
    "#### Create MESH-STY-LITVAR subset from MESH-STY.csv\n",
    "if os.path.exists('ref-mesh-archive/MESH_STY_LITVAR1.csv'):\n",
    "    mesh_litvar_sty = pd.read_csv('ref-mesh-archive/MESH_STY_LITVAR1.csv',index_col=0)\n",
    "    print('MESH_STY_LITVAR1 imported from csv')\n",
    "else:\n",
    "    grpm_mesh = pd.read_csv('ref-mesh-archive/grpm_db_mesh.csv', index_col=0)\n",
    "    mask = mesh_large_df_sty['Preferred Label'].isin(grpm_mesh.mesh)\n",
    "    mesh_litvar_sty = mesh_large_df_sty[mask]\n",
    "    mesh_litvar_sty.to_csv('ref-mesh-archive/MESH_STY_LITVAR1.csv')\n",
    "    print('MESH_STY_LITVAR1 created')\n",
    "\n",
    "print('\\n => For Reference MeSH list creation skip to Section 3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# MeSH stats\n",
    "if 'mesh_large_df_sty' in globals():\n",
    "    print('MESH_STY', mesh_large_df_sty['Preferred Label'].nunique(), 'mesh')\n",
    "    print('MESH_STY', mesh_large_df_sty['Semantic Types Label'].nunique(), 'semantic types')\n",
    "\n",
    "print('MESH_STY_LITVAR1', mesh_litvar_sty['Preferred Label'].nunique(), 'mesh')\n",
    "print('MESH_STY_LITVAR1', mesh_litvar_sty['Semantic Types Label'].nunique(), 'semantic types\\n')\n",
    "\n",
    "mesh_litvar_sty[['Preferred Label', 'Semantic Types Label', 'Class ID', 'mesh_id','Semantic Types']]#.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## ADD mesh_id col\n",
    "if 'mesh_id' not in mesh_litvar_sty.columns:\n",
    "    mesh_litvar_sty['mesh_id'] = mesh_litvar_sty['Class ID'].str.replace('http://purl.bioontology.org/ontology/MESH/', '')\n",
    "    mesh_litvar_sty['mesh_id']\n",
    "    mesh_litvar_sty.columns\n",
    "    new_order = ['Preferred Label', 'Semantic Types Label', 'Class ID', 'mesh_id', 'Semantic Types',\n",
    "                 'Synonyms', 'Parents', 'CUI', 'AQL', 'TERMUI']\n",
    "    mesh_litvar_sty = mesh_litvar_sty[new_order]\n",
    "    mesh_litvar_sty.to_csv('ref-mesh-archive/MESH_STY_LITVAR1.csv')\n",
    "else:\n",
    "    print('mesh_id aready added to csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add mesh_id to GRPM dataset (not required)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"note: in GRPM Dataset screening using 'Preferred Label' results faster then numerical 'mesh_id'\")\n",
    "\n",
    "if simple_bool(\"note: in GRPM Dataset screening using 'Preferred Label' results faster then numerical 'mesh_id'\\n add mesh_id to grpm_ds and replace 'Preferred Label' anyway?\") and os.path.exists('grpm_dataset/grpm_db_pcg/grpm_table_output.csv'):\n",
    "    grpm_b_df = pd.read_csv('grpm_dataset/grpm_db_pcg/grpm_table_output.csv',index_col=0)\n",
    "\n",
    "    if not 'mesh_id' in grpm_b_df.columns:\n",
    "        # add mesh_id to grpm db\n",
    "        mesh_id_ref_df = mesh_litvar_sty[['Preferred Label','mesh_id']]\n",
    "        grpm_db_merge_id = pd.merge(grpm_b_df, mesh_id_ref_df, left_on='mesh', right_on='Preferred Label')\n",
    "\n",
    "        # replace 'Preferred Label' with 'mesh_id'\n",
    "        grpm_db_merge_id = grpm_db_merge_id.drop('Preferred Label', axis=1)\n",
    "        grpm_db_merge_id.columns\n",
    "        new_order = ['gene', 'rsid', 'pmids', 'mesh_id']\n",
    "        grpm_db_merge_id = grpm_db_merge_id[new_order].drop_duplicates()\n",
    "        grpm_db_merge_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if simple_bool('save it?'):\n",
    "    grpm_db_merge_id.to_csv('grpm_dataset/grpm_db_pcg/grpm_table_output_id.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if simple_bool('check?'):\n",
    "    check = pd.read_csv('grpm_dataset/grpm_db_pcg/grpm_table_output_id.csv', index_col=0)\n",
    "    check"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sorting Mesh-STY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if 'mesh_large_df_sty' in globals():\n",
    "    # groupby and bar module\n",
    "    mesh_large_df_sty_less = mesh_large_df_sty[['Preferred Label','Semantic Types Label']]\n",
    "\n",
    "    ### groupby.describe analysis by rsid--------------------\n",
    "    mesh_large_df_sty_less_count = mesh_large_df_sty_less.groupby('Semantic Types Label').describe().reset_index()\n",
    "    mesh_large_df_sty_less_count.columns = mesh_large_df_sty_less_count.columns.to_flat_index()\n",
    "    new_column_names = ['Semantic Types Label', 'mesh-count', 'mesh-unique','mesh-top','mesh-freq']\n",
    "    mesh_large_df_sty_less_count.columns = new_column_names\n",
    "\n",
    "    mesh_large_df_sty_less_count_sort = mesh_large_df_sty_less_count.sort_values(by='mesh-count',ascending=False).reset_index(drop=True)\n",
    "    print('MESH_STY')\n",
    "    print('Semantic Type count:',len(mesh_large_df_sty_less_count_sort),'\\n\\nMeSH count for Sty:')\n",
    "    display(mesh_large_df_sty_less_count_sort[['mesh-count','Semantic Types Label']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Graph Barh\n",
    "num = 20\n",
    "x = mesh_large_df_sty_less_count_sort['Semantic Types Label'].iloc[:num]\n",
    "y = mesh_large_df_sty_less_count_sort['mesh-count'].iloc[:num]\n",
    "plt.figure(figsize=(4, len(x) *0.25))\n",
    "plt.title('Global Mesh- Semantic Type enrichment', loc='center',pad=10)\n",
    "\n",
    "plt.barh(x,y)\n",
    "plt.gca().invert_yaxis()\n",
    "plt.tick_params(axis='x', which='both', top=True, bottom=False, labeltop=True, labelbottom=False)\n",
    "#plt.xlabel('pmid count', position=(0.5, 1.08))\n",
    "plt.ylabel('Semantic Types')\n",
    "plt.xlabel('mesh', position=(0.5, 1.08))\n",
    "ax = plt.gca()\n",
    "ax.xaxis.set_label_position('top')\n",
    "# use log scale\n",
    "plt.gca().set_xscale('log')\n",
    "#plt.savefig('Reference Mesh- Semantic Type enrichment.png',dpi=300, bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#'Global \"Mesh-Semantic Type\" Zipf s law'\n",
    "x = mesh_large_df_sty_less_count_sort['Semantic Types Label'].iloc[:]\n",
    "y = mesh_large_df_sty_less_count_sort['mesh-count'].sort_values().iloc[:]\n",
    "plt.figure(figsize=(6, 5))\n",
    "plt.title('Global \"Mesh-Semantic Type\" Zipf s law', loc='center',pad=10)\n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.gca().invert_yaxis()\n",
    "\n",
    "plt.ylabel('mesh number')\n",
    "plt.xlabel('semantic type rank', position=(0.5, 1.08))\n",
    "\n",
    "plt.gca().set_xscale('log')\n",
    "plt.gca().set_yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Density Rugged Plot\n",
    "data = mesh_large_df_sty_less_count_sort['mesh-count'].iloc[:]\n",
    "\n",
    "sns.set_style('whitegrid')\n",
    "sns.kdeplot(np.array(data), bw_method=0.5)\n",
    "sns.rugplot(np.array(data), color='r')\n",
    "\n",
    "#plt.yscale('log')\n",
    "plt.title('Mesh abundance for Semantic Type')\n",
    "#plt.yscale('log')\n",
    "print('Mesh abundance for Semantic Type:')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze MESH-STY-LITVAR (subset of MESH-STY.csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('mesh_litvar_sty mesh:',mesh_litvar_sty['Preferred Label'].nunique())\n",
    "\n",
    "memory = mesh_litvar_sty.memory_usage().sum()\n",
    "print(f'The memory_usage of mesh_litvar_sty is {memory/ (1024 * 1024):.2f} MB.')\n",
    "\n",
    "file_size = os.path.getsize('ref-mesh-archive/MESH_STY_LITVAR1.csv')\n",
    "print(f'The size of MESH_STY_LITVAR1.csv is {file_size/ (1024 * 1024):.2f} MB.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sort MeSH-Sty-Litvar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# groupby and bar module\n",
    "mesh_litvar_sty_less = mesh_litvar_sty[['Preferred Label','Semantic Types Label']]\n",
    "\n",
    "### groupby.describe analysis by rsid--------------------\n",
    "mesh_litvar_sty_less_count = mesh_litvar_sty_less.groupby('Semantic Types Label').describe().reset_index()\n",
    "mesh_litvar_sty_less_count.columns = mesh_litvar_sty_less_count.columns.to_flat_index()\n",
    "new_column_names = ['Semantic Types Label', 'mesh-count', 'mesh-unique','mesh-top','mesh-freq']\n",
    "mesh_litvar_sty_less_count.columns = new_column_names\n",
    "\n",
    "mesh_litvar_sty_less_count_sort = mesh_litvar_sty_less_count.sort_values(by='mesh-count',ascending=False).reset_index(drop=True)\n",
    "print('MESH_STY_LITVAR1')\n",
    "mesh_litvar_sty_less_count_sort[['mesh-count','Semantic Types Label']]#.to_clipboard()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Graph Barh\n",
    "num = 20\n",
    "x = mesh_litvar_sty_less_count_sort['Semantic Types Label'].iloc[:num]\n",
    "y = mesh_litvar_sty_less_count_sort['mesh-count'].iloc[:num]\n",
    "plt.figure(figsize=(4, len(x)*0.25))\n",
    "plt.title('Litvar1 Mesh- Semantic Type enrichment', loc='center',pad=10)\n",
    "\n",
    "plt.barh(x,y)\n",
    "plt.gca().invert_yaxis()\n",
    "plt.tick_params(axis='x', which='both', top=True, bottom=False, labeltop=True, labelbottom=False)\n",
    "plt.ylabel('Semantic Types')\n",
    "plt.xlabel('mesh', position=(0.5, 1.08))\n",
    "ax = plt.gca()\n",
    "ax.xaxis.set_label_position('top')\n",
    "# use log scale\n",
    "plt.gca().set_xscale('log')\n",
    "#plt.savefig('Reference Mesh- Semantic Type enrichment.png',dpi=300, bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#'Global \"Mesh-Semantic Type\" Zipf s law'\n",
    "x = mesh_litvar_sty_less_count_sort['Semantic Types Label'].iloc[:]\n",
    "y = mesh_litvar_sty_less_count_sort['mesh-count'].sort_values().iloc[:]\n",
    "plt.figure(figsize=(6, 5))\n",
    "plt.title('LitVar \"Mesh-Semantic Type\" Zipf s law', loc='center',pad=10)\n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.gca().invert_yaxis()\n",
    "\n",
    "plt.ylabel('mesh number')\n",
    "plt.xlabel('semantic type rank', position=(0.5, 1.08))\n",
    "\n",
    "plt.gca().set_xscale('log')\n",
    "plt.gca().set_yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# VISUALIZE DIFFERENCES\n",
    "\n",
    "#reload sorted df\n",
    "if 'mesh_large_df_sty_less_count_sort' in locals() and isinstance(mesh_large_df_sty_less_count_sort, pd.DataFrame):\n",
    "    pass\n",
    "else:\n",
    "    mesh_large_df_sty_less = mesh_large_df_sty[['Preferred Label','Semantic Types Label']]\n",
    "    mesh_large_df_sty_less_count = mesh_large_df_sty_less.groupby('Semantic Types Label').describe().reset_index()\n",
    "    mesh_large_df_sty_less_count.columns = mesh_large_df_sty_less_count.columns.to_flat_index()\n",
    "    new_column_names = ['Semantic Types Label', 'mesh-count', 'mesh-unique','mesh-top','mesh-freq']\n",
    "    mesh_large_df_sty_less_count.columns = new_column_names\n",
    "    mesh_large_df_sty_less_count_sort = mesh_large_df_sty_less_count.sort_values(by='mesh-count',ascending=False).reset_index(drop=True)\n",
    "if 'mesh_litvar_sty_less_count_sort' in locals() and isinstance(mesh_litvar_sty_less_count_sort, pd.DataFrame):\n",
    "    pass\n",
    "else:\n",
    "    mesh_litvar_sty_less = mesh_litvar_sty[['Preferred Label','Semantic Types Label']]\n",
    "    mesh_litvar_sty_less_count = mesh_litvar_sty_less.groupby('Semantic Types Label').describe().reset_index()\n",
    "    mesh_litvar_sty_less_count.columns = mesh_litvar_sty_less_count.columns.to_flat_index()\n",
    "    new_column_names = ['Semantic Types Label', 'mesh-count', 'mesh-unique','mesh-top','mesh-freq']\n",
    "    mesh_litvar_sty_less_count.columns = new_column_names\n",
    "    mesh_litvar_sty_less_count_sort = mesh_litvar_sty_less_count.sort_values(by='mesh-count',ascending=False).reset_index(drop=True)\n",
    "\n",
    "# create two sample dataframes\n",
    "df1 = mesh_large_df_sty_less_count_sort[['mesh-count', 'Semantic Types Label']]\n",
    "df2 = mesh_litvar_sty_less_count_sort[['mesh-count', 'Semantic Types Label']]\n",
    "\n",
    "# add a 'source' column to each dataframe\n",
    "df1['source'] = 'global'\n",
    "df2['source'] = 'litvar'\n",
    "\n",
    "# combine the dataframes\n",
    "combined_df = pd.merge(df1, df2, on=['Semantic Types Label'], how='outer', suffixes=('_df1', '_df2'))\n",
    "\n",
    "# sort the dataframe by column 'A'\n",
    "#combined_df = combined_df.sort_values('mesh-count')\n",
    "\n",
    "# reset the index\n",
    "combined_df = combined_df.reset_index(drop=True)\n",
    "\n",
    "# display the combined dataframe\n",
    "combined_df = combined_df[-(combined_df['Semantic Types Label'] == 'Drug Delivery Device')]\n",
    "combined_df\n",
    "mesh_large_df_sty_less['Preferred Label'].nunique()\n",
    "combined_df = combined_df.sort_values(by='mesh-count_df2', ascending=False).reset_index(drop=True)\n",
    "\n",
    "\n",
    "# define a formatting function that generates a proportional bar\n",
    "def format_bar(value):\n",
    "    max_value = combined_df['mesh-count_df1'].max().max()  # get the maximum value in the dataframe\n",
    "    bar_width = int(value / max_value * 100)  # calculate the width of the bar as a percentage\n",
    "    return f'<div style=\"background-color: blue; width: {bar_width}%\">{value}</div>'\n",
    "\n",
    "def format_bar_2(value):\n",
    "    max_value = combined_df['mesh-count_df2'].max().max()  # get the maximum value in the dataframe\n",
    "    bar_width = int(value / max_value * 100)  # calculate the width of the bar as a percentage\n",
    "    return f'<div style=\"background-color: blue; width: {bar_width}%\">{value}</div>'\n",
    "\n",
    "# apply the formatting function to the dataframe\n",
    "df_formatted = combined_df.style.format({'mesh-count_df1': format_bar, 'mesh-count_df2': format_bar_2})\n",
    "\n",
    "# save the formatted dataframe to an HTML file\n",
    "with open('formatted_dataframe.html', 'w') as f:\n",
    "    f.write(df_formatted.render())\n",
    "\n",
    "# display the formatted dataframe\n",
    "df_formatted"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Reference MeSH build"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build coherent and comprehensive lists of mesh term related to different topics\n",
    "- nutrigentics\n",
    "- neurodegenerative diseases\n",
    "- skin diseases\n",
    "- infective diseases\n",
    "- reproductive physiology"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check available ref-MeSH lists:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Check available refs:\n",
    "folder_path = \"ref-mesh-archive\"  # Replace with the actual folder path\n",
    "#---------------------------\n",
    "\n",
    "# Create a file path pattern to match CSV files\n",
    "file_pattern = os.path.join(folder_path, \"*.csv\")\n",
    "\n",
    "# Use glob to get a list of file paths matching the pattern\n",
    "csv_files = glob.glob(file_pattern)\n",
    "csv_files_name = []\n",
    "# Print the list of CSV files\n",
    "for file in csv_files:\n",
    "    file_name = os.path.basename(file)\n",
    "    csv_files_name.append(file_name)\n",
    "\n",
    "print('Available reference mesh lists:')\n",
    "csv_files_df = pd.Series(csv_files_name)\n",
    "ref_files_df = csv_files_df[csv_files_df.str.contains('ref_mesh_')].reset_index(drop=True)\n",
    "ref_files_df_small = csv_files_df[csv_files_df.str.contains('small_ref_mesh_ng_')].reset_index(drop=True)\n",
    "\n",
    "file_names = [x.split('mesh_')[1].split('.csv')[0] for x in ref_files_df]\n",
    "file_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build ref-MeSH list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-11T17:03:14.424897700Z",
     "start_time": "2024-04-11T17:03:14.158146800Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "             Preferred Label  Semantic Types Label  \\\n1  Electronic Health Records  Intellectual Product   \n2              Consent Forms  Intellectual Product   \n3     Genealogy and Heraldry  Intellectual Product   \n4               Publications  Intellectual Product   \n5    Pharmaceutical Services  Intellectual Product   \n\n                                            Class ID  mesh_id Semantic Types  \\\n1  http://purl.bioontology.org/ontology/MESH/D057286  D057286           T170   \n2  http://purl.bioontology.org/ontology/MESH/D032962  D032962           T170   \n3  http://purl.bioontology.org/ontology/MESH/D005789  D005789           T170   \n4  http://purl.bioontology.org/ontology/MESH/D011642  D011642           T170   \n5  http://purl.bioontology.org/ontology/MESH/D010593  D010593           T170   \n\n                                            Synonyms  \\\n1  Electronic Medical Record|Medical Record, Elec...   \n2  Informed Consent Documents|Informed Consent Fo...   \n3  Geneology and Heraldry|Heraldry and Genealogy|...   \n4                                        Publication   \n5  Pharmaceutical Service|Services, Pharmaceutic|...   \n\n                                             Parents                CUI  \\\n1  http://purl.bioontology.org/ontology/MESH/D016347  C0079150|C2362543   \n2  http://purl.bioontology.org/ontology/MESH/D007...           C0009797   \n3  http://purl.bioontology.org/ontology/MESH/D006664  C0019232|C0017299   \n4  http://purl.bioontology.org/ontology/MESH/D003146           C0034036   \n5  http://purl.bioontology.org/ontology/MESH/D006296  C0031321|C1449618   \n\n                                AQL  \\\n1  CL EC ES HI IS LJ OG SD SN ST TD   \n2        CL EC ES HI LJ OG SN ST TD   \n3                               NaN   \n4        CL EC ES HI LJ SD SN ST TD   \n5     CL EC ES HI LJ OG SD SN ST TD   \n\n                                              TERMUI  \n1  T751083|T048813|T048814|T048815|T749887|T75108...  \n2                    T466073|T021753|T466072|T450735  \n3                    T017448|T017447|T017449|T017451  \n4                                            T034372  \n5  T552624|T031274|T031273|T031272|T031271|T03126...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Preferred Label</th>\n      <th>Semantic Types Label</th>\n      <th>Class ID</th>\n      <th>mesh_id</th>\n      <th>Semantic Types</th>\n      <th>Synonyms</th>\n      <th>Parents</th>\n      <th>CUI</th>\n      <th>AQL</th>\n      <th>TERMUI</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>Electronic Health Records</td>\n      <td>Intellectual Product</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D057286</td>\n      <td>D057286</td>\n      <td>T170</td>\n      <td>Electronic Medical Record|Medical Record, Elec...</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D016347</td>\n      <td>C0079150|C2362543</td>\n      <td>CL EC ES HI IS LJ OG SD SN ST TD</td>\n      <td>T751083|T048813|T048814|T048815|T749887|T75108...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Consent Forms</td>\n      <td>Intellectual Product</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D032962</td>\n      <td>D032962</td>\n      <td>T170</td>\n      <td>Informed Consent Documents|Informed Consent Fo...</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D007...</td>\n      <td>C0009797</td>\n      <td>CL EC ES HI LJ OG SN ST TD</td>\n      <td>T466073|T021753|T466072|T450735</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Genealogy and Heraldry</td>\n      <td>Intellectual Product</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D005789</td>\n      <td>D005789</td>\n      <td>T170</td>\n      <td>Geneology and Heraldry|Heraldry and Genealogy|...</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D006664</td>\n      <td>C0019232|C0017299</td>\n      <td>NaN</td>\n      <td>T017448|T017447|T017449|T017451</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Publications</td>\n      <td>Intellectual Product</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D011642</td>\n      <td>D011642</td>\n      <td>T170</td>\n      <td>Publication</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D003146</td>\n      <td>C0034036</td>\n      <td>CL EC ES HI LJ SD SN ST TD</td>\n      <td>T034372</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Pharmaceutical Services</td>\n      <td>Intellectual Product</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D010593</td>\n      <td>D010593</td>\n      <td>T170</td>\n      <td>Pharmaceutical Service|Services, Pharmaceutic|...</td>\n      <td>http://purl.bioontology.org/ontology/MESH/D006296</td>\n      <td>C0031321|C1449618</td>\n      <td>CL EC ES HI LJ OG SD SN ST TD</td>\n      <td>T552624|T031274|T031273|T031272|T031271|T03126...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if 'mesh_litvar_sty' not in locals():\n",
    "    mesh_litvar_sty = pd.read_csv('ref-mesh-archive/MESH_STY_LITVAR1.csv',index_col=0)\n",
    "    print('MESH_STY_LITVAR1 imported from csv')\n",
    "mesh_litvar_sty.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Ref-MeSH from Topic Term Lists\n",
    "For list of terms defined by the user: search in Preferred Labels and Synonyms corresponding mesh entry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define Biomedical topics\n",
    "nutritional_topic = [['diseases and disorders realted to nutrition and diet ', 'diet, food consuption, eating behaviour and nutrition']]\n",
    "infective_topic = [['infective agents, bacteria, virus and protozoan','infective diseases']]\n",
    "reproductive_topic = [['reproductive system physiology','reproductive system pathology', 'Assisted reproductive technology']]\n",
    "female_infertility_topic = [['female infertility, genetic imprinting and maternal effect']]\n",
    "\n",
    "nutritional_topics = [\n",
    "    ['Obesity, overweight and body weight control', 'compulsive eating behavior'],\n",
    "    ['cardiovascular diseases','physiological processes realted to cardiovascular diseases','lipid metabolism in the context of cardiovascular diseases'],\n",
    "    ['Diabetes Melitus Type II and metabolic syndrome'],\n",
    "    ['Vitamin metabolism and Vitamins recommended intake levels','Micronutrients metabolism and Micronutrient recommended intake levels', 'disease related to vitamins and micronutrients deficiency'],\n",
    "    ['eating behaviour and taste sensation'],\n",
    "    ['food intolerances'],\n",
    "    ['food allergies'],\n",
    "    ['diet-induced oxidative stress'],\n",
    "    ['metabolism of xenobiotics'],\n",
    "]\n",
    "chosen_topic = nutritional_topics\n",
    "pd.Series(chosen_topic)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### NLTK Processor"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# download nltk requirements\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['exampl', 'sentenc', 'demonstr', 'stem', 'stopword', 'remov', 'lower', 'oper', 'punctuat', 'remov']\n"
     ]
    },
    {
     "data": {
      "text/plain": "                  Preferred Label                label_tokens  mesh_id  \\\n1       Electronic Health Records  [electron, health, record]  D057286   \n2                   Consent Forms             [consent, form]  D032962   \n3          Genealogy and Heraldry        [genealog, heraldri]  D005789   \n4                    Publications                    [public]  D011642   \n5         Pharmaceutical Services        [pharmaceut, servic]  D010593   \n...                           ...                         ...      ...   \n516150          Animals, Congenic              [anim, congen]  D020296   \n516151                Vertebrates                   [vertebr]  D014714   \n516152                     Humans                     [human]  D006801   \n516153                 Human Body               [human, bodi]  D018594   \n516155      Carbohydrate Sequence        [carbohydr, sequenc]  D002240   \n\n         Semantic Types Label  \n1        Intellectual Product  \n2        Intellectual Product  \n3        Intellectual Product  \n4        Intellectual Product  \n5        Intellectual Product  \n...                       ...  \n516150             Vertebrate  \n516151             Vertebrate  \n516152                  Human  \n516153                  Human  \n516155  Carbohydrate Sequence  \n\n[31741 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Preferred Label</th>\n      <th>label_tokens</th>\n      <th>mesh_id</th>\n      <th>Semantic Types Label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>Electronic Health Records</td>\n      <td>[electron, health, record]</td>\n      <td>D057286</td>\n      <td>Intellectual Product</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Consent Forms</td>\n      <td>[consent, form]</td>\n      <td>D032962</td>\n      <td>Intellectual Product</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Genealogy and Heraldry</td>\n      <td>[genealog, heraldri]</td>\n      <td>D005789</td>\n      <td>Intellectual Product</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Publications</td>\n      <td>[public]</td>\n      <td>D011642</td>\n      <td>Intellectual Product</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Pharmaceutical Services</td>\n      <td>[pharmaceut, servic]</td>\n      <td>D010593</td>\n      <td>Intellectual Product</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>516150</th>\n      <td>Animals, Congenic</td>\n      <td>[anim, congen]</td>\n      <td>D020296</td>\n      <td>Vertebrate</td>\n    </tr>\n    <tr>\n      <th>516151</th>\n      <td>Vertebrates</td>\n      <td>[vertebr]</td>\n      <td>D014714</td>\n      <td>Vertebrate</td>\n    </tr>\n    <tr>\n      <th>516152</th>\n      <td>Humans</td>\n      <td>[human]</td>\n      <td>D006801</td>\n      <td>Human</td>\n    </tr>\n    <tr>\n      <th>516153</th>\n      <td>Human Body</td>\n      <td>[human, bodi]</td>\n      <td>D018594</td>\n      <td>Human</td>\n    </tr>\n    <tr>\n      <th>516155</th>\n      <td>Carbohydrate Sequence</td>\n      <td>[carbohydr, sequenc]</td>\n      <td>D002240</td>\n      <td>Carbohydrate Sequence</td>\n    </tr>\n  </tbody>\n</table>\n<p>31741 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###python\n",
    "# Updating libraries from NLTK and adding string for punctuation removal\n",
    "import nltk\n",
    "import string\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "# Function for text preprocessing: tokenization, stopping, stemming, lowering, and punctuation removal\n",
    "def process_text(text, replace=False):\n",
    "    # Removing punctuation\n",
    "    if replace:\n",
    "        text = text.replace('-',' ')\n",
    "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "\n",
    "    # Tokenizing the text\n",
    "    tokens = word_tokenize(text)\n",
    "    # Lowercasing all tokens\n",
    "    tokens = [token.lower() for token in tokens]\n",
    "    # Removing stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "    # Stemming words\n",
    "    stemmer = PorterStemmer()\n",
    "    stemmed_tokens = [stemmer.stem(token) for token in filtered_tokens]\n",
    "\n",
    "    return stemmed_tokens\n",
    "\n",
    "## Clean ref-mesh from biases term >>>>>>>>>>>>>>>\n",
    "def clean_mesh(get_rid_list, df, save_clean=False, path = 'ref-mesh-archive/candidate_ref_mesh/'):\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    for get_rid in get_rid_list:\n",
    "        mask = df['Preferred Label'].str.contains(get_rid)\n",
    "        df = df[-mask]\n",
    "\n",
    "    if save_clean:\n",
    "        df.to_csv(path+'ref_mesh_'+topic_label+'.csv')\n",
    "    return df\n",
    "\n",
    "## Rip semantic-types from ref-mesh >>>>>>>>>>>>>>>\n",
    "def semantic_ripper(rip_list, topic_mesh_all, save=False, path = 'ref-mesh-archive/candidate_ref_mesh/', tag=''):\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    mask = topic_mesh_all['Semantic Types Label'].isin(rip_list)\n",
    "    topic_mesh_all_redux = topic_mesh_all[-mask]\n",
    "\n",
    "    if save:\n",
    "        topic_mesh_all_redux[['Preferred Label', 'Semantic Types Label', 'mesh_id']].to_csv(path+'ref_mesh_'+topic_label+tag+'.csv')\n",
    "    return topic_mesh_all_redux\n",
    "\n",
    "# Sample text\n",
    "text = \"This is an example sentence demonstrating the stemming, stop-word removal, lowering operations, and punctuation removal!\"\n",
    "preprocessed_text = process_text(text)\n",
    "print(preprocessed_text)\n",
    "\n",
    "\n",
    "\n",
    "#====================================================================================\n",
    "# Tokenize and process MeSH 'Preferred Labels' =======================================\n",
    "mother_df = mesh_litvar_sty\n",
    "replace = True\n",
    "mother_df['label_tokens'] = mother_df['Preferred Label'].apply(process_text, replace = replace)\n",
    "mother_df[['Preferred Label','label_tokens','mesh_id','Semantic Types Label']]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-11T17:03:55.121735500Z",
     "start_time": "2024-04-11T17:03:23.772557Z"
    }
   },
   "execution_count": 6
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Import topic-terms"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "folder_path = \"ref-mesh-archive/topic_terms\"\n",
    "file_pattern = os.path.join(folder_path, \"*.csv\")\n",
    "csv_files = glob.glob(file_pattern)\n",
    "csv_files_name = []\n",
    "for file in csv_files:\n",
    "    file_name = os.path.basename(file)\n",
    "    csv_files_name.append(file_name)\n",
    "\n",
    "#print('Available topic terms:')\n",
    "csv_files_df = pd.Series(csv_files_name)\n",
    "topic_terms_file = csv_files_df[csv_files_df.str.contains('topic_')]\n",
    "\n",
    "file_id = int(input('Available topic terms, select index:\\n'+str(pd.Series(topic_terms_file))))\n",
    "filename = topic_terms_file[file_id]\n",
    "topic_all_ser_df = pd.read_csv(folder_path+'/'+filename)\n",
    "\n",
    "topic_label = re.search('topic_terms_(.*).csv', filename).group(1)\n",
    "#-----------------------------------------------\n",
    "\n",
    "topic_all_ser = topic_all_ser_df.topic_terms.to_list()\n",
    "print(len(topic_all_ser))\n",
    "print(filename)\n",
    "\n",
    "################################################################\n",
    "### get gpt-terms\n",
    "gpt_filename = filename.replace('topic','gpt')\n",
    "print(gpt_filename)\n",
    "\n",
    "folder_path = \"ref-mesh-archive/gpt_terms\"\n",
    "filename = gpt_filename\n",
    "gpt_all_ser_df = pd.read_csv(folder_path+'/'+filename)\n",
    "\n",
    "topic_label = re.search('gpt_terms_(.*).csv', filename).group(1)\n",
    "#-----------------------------------------------\n",
    "\n",
    "gpt_all_ser = gpt_all_ser_df.gpt_terms.to_list()\n",
    "print('gpt terms loaded')\n",
    "\n",
    "gpt_all_ser = gpt_all_ser_df['gpt_terms'].drop_duplicates()\n",
    "#topic_all_ser = gpt_all_ser_df['gpt_terms'].astype(str).str.lower().drop_duplicates()\n",
    "#topic_all_ser.name = 'topic_terms'\n",
    "len(gpt_all_ser)"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cross 'topic-terms' with 'litvar_mesh'"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "#get mesh\n",
    "topic_mesh_all = pd.DataFrame(columns=mother_df.columns) #Empty DataFrame, same structure\n",
    "time1 =datetime.now()\n",
    "for term in topic_all_ser:\n",
    "    term_tokens = process_text(term, replace = replace)\n",
    "    mask = mother_df['label_tokens'].apply(lambda x: all(term in x for term in term_tokens))\n",
    "    meshy = mother_df[mask]\n",
    "\n",
    "    topic_mesh_all = pd.concat([meshy,topic_mesh_all])\n",
    "time2 =datetime.now()\n",
    "print(time2-time1)\n",
    "\n",
    "#analyze semantics\n",
    "topic_mesh_all_group = topic_mesh_all.groupby('Semantic Types Label').describe()\n",
    "topic_mesh_all_group_sem = topic_mesh_all_group.index.to_list()\n",
    "\n",
    "new_name = 'topic_mesh_'+topic_label\n",
    "exec(f'{new_name} = topic_mesh_all')\n",
    "print(new_name, 'created')\n",
    "print('   topic terms:', len(topic_all_ser))\n",
    "print('   mesh found', topic_mesh_all['Preferred Label'].nunique())\n",
    "print('   semantic groups',topic_mesh_all['Semantic Types Label'].nunique())\n",
    "\n",
    "# GET ref mesh list\n",
    "if os.path.exists('ref-mesh-archive/ref_mesh_'+topic_label+'.csv'):\n",
    "    print('ref_mesh_'+topic_label+'.csv alreday exists')\n",
    "    if False:#simple_bool('Overwrite ref-mesh.csv?'):\n",
    "        globals()[new_name][['Preferred Label', 'Semantic Types Label']].to_csv('ref-mesh-archive/ref_mesh_'+topic_label+'.csv')\n",
    "        print('overwritten')\n",
    "else:\n",
    "    globals()[new_name][['Preferred Label', 'Semantic Types Label']].to_csv('ref-mesh-archive/ref_mesh_'+topic_label+'.csv')\n",
    "    print('\\nref_mesh_'+topic_label+'.csv saved')\n",
    "\n",
    "topic_mesh_all[['Preferred Label','mesh_id','Semantic Types Label']].drop_duplicates()\n",
    "\n",
    "# Please clean manually MeSH List to remove ambiguity\n",
    "\n",
    "\n",
    "\n",
    "##################################################################################\n",
    "# Use gpt terms to filter mother_df and Get MeSH:\n",
    "\n",
    "#get mesh\n",
    "chat_mesh_all = pd.DataFrame(columns=mother_df.columns) #Empty DataFrame, same structure\n",
    "time1 =datetime.now()\n",
    "for i in gpt_all_ser:\n",
    "    #meshy = mother_df[mother_df['Preferred Label'].str.contains(i).fillna(False)]\n",
    "    meshy = mother_df[mother_df['Preferred Label'].str.extract(f'({i})').notna().any(axis=1)]\n",
    "    #print(meshy['Preferred Label'])\n",
    "    chat_mesh_all = pd.concat([meshy,chat_mesh_all])\n",
    "time2 =datetime.now()\n",
    "print(time2-time1)\n",
    "#chat_mesh_all[['Preferred Label','mesh_id','Semantic Types Label']].drop_duplicates()"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Clean and save 'candidate_ref_mesh'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Clean and save 'candidate_ref_mesh'\n",
    "folder_path_ = \"ref-mesh-archive/candidate_ref_mesh\"\n",
    "if not os.path.exists(folder_path_):\n",
    "    os.makedirs(folder_path_)\n",
    "\n",
    "mesh_rid_list = ['oma','Apop','Cell','gen Ty','Cilia','Cytokinesis','DNA','Fluorescein','Intercellular Signaling','Leukem','Lysergic Acid','Loeys-Dietz','Mitochondrial','Peptide Hormones','Toll-Like','Tumor', 'Neoplasm', 'RNA','Gene','Diethyl', 'Nucleos', 'Motif', 'Domain', 'Virus', 'virus','Infant', 'Child', 'Adolescent', 'Elder', 'Maternal', 'Youth', 'Man', 'Woman', 'National', 'Neoplasm', 'Reproductive', 'Sexual', 'Genome', 'Animal', 'Doping', 'Social', 'Urban','Health C','Health E','Health F','Health I','Health P','Health S','Health T','ty Health','le Health','ic Health','Polymor','Genetic','Risk F','Proteins','Protein','DNA','RNA', 'Polymor','Genetic', 'Pharma', 'Cosme', 'Drug', 'Distribution']\n",
    "\n",
    "topic_mesh_all = clean_mesh(mesh_rid_list, topic_mesh_all, save_clean=False)\n",
    "chat_mesh_all = clean_mesh(mesh_rid_list, chat_mesh_all, save_clean=False)\n",
    "\n",
    "semantic_rip_list = ['Cell', 'Receptor', 'Hormone', 'Tissue', 'Body Part, Organ, or Organ Component', 'Congenital Abnormality', 'Anatomical Abnormality', 'Indicator, Reagent, or Diagnostic Aid', 'Neoplastic Process', 'Mammal', 'Gene or Genome', 'Element, Ion, or Isotope']\n",
    "\n",
    "topic_mesh_all = semantic_ripper(semantic_rip_list, topic_mesh_all, save=True)\n",
    "chat_mesh_all  = semantic_ripper(semantic_rip_list, chat_mesh_all, save=False)\n",
    "\n",
    "# Get the unique values in topic_mesh_all['Preferred Label'] that are not in chat_mesh_all['Preferred Label']\n",
    "unique_values = list(set(topic_mesh_all['Preferred Label']) - set(chat_mesh_all['Preferred Label']))\n",
    "rev_values = list(set(chat_mesh_all['Preferred Label']) - set(topic_mesh_all['Preferred Label']))\n",
    "# Print the unique values\n",
    "display(pd.Series(unique_values))\n",
    "if len(rev_values) > 0:\n",
    "    display(pd.Series(rev_values))\n",
    "else:\n",
    "    print(rev_values)"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "topic_mesh_all['Semantic Types Label'].drop_duplicates().sort_values()\n",
    "chat_mesh_all['Semantic Types Label'].drop_duplicates().sort_values()\n",
    "''"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "###python\n",
    "# Find the intersection of the two series\n",
    "intersection_terms = set(chat_mesh_all['Semantic Types Label'].drop_duplicates()).intersection(set(topic_mesh_all['Semantic Types Label'].drop_duplicates()))\n",
    "\n",
    "# Get all terms that are in the first series but not in the second\n",
    "terms_not_in_second = set(topic_mesh_all['Semantic Types Label'].drop_duplicates()).difference(set(chat_mesh_all['Semantic Types Label'].drop_duplicates()))\n",
    "\n",
    "intersection_terms\n",
    "op.pc.copy(str(list(terms_not_in_second)))\n",
    "terms_not_in_second\n",
    "###"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "161-17 CVD\n",
    "17-4 OBBMI\n",
    "21-5\n",
    "3-0 OXI\n",
    "71-3 DIAB\n",
    "80-0 NUTRI\n",
    "88-0 xeno\n",
    "32-4 vitam\n",
    "50-0 eat\n",
    "80-0 aller\n",
    "10-3 intoll\n",
    "\"\"\"\n",
    "topic_mesh_all['Preferred Label'].drop_duplicates()\n",
    "pd.Series(unique_values)"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "Get rid of gpt terms: [history]\n",
    "on nutri: [ 'Immune System',    'Nail Concentrations',    'Natural Killer Cells',    'Nerve Function',    'PYY3-36',    'Phenotype',    'Pregnancy',    'Single Nucleotide Polymorphisms',    'Transplantation',    'T-Lymphocytes',    'Dendritic Cells',    'Macrophages',    'Graft',    'Phagocytosis',    'Calcium',    'Iron',    'Phosphorus',    'Copper',    'Development',    'Developmental',    'Gene Expression']\n",
    "on ob_bmi: ['Transcriptome',\n",
    "'RNA, Ribosomal, 16S','Proteomics','Prevalence','Epidemiologic Studies','Epidemiology','Epidemics','Sick Role']\n",
    "oxi_stress. ['Chemokines','Cytokines','Diet','Interleukin']\n",
    "xeno: ['Pharma','Cosme','Drug','Distribution']"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "print(len(pd.read_csv('ref-mesh-archive/ref_mesh_cvd.csv')['Preferred Label'].drop_duplicates()))\n",
    "print(len(pd.read_csv('ref-mesh-archive/ref_mesh_cvd_lipo.csv')['Preferred Label'].drop_duplicates()))\n",
    "gpt_all_ser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "get rid: storical\n",
    "ob_bmi mesh ['Asthma','Cardio','Child H','Chromosome','Delivery','Diethyl','Drug','Employer','Electronic','Epigenomics','Ocular','Pulmon','Malignant','Organizat','Person','Occupat','Literacy','Level Se','h Prop','h Reso','es Res','for the Age','Indigen','h Syst','Deter','Antibod','Insulin-Like Growth Factor','Express','Patient','Pregnancy Complications','Rural','Sleep','Snow','Veter','Volunt','Water']\n",
    "\n",
    "cvd_list = ['oma','Apop','Cell','gen Ty','Cilia','Cytokinesis','DNA','Diabet','Fluorescein','Intercellular Signaling','Leukem','Lysergic Acid','Loeys-Dietz','Mitochondrial','Peptide Hormones','Toll-Like','Tumor']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Ref-MeSH Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check available ref-MeSH lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Check avalable refs:\n",
    "folder_path = \"ref-mesh-archive\"  # Replace with the actual folder path\n",
    "#---------------------------\n",
    "\n",
    "# Create a file path pattern to match CSV files\n",
    "file_pattern = os.path.join(folder_path, \"*.csv\")\n",
    "\n",
    "# Use glob to get a list of file paths matching the pattern\n",
    "csv_files = glob.glob(file_pattern)\n",
    "csv_files_name = []\n",
    "# Print the list of CSV files\n",
    "for file in csv_files:\n",
    "    file_name = os.path.basename(file)\n",
    "    csv_files_name.append(file_name)\n",
    "\n",
    "print('Available reference mesh lists:')\n",
    "csv_files_df = pd.Series(csv_files_name)\n",
    "ref_files_df = csv_files_df[csv_files_df.str.contains('ref_mesh_')].reset_index(drop=True)\n",
    "ref_files_df_small = csv_files_df[csv_files_df.str.contains('small_ref_mesh_ng_')].reset_index(drop=True)\n",
    "\n",
    "file_names = [x.split('mesh_')[1].split('.csv')[0] for x in ref_files_df]\n",
    "file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ref_names = ['ref_'] * 10\n",
    "ref_names = [string + str(i) for i, string in enumerate(ref_names)]\n",
    "\n",
    "ref_list = []\n",
    "for name, ref, save  in zip(ref_names, ref_files_df, ref_files_df):\n",
    "    df = pd.read_csv('ref-mesh-archive/'+ref, index_col=0).reset_index(drop=True)\n",
    "    df = df.drop('Semantic Types Label', axis=1).drop_duplicates()\n",
    "    df = df.sort_values(by='mesh_id',ascending=True)\n",
    "    globals()[name] = df\n",
    "    #df.to_csv('ref-mesh-archive/small_'+save) # 'small' for representation purposes only\n",
    "    ref_list.append(df)\n",
    "\n",
    "ref_list[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1. Ref MeSH cleaner"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Mesh Cleaner"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "#Get rid from bias generating MeSH and run again GRPM Module 3\n",
    "\n",
    "# put here manually selected mesh to purge\n",
    "xeno = ['Polymorphism, Genetic', 'DNA Methylation','Liver Neoplasms','Energy Metabolism','Metabolism, Inborn Errors','X-Ray Absorption Spectroscopy','Blood-Testis Barrier']\n",
    "oxi_stress = ['Food Deprivation','Food Analysis','Food, Formulated','Food Microbiology','Food Intolerance','Food Quality','Food Industry','Foods, Specialized','Food Chain','Foods, Specialized','Food Chain']\n",
    "intol = ['Depression','Osteoporosis','Immunotherapy','Anxiety','Anti-Inflammatory Agents, Non-Steroidal','Immunotherapy, Adoptive','Desensitization, Immunologic','Peanut Hypersensitivity','Milk Hypersensitivity', 'Egg Hypersensitivity','Immunotherapy, Active','Neurologic Manifestations' ,'Infectious Anemia Virus, Equine','Nut and Peanut Hypersensitivity' ,'Diarrhea Virus 1, Bovine Viral','Eye Movement Desensitization Reprocessing','Diarrhea Virus 2, Bovine Viral']\n",
    "eat_taste = ['Blood Glucose','Mouth Neoplasms','Glucose Transporter Type 1','Glucose Transporter Type 2','Auditory Perception','Citric Acid Cycle','Glucose Transporter Type 4','Loeys-Dietz Syndrome','United States Food and Drug Administration','Sodium-Glucose Transporter 2','Sodium-Glucose Transporter 2 Inhibitors','Glucose-6-Phosphate','Glucose Transporter Type 3','Pitch Perception','Depth Perception','Glucose-1-Phosphate Adenylyltransferase','Glucose Dehydrogenases','Glucosephosphates']\n",
    "cvd = ['DNA, Mitochondrial','Cell Differentiation','Protein Transport','Mitochondrial Proteins','Toll-Like Receptors','Genome, Mitochondrial','Genes, Mitochondrial','Mitochondrial Precursor Protein Import Complex Proteins','Mitochondrial Precursor Protein Import Complex Proteins','Mitochondrial Proton-Translocating ATPases','Mitochondrial Dynamics','Mitochondrial Membranes','Leukemia, Plasma Cell','Heart Neoplasms']\n",
    "dmt2_ms = [ 'MicroRNAs', 'Mitochondria', 'DNA, Mitochondrial', 'Mitochondrial Proteins', 'Pancreatic Neoplasms','Autophagy','Mitochondrial Diseases','Genome, Mitochondrial','Genes, Mitochondrial','Mitochondrial Precursor Protein Import Complex Proteins','Mitochondrial Precursor Protein Import Complex Proteins','Mitochondrial Dynamics','Mitochondrial Membranes','Mitochondrial Myopathies','RNA, Mitochondrial', 'Mitochondrial Encephalomyopathies','AIDS-Associated Nephropathy','Familial Primary Pulmonary Hypertension','Mitochondria, Heart','Autophagy-Related Protein-1 Homolog','Ocular Hypertension','Autophagy-Related Protein 7','Mitochondrial Trifunctional Protein','Mitochondrial Permeability Transition Pore','Mitochondrial Uncoupling Proteins','Mitochondrial ADP, ATP Translocases','Autophagy-Related Protein 8 Family','Mitochondrial Ribosomes','Mitochondrial Trifunctional Protein, alpha Subunit','Autophagy-Related Protein 12','Mitochondrial Turnover','Lysergic Acid Diethylamide','Mitochondrial Trifunctional Protein, beta Subunit','Chaperone-Mediated Autophagy','Mitochondrial Swelling','Mitochondrial Transmembrane Permeability-Driven Necrosis','Mitochondrial Size']\n",
    "ob_bmi = ['Infant','Child','Adolescent','Elder','Maternal','Youth','Man','Woman','National','Neoplasm''Epidemiolo','Reproductive','Sexual','Genome','Animal','Doping','Social','Urban''Health C','Health E','Health F','Health I','Health P','Health S','Health T','ty Health','le Health','ic Health','Polymor','Genetic','Risk F']\n",
    "\n",
    "# note: to list above stands for examples (ref mesh already cleaned in archive)\n",
    "\n",
    "#clean mesh ref\n",
    "get_rid_list = xeno\n",
    "\n",
    "#check mesh ref\n",
    "topic_label = 'ng_' + 'xeno'\n",
    "path_ = 'ref-mesh-archive/candidate_ref_mesh/'\n",
    "dff = pd.read_csv(path_+'ref_mesh_'+topic_label+'.csv', index_col=0)\n",
    "\n",
    "dff_rid = clean_mesh(get_rid_list, dff, path = 'ref-mesh-archive/candidate_ref_mesh/', save_clean =True)\n",
    "\n",
    "print(dff.describe(),'\\n-----------------------------------------------------------------')\n",
    "print(dff_rid.describe())"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Semantic type ripper"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       Preferred Label                 Semantic Types Label     mesh_id\n",
      "count              609                                  609         609\n",
      "unique             243                                   58         243\n",
      "top      Diet, Healthy  Therapeutic or Preventive Procedure  D000072001\n",
      "freq                18                                   60          18 \n",
      "-----------------------------------------------------------------\n",
      "       Preferred Label                 Semantic Types Label     mesh_id\n",
      "count              543                                  543         543\n",
      "unique             232                                   51         232\n",
      "top      Diet, Healthy  Therapeutic or Preventive Procedure  D000072001\n",
      "freq                18                                   60          18\n"
     ]
    }
   ],
   "source": [
    "#Get rid from bias generating MeSH and run again GRPM Module 3\n",
    "\n",
    "semantic_rip_list = ['Cell', 'Receptor', 'Hormone', 'Tissue', 'Body Part, Organ, or Organ Component', 'Congenital Abnormality', 'Anatomical Abnormality', 'Indicator, Reagent, or Diagnostic Aid', 'Neoplastic Process', 'Mammal', 'Gene or Genome', 'Element, Ion, or Isotope']\n",
    "\n",
    "#check mesh ref\n",
    "topic_label = 'ng_' + 'ob_bmi'\n",
    "#path_ = 'ref-mesh-archive/candidate_ref_mesh/'\n",
    "path_ = 'ref-mesh-archive/'\n",
    "dff = pd.read_csv(path_+'ref_mesh_'+topic_label+'.csv', index_col=0)\n",
    "\n",
    "dff_rid = semantic_ripper(semantic_rip_list, dff, path = 'ref-mesh-archive/', save=False)\n",
    "\n",
    "print(dff.describe(),'\\n-----------------------------------------------------------------')\n",
    "print(dff_rid.describe())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-11T17:10:06.578508400Z",
     "start_time": "2024-04-11T17:10:06.334984500Z"
    }
   },
   "execution_count": 10
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 2. create JSON sheet for ref-mesh lists\n",
    "https://jsoneditoronline.org/#left=local.lejobi&right=local.lejobi\n",
    "https://www.convertjson.com/json-to-html-table.htm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "\n",
    "folder_path = \"ref-mesh-archive\"\n",
    "\n",
    "num_elements = len(ref_files_df)  # Number of elements you want in the list\n",
    "value_list = []\n",
    "for i in range(1, num_elements + 1):\n",
    "    element = \"file_pat\" + str(i)\n",
    "    value_list.append(element)\n",
    "\n",
    "jdata_list_n = ['jdata_'] * 10\n",
    "jdata_list_n = [string + str(i) for i, string in enumerate(jdata_list_n)]\n",
    "\n",
    "jdata_list = []\n",
    "for name in jdata_list_n:\n",
    "    empty =[]\n",
    "    globals()[name] = empty\n",
    "    jdata_list.append(empty)\n",
    "\n",
    "\n",
    "file_patterns = []\n",
    "choose_ref = ref_files_df # or ref_files_df_small\n",
    "for i in choose_ref:\n",
    "    file_patterns.append(os.path.join(folder_path, i))\n",
    "\n",
    "for i, j  in zip(range(len(ref_files_df)), jdata_list):\n",
    "\n",
    "    with open(file_patterns[i], 'r') as file1:\n",
    "        csv_reader = csv.DictReader(file1)\n",
    "        # for row in csv_reader:\n",
    "        #     j_data1.append(row)\n",
    "        for row in csv_reader:\n",
    "            modified_row = {key: value for key, value in row.items() if key != ''} #drop col\n",
    "            j.append(modified_row)"
   ],
   "metadata": {
    "collapsed": false
   },
   "execution_count": null
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#combined_data = {\n",
    "#    'data1': j_data1,\n",
    "#    'data2': j_data2 }\n",
    "\n",
    "combined_data = {}\n",
    "\n",
    "titles = [\n",
    "    \"Genaral Nutrition\",\n",
    "    \"Obesity, Weight Control and Compulsive Eating\",\n",
    "    \"Cardiovascular Health and Lipid Metabolism\",\n",
    "    \"Diabetes Mellitus Type II and Metabolic Syndrome\",\n",
    "    \"Vitamin and Micronutrients Metabolism and Deficiency-Related Diseases\",\n",
    "    \"Eating Behavior and Taste Sensation\",\n",
    "    \"Food Intolerances\",\n",
    "    \"Food Allergies\",\n",
    "    \"Diet-induced Oxidative Stress\",\n",
    "    \"Xenobiotics Metabolism\",\n",
    "]\n",
    "\n",
    "for tit, j_data in zip(titles, jdata_list):\n",
    "    key = tit\n",
    "    combined_data[key] = j_data\n",
    "\n",
    "#for i, j_data in enumerate(jdata_list, start=1):\n",
    "#    key = 'data' + str(i)\n",
    "#    combined_data[key] = j_data\n",
    "\n",
    "print(combined_data)\n",
    "#pd.DataFrame(combined_data)\n",
    "import pyperclip\n",
    "pyperclip.copy(str(combined_data))\n",
    "\n",
    "json_file = 'ref-mesh-archive/nutrigenetis_referencemesh_small.json'\n",
    "with open(json_file, 'w') as outfile:\n",
    "    json.dump(combined_data, outfile, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. add mesh_id to MeSH list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# add mesh id to every mesh list\n",
    "label = 'ng_nutri'\n",
    "dff = pd.read_csv('ref-mesh-archive/ref_mesh_'+label+'.csv',index_col=0).reset_index(drop=True)#.drop('Unnamed: 0', axis=1)\n",
    "\n",
    "dff_id = pd.merge(dff, mesh_id_ref_df, left_on='Preferred Label', right_on='Preferred Label')\n",
    "dff_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dff_id.to_csv('ref-mesh-archive/ref_mesh_'+label+'.csv')\n",
    "#grpm_db_merge_id = grpm_db_merge_id.drop('Preferred Label', axis=1)\n",
    "check = pd.read_csv('ref-mesh-archive/ref_mesh_'+label+'.csv', index_col=0)\n",
    "check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chack = check.drop('mesh_id_y',axis=1)\n",
    "check = check.rename(columns={'mesh_id_y': 'mesh_id'})\n",
    "check.to_csv('ref-mesh-archive/ref_mesh_'+label+'.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Generate Random Mesh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random grpm mesh list from MESH-STY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mesh_large_df_sty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Generate Random mesh list from complete df\n",
    "print(mesh_large_df_sty['Preferred Label'].nunique())\n",
    "mesh_large_df_sty_mesh = mesh_large_df_sty['Preferred Label'].drop_duplicates()\n",
    "random_path = 'ref-mesh-archive/random_lists/'\n",
    "\n",
    "size = 603\n",
    "sample_01 = mesh_large_df_sty_mesh.sample(n=size, random_state = 78954)\n",
    "sample_02 = mesh_large_df_sty_mesh.sample(n=size, random_state = 12245)\n",
    "sample_03 = mesh_large_df_sty_mesh.sample(n=size, random_state = 87498)\n",
    "sample_04 = mesh_large_df_sty_mesh.sample(n=size, random_state = 56798)\n",
    "sample_05 = mesh_large_df_sty_mesh.sample(n=size, random_state = 34565)\n",
    "sample_06 = mesh_large_df_sty_mesh.sample(n=size, random_state = 76523)\n",
    "sample_07 = mesh_large_df_sty_mesh.sample(n=size, random_state = 78968)\n",
    "sample_08 = mesh_large_df_sty_mesh.sample(n=size, random_state = 56845)\n",
    "sample_09 = mesh_large_df_sty_mesh.sample(n=size, random_state = 76624)\n",
    "sample_10 = mesh_large_df_sty_mesh.sample(n=size, random_state = 23845)\n",
    "\n",
    "sample_01.to_csv(random_path+'random_01.csv')\n",
    "sample_02.to_csv(random_path+'random_02.csv')\n",
    "sample_03.to_csv(random_path+'random_03.csv')\n",
    "sample_04.to_csv(random_path+'random_04.csv')\n",
    "sample_05.to_csv(random_path+'random_05.csv')\n",
    "sample_06.to_csv(random_path+'random_06.csv')\n",
    "sample_07.to_csv(random_path+'random_07.csv')\n",
    "sample_08.to_csv(random_path+'random_08.csv')\n",
    "sample_09.to_csv(random_path+'random_09.csv')\n",
    "sample_10.to_csv(random_path+'random_10.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random grpm mesh list from MESH-STY-LITVAR\n",
    "Generate Random Mesh list from grpm df\n",
    "\n",
    "    grpm_db_mesh = pd.read_csv('ref-mesh-archive/grpm_db_mesh.csv')\n",
    "    grpm_db_mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "if os.path.exists('ref-mesh-archive/grpm_db_mesh.csv'):\n",
    "    grpm_db_mesh = pd.read_csv('ref-mesh-archive/grpm_db_mesh.csv',index_col=0)\n",
    "    print('grpm mesh imported from archive')\n",
    "else:\n",
    "    grpmdb_table = pd.read_csv('grpm_dataset\\grpm_db_pcg\\grpm_table_output.csv')\n",
    "    grpm_db_mesh = pd.DataFrame(grpmdb_table.mesh.drop_duplicates())\n",
    "    type(grpm_db_mesh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "rand = pd.read_csv('ref-mesh-archive/random_grpm_03.csv')\n",
    "mask= grpm_db_mesh.mesh.isin(rand['mesh'])\n",
    "grpm_db_mesh[mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# randomize 10 samples\n",
    "size = 450\n",
    "sample_grpm_01 = grpm_db_mesh.mesh.sample(n=size, random_state = 78954)\n",
    "sample_grpm_02 = grpm_db_mesh.mesh.sample(n=size, random_state = 12245)\n",
    "sample_grpm_03 = grpm_db_mesh.mesh.sample(n=size, random_state = 87498)\n",
    "sample_grpm_04 = grpm_db_mesh.mesh.sample(n=size, random_state = 56798)\n",
    "sample_grpm_05 = grpm_db_mesh.mesh.sample(n=size, random_state = 34565)\n",
    "sample_grpm_06 = grpm_db_mesh.mesh.sample(n=size, random_state = 76523)\n",
    "sample_grpm_07 = grpm_db_mesh.mesh.sample(n=size, random_state = 78968)\n",
    "sample_grpm_08 = grpm_db_mesh.mesh.sample(n=size, random_state = 56845)\n",
    "sample_grpm_09 = grpm_db_mesh.mesh.sample(n=size, random_state = 76624)\n",
    "sample_grpm_00 = grpm_db_mesh.mesh.sample(n=size, random_state = 23845)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#check fidelity\n",
    "mask= grpm_db_mesh.mesh.isin(sample_grpm_01)\n",
    "grpm_db_mesh[mask]\n",
    "#type(sample_grpm_10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save random samples\n",
    "random_path = 'ref-mesh-archive/random_lists/'\n",
    "\n",
    "if os.path.exists('ref-mesh-archive/random_grpm_00.csv'):\n",
    "    sample_grpm_01 = pd.read_csv(random_path+'random_grpm_01.csv',index_col=0)\n",
    "    sample_grpm_02 = pd.read_csv(random_path+'random_grpm_02.csv',index_col=0)\n",
    "    sample_grpm_03 = pd.read_csv(random_path+'random_grpm_03.csv',index_col=0)\n",
    "    sample_grpm_04 = pd.read_csv(random_path+'random_grpm_04.csv',index_col=0)\n",
    "    sample_grpm_05 = pd.read_csv(random_path+'random_grpm_05.csv',index_col=0)\n",
    "    sample_grpm_06 = pd.read_csv(random_path+'random_grpm_06.csv',index_col=0)\n",
    "    sample_grpm_07 = pd.read_csv(random_path+'random_grpm_07.csv',index_col=0)\n",
    "    sample_grpm_08 = pd.read_csv(random_path+'random_grpm_08.csv',index_col=0)\n",
    "    sample_grpm_09 = pd.read_csv(random_path+'random_grpm_09.csv',index_col=0)\n",
    "    sample_grpm_00 = pd.read_csv(random_path+'random_grpm_00.csv',index_col=0)\n",
    "\n",
    "save_rand = False\n",
    "if save_rand == True:\n",
    "    sample_grpm_01.to_csv(random_path+'random_grpm_01.csv')\n",
    "    sample_grpm_02.to_csv(random_path+'random_grpm_02.csv')\n",
    "    sample_grpm_03.to_csv(random_path+'random_grpm_03.csv')\n",
    "    sample_grpm_04.to_csv(random_path+'random_grpm_04.csv')\n",
    "    sample_grpm_05.to_csv(random_path+'random_grpm_05.csv')\n",
    "    sample_grpm_06.to_csv(random_path+'random_grpm_06.csv')\n",
    "    sample_grpm_07.to_csv(random_path+'random_grpm_07.csv')\n",
    "    sample_grpm_08.to_csv(random_path+'random_grpm_08.csv')\n",
    "    sample_grpm_09.to_csv(random_path+'random_grpm_09.csv')\n",
    "    sample_grpm_00.to_csv(random_path+'random_grpm_00.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "type(sample_grpm_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sample_grpm_01_list = sample_grpm_01.to_list()\n",
    "sample_grpm_02_list = sample_grpm_02.to_list()\n",
    "sample_grpm_03_list = sample_grpm_03.to_list()\n",
    "sample_grpm_04_list = sample_grpm_04.to_list()\n",
    "sample_grpm_05_list = sample_grpm_05.to_list()\n",
    "sample_grpm_06_list = sample_grpm_06.to_list()\n",
    "sample_grpm_07_list = sample_grpm_07.to_list()\n",
    "sample_grpm_08_list = sample_grpm_08.to_list()\n",
    "sample_grpm_09_list = sample_grpm_09.to_list()\n",
    "sample_grpm_00_list = sample_grpm_00.to_list()\n",
    "\n",
    "lists= [sample_grpm_01_list,\n",
    "        sample_grpm_02_list,\n",
    "        sample_grpm_03_list,\n",
    "        sample_grpm_04_list,\n",
    "        sample_grpm_05_list,\n",
    "        sample_grpm_06_list,\n",
    "        sample_grpm_07_list,\n",
    "        sample_grpm_08_list,\n",
    "        sample_grpm_09_list,\n",
    "        sample_grpm_00_list]\n",
    "\n",
    "num_lists = len(lists)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Coocuurance method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#(1) Initialize a 2D list of zeros with dimensions equal to the number of lists\n",
    "cooccur_matrix = [[0] * num_lists for i in range(num_lists)]\n",
    "cooccur_matrix = np.zeros((len(lists), len(lists)), dtype=int)\n",
    "\n",
    "type(cooccur_matrix)\n",
    "print(len(set(lists[1]) & set(lists[1])))\n",
    "print(len(set(lists[1]) & set(lists[3])))\n",
    "\n",
    "# Loop over all pairs of lists and count the number of co-occurring elements\n",
    "for i in range(num_lists):\n",
    "    for j in range(num_lists):\n",
    "        if i == j:\n",
    "            cooccur_matrix[i][j] = len(set(lists[i]))\n",
    "        else:\n",
    "            cooccur_matrix[i][j] = len(set(lists[i]) & set(lists[j])) # use set() to extract unique elements\n",
    "\n",
    "print(type(cooccur_matrix))\n",
    "\n",
    "# Convert the 2D list to a Pandas DataFrame\n",
    "cooccur_df = pd.DataFrame(cooccur_matrix,\n",
    "                          columns=['random{}'.format(i+1) for i in range(num_lists)],\n",
    "                          index=['random{}'.format(i+1) for i in range(num_lists)])\n",
    "\n",
    "# Print the resulting DataFrame\n",
    "cooccur_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Random grpm mesh list without rep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# COOCCURANCE MATRIX MODULE------------\n",
    "# second matrix build check\n",
    "sample_grpm_01_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep1.csv')\n",
    "sample_grpm_02_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep2.csv')\n",
    "sample_grpm_03_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep3.csv')\n",
    "sample_grpm_04_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep4.csv')\n",
    "sample_grpm_05_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep5.csv')\n",
    "sample_grpm_06_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep6.csv')\n",
    "sample_grpm_07_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep7.csv')\n",
    "sample_grpm_08_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep8.csv')\n",
    "sample_grpm_09_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep9.csv')\n",
    "sample_grpm_00_norep = pd.read_csv(random_path+'random_grpm_mesh_norep/random_grpm_norep10.csv')\n",
    "\n",
    "lists= [sample_grpm_01_norep.mesh.to_list(),\n",
    "        sample_grpm_02_norep.mesh.to_list(),\n",
    "        sample_grpm_03_norep.mesh.to_list(),\n",
    "        sample_grpm_04_norep.mesh.to_list(),\n",
    "        sample_grpm_05_norep.mesh.to_list(),\n",
    "        sample_grpm_06_norep.mesh.to_list(),\n",
    "        sample_grpm_07_norep.mesh.to_list(),\n",
    "        sample_grpm_08_norep.mesh.to_list(),\n",
    "        sample_grpm_09_norep.mesh.to_list(),\n",
    "        sample_grpm_00_norep.mesh.to_list()]\n",
    "\n",
    "num_lists = len(lists)\n",
    "\n",
    "# Initialize a 2D list of zeros with dimensions equal to the number of lists\n",
    "cooccur_matrix = [[0] * num_lists for i in range(num_lists)]\n",
    "type(cooccur_matrix)\n",
    "print(len(set(lists[1]) & set(lists[1])))\n",
    "print(len(set(lists[1]) & set(lists[3])))\n",
    "\n",
    "# Loop over all pairs of lists and count the number of co-occurring elements\n",
    "for i in range(num_lists):\n",
    "    for j in range(num_lists):\n",
    "        if i == j:\n",
    "            cooccur_matrix[i][j] = len(lists[i])\n",
    "        else:\n",
    "            cooccur_matrix[i][j] = len(set(lists[i]) & set(lists[j]))\n",
    "\n",
    "# Convert the 2D list to a Pandas DataFrame\n",
    "cooccur_df = pd.DataFrame(cooccur_matrix, columns=['random{}'.format(i+1) for i in range(num_lists)], index=['random{}'.format(i+1) for i in range(num_lists)])\n",
    "\n",
    "# Print the resulting DataFrame\n",
    "cooccur_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Print the resulting matrix with row and column headers\n",
    "print(', '.join([''] + ['list{}'.format(i+1) for i in range(num_lists)]))\n",
    "\n",
    "for i in range(num_lists):\n",
    "    row = ['list{}'.format(i+1)]\n",
    "    row.extend(cooccur_matrix[i])\n",
    "    print(', '.join(str(x) for x in row))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "crea 2 set da 5 random mesh senza ripetizioni"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Random grpm mesh list without rep from MESH-STY-LITVAR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# randomize\n",
    "sample_grpm_full = grpm_db_mesh.mesh.sample(n=len(grpm_db_mesh), random_state = 782954)\n",
    "sample_grpm_full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# chunk size\n",
    "size = 450\n",
    "# Split the DataFrame into smaller DataFrames of chunk length 450\n",
    "chunks = np.array_split(sample_grpm_full, len(sample_grpm_full) // 450 + 1)\n",
    "\n",
    "# Print the number of chunks and the length of each chunk\n",
    "for i, chunk in enumerate(chunks):\n",
    "    print('Chunk {}: {} rows'.format(i+1, len(chunk)))\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Save each DataFrame chunk to a separate CSV file\n",
    "for i, chunk in enumerate(chunks):\n",
    "    chunk.to_csv('ref-mesh-archive/random_grpm_norep{}.csv'.format(i+1), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try random mesh list based on STY proportions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#define ref_sty\n",
    "ref_sty = pd.concat([new_ref_sty_neuro_sty, new_nutri_refmesh_sty]).drop_duplicates()\n",
    "strip  = ['Enzyme', 'Bacterium','Organism']\n",
    "strip2  = ['Disease or Syndrome']\n",
    "result = [item for item in new_nutri_refmesh_sty.to_list() if item not in strip]\n",
    "result2 = [item for item in result if item not in strip2]\n",
    "ref_sty_nutri = pd.Series(result)\n",
    "ref_sty_nutri_rest = pd.Series(result2)\n",
    "\n",
    "#mother df\n",
    "print('mother df',mesh_large_df_sty['Preferred Label'].nunique(), len(mesh_large_df_sty['Preferred Label']))\n",
    "\n",
    "#filter mother df with ref sty\n",
    "mesh_large_df_sty_subset = mesh_large_df_sty[mesh_large_df_sty['Semantic Types Label'].isin(ref_sty_nutri)]\n",
    "print('filtered df', mesh_large_df_sty_subset['Preferred Label'].nunique(), len(mesh_large_df_sty_subset['Preferred Label']))\n",
    "#mesh_large_df_sty_subset['Semantic Types Label'].drop_duplicates()\n",
    "mesh_large_df_sty_subset_rest = mesh_large_df_sty_subset[mesh_large_df_sty_subset['Semantic Types Label'].isin(ref_sty_nutri_rest)]\n",
    "\n",
    "mesh_large_df_sty_subset_rest\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mesh_large_df_sty_subset.groupby('Semantic Types Label').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mesh_large_df_sty['Preferred Label'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#get random mesh list from filtered df\n",
    "sample_rest = 603 - 255\n",
    "disease_sample = 255\n",
    "\n",
    "mesh_large_df_sty_subset_disease = mesh_large_df_sty_subset.loc[mesh_large_df_sty_subset['Semantic Types Label'] == 'Disease or Syndrome']\n",
    "random_mesh_disease_01 = mesh_large_df_sty_subset_disease.sample(n=disease_sample, random_state = 42)\n",
    "random_mesh_disease_02 = mesh_large_df_sty_subset_disease.sample(n=disease_sample, random_state = 124)\n",
    "random_mesh_01 = mesh_large_df_sty_subset.sample(n=sample_rest, random_state = 42)\n",
    "random_mesh_02 = mesh_large_df_sty_subset.sample(n=sample_rest, random_state = 124)\n",
    "\n",
    "random_mesh_01_con = pd.concat([random_mesh_disease_01, random_mesh_01])\n",
    "random_mesh_02_con = pd.concat([random_mesh_disease_02, random_mesh_02])\n",
    "random_mesh_02_con"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "if os.path.exists('ref-mesh-archive/random_mesh_01_con.csv'):\n",
    "    pass\n",
    "else:\n",
    "    random_mesh_01_con.to_csv('ref-mesh-archive/random_mesh_01_con.csv')\n",
    "    random_mesh_02_con.to_csv('ref-mesh-archive/random_mesh_02_con.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "random_mesh_02_con.groupby('Semantic Types Label').describe()\n",
    "random_mesh_01_con.groupby('Semantic Types Label').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "disease_sample = 255\n",
    "amino_sample = 92\n",
    "biolog_sample = 65\n",
    "new_nutri_refmesh_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze Reference MeSH list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "if os.path.exists('ref-mesh-archive/ref_mesh_nest.csv'):\n",
    "    ref = pd.read_csv('ref-mesh-archive/ref_mesh_nest.csv', sep=\";\")\n",
    "    print('nested ref mesh', ref['Class ID'].nunique())\n",
    "\n",
    "    ref['Semantic Types'] = ref['Semantic Types'].apply(ast.literal_eval)\n",
    "    #otherwise= dfg['col1'] = df['col1'].str.strip('[]').str.split(', ').apply(lambda x: [int(i) for i in x])\n",
    "    sty = pd.read_csv('ref-mesh-archive/MeshSTY-code.csv',sep=';')\n",
    "    sty = sty.rename(columns={'ID':'Semantic Types'})\n",
    "    print(ref)\n",
    "else:\n",
    "    print('ref_mesh_nest.csv not available')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if 'ref' in globals():\n",
    "    ref_large = []\n",
    "    for i in range(len(ref)):\n",
    "        for sem in ref['Semantic Types'][i]: #dfrspost = mother table\n",
    "            out = ref['Preferred Label'][i],ref['Class ID'][i],sem,ref['Synonyms'][i],ref['Parents'][i],ref['CUI'][i],ref['AQL'][i],ref['TERMUI'][i]\n",
    "            ref_large.append(out)\n",
    "\n",
    "    ref_large_df = pd.DataFrame(ref_large)\n",
    "    new_col_names = ['Preferred Label','Class ID','Semantic Types','Synonyms','Parents','CUI','AQL','TERMUI']\n",
    "    ref_large_df.columns = new_col_names\n",
    "    display(ref_large_df)\n",
    "    #ref_large_df.to_csv('ref-mesh-archive/ref_mesh_large.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mesh semantics analyze on Ref list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## set df on ref mesh sty large\n",
    "\n",
    "if os.path.exists('ref-mesh-archive/ref_mesh_sty_large.csv'):\n",
    "    ref_sty = pd.read_csv('ref-mesh-archive/ref_mesh_sty_large.csv', index_col=0)\n",
    "else:\n",
    "    ref_sty = pd.merge(ref_large_df, sty, on='Semantic Types', how='inner').reset_index(drop=True)\n",
    "    #Add rsid coulmn con merge\n",
    "    #rspmidmesh_merge = pd.merge(pmidmesh, rsidpmid, on= 'pmids', how='inner').drop_duplicates().reindex(columns=['pmids', 'rsid', 'mesh'])\n",
    "    ref_sty = ref_sty.rename(columns={'Preferred Label_y':'Semantic Types Label','Preferred Label_x':'Preferred Label'})\n",
    "    #ref_sty = ref_sty.drop('Column1',axis=1)\n",
    "    #ref_sty.to_csv('ref-mesh-archive/ref_mesh_sty_large.csv')\n",
    "\n",
    "ref_sty = ref_sty[['Preferred Label', 'Semantic Types Label', 'Class ID', 'Semantic Types', 'Synonyms', 'Parents', 'CUI', 'AQL', 'TERMUI']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(ref_sty['Preferred Label'].nunique(), 'mesh total')\n",
    "ref_sty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if 'ref_sty' in globals():\n",
    "    #modulo groupby and bar\n",
    "    ref_sty_less = ref_sty[['Preferred Label','Semantic Types Label']]\n",
    "\n",
    "    ### groupby.describe analysis by rsid--------------------\n",
    "    ref_sty_less_count = ref_sty_less.groupby('Semantic Types Label').describe().reset_index()\n",
    "    ref_sty_less_count.columns = ref_sty_less_count.columns.to_flat_index()\n",
    "    new_column_names = ['Semantic Types Label', 'mesh-count', 'mesh-unique','mesh-top','mesh-freq']\n",
    "    ref_sty_less_count.columns = new_column_names\n",
    "    ref_sty_less_count_sort = ref_sty_less_count.sort_values(by='mesh-count',ascending=False).reset_index(drop=True)\n",
    "\n",
    "    x = ref_sty_less_count_sort['Semantic Types Label'].iloc[:]\n",
    "    y = ref_sty_less_count_sort['mesh-count'].iloc[:]\n",
    "    plt.figure(figsize=(4, len(ref_sty_less_count_sort)*0.25))\n",
    "    plt.title('Reference Mesh- Semantic Type enrichment', loc='center',pad=10)\n",
    "\n",
    "    plt.barh(x,y)\n",
    "    plt.gca().invert_yaxis()\n",
    "    plt.tick_params(axis='x', which='both', top=True, bottom=False, labeltop=True, labelbottom=False)\n",
    "    #plt.xlabel('pmid count', position=(0.5, 1.08))\n",
    "    plt.ylabel('Semantic Types')\n",
    "    plt.xlabel('mesh', position=(0.5, 1.08))\n",
    "    ax = plt.gca()\n",
    "    ax.xaxis.set_label_position('top')\n",
    "    # use log scale\n",
    "    plt.gca().set_xscale('log')\n",
    "    #plt.savefig('Reference Mesh- Semantic Type enrichment.png',dpi=300, bbox_inches = \"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ref_sty_less_count_sort#['Semantic Types Label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# analizing abundance\n",
    "#count = [12, 21]\n",
    "#remove_value = ref_sty_less_count_sort[ref_sty_less_count_sort['mesh-count'].isin(count)]['Semantic Types Label'].reset_index(drop=True).to_list()\n",
    "remove_value = ref_sty_less_count_sort[ref_sty_less_count_sort['mesh-count']> 56]['Semantic Types Label'].reset_index(drop=True).to_list()\n",
    "ref_sty[ref_sty['Semantic Types Label'].isin(remove_value)]['Semantic Types Label'].drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### STY Ripper form complete ref mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Rip form complete ref mesh\n",
    "save = False\n",
    "rip_list = ['Cell', 'Receptor', 'Hormone', 'Tissue', 'Body Part, Organ, or Organ Component', 'Congenital Abnormality', 'Anatomical Abnormality', 'Indicator, Reagent, or Diagnostic Aid']\n",
    "\n",
    "remove_value = ref_sty_less_count_sort[ref_sty_less_count_sort['Semantic Types Label'].isin(rip_list)]['Semantic Types Label'].reset_index(drop=True).to_list()\n",
    "remove_value = ref_sty_less_count_sort[ref_sty_less_count_sort['mesh-count']> 56]['Semantic Types Label'].reset_index(drop=True).to_list()\n",
    "\n",
    "#new_ref_sty = ref_sty[ref_sty['Semantic Types Label'].isin(remove_value)]\n",
    "mask = ref_sty['Semantic Types Label'].isin(rip_list)\n",
    "new_ref_sty = ref_sty[-mask]\n",
    "\n",
    "if save == True:\n",
    "    new_ref_sty.to_csv('ref-mesh-archive/new_ref_mesh_large_corrected.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "new_ref_sty['Preferred Label'].drop_duplicates().to_csv('ref-mesh-archive/new_ref_mesh_corrected.csv')\n",
    "new_ref_sty['Preferred Label'].drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global Mesh df break through:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Search for all sons from a parent ID (trial)\n",
    "    goal: merge all child mesh together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Search for all brothers from a parent ID:\n",
    "var4 = 'D000066888' #'Diet, Food, and Nutrition'\n",
    "var5 = 'D044623' #'Nutrition Therapy'\n",
    "var6 = 'D014808' #Nutritional and Metabolic Diseases\n",
    "var7 = 'D002318'#Cardiovascular Diseases\n",
    "var8 = 'D004066'#Digestive System Diseases\n",
    "var9 = 'D004700'#Endocrine System Diseases\n",
    "var10 = 'D006967'#Hypersensitivity\n",
    "\n",
    "sub = df[df['Parents'].str.contains(var4).fillna(False)]\n",
    "# use .dropna(inplace=True)   OR   df.fillna(0, inplace=True)\n",
    "sub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Concatenate the results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfgb = pd.DataFrame(columns=df.columns) #Empty DataFrame, same structure\n",
    "#dfgg = pd.DataFrame()\n",
    "dfgg = sub\n",
    "# I MUST search in 'Parents' for the top category ID.\n",
    "\n",
    "#Parents Class ID list:\n",
    "kids = sub['Class ID'].tolist() #search for nested childs\n",
    "\n",
    "# Multiple search:\n",
    "for i in kids:\n",
    "    child = df[df['Parents'].str.contains(i).fillna(False)]\n",
    "    dfgg = dfgg.append([child])\n",
    "    dfgb = pd.concat([child, dfgb.loc[:]])\n",
    "    kids = child['Class ID'].tolist()\n",
    "    for i in kids:\n",
    "        child = df[df['Parents'].str.contains(i).fillna(False)]\n",
    "        dfgg = dfgg.append([child])\n",
    "        #dfgb = pd.concat([child, dfgb.loc[:]])\n",
    "        kids = child['Class ID'].tolist()\n",
    "        for i in kids:\n",
    "            child = df[df['Parents'].str.contains(i).fillna(False)]\n",
    "            dfgg = dfgg.append([child])\n",
    "            #dfgb = pd.concat([child, dfgb.loc[:]])\n",
    "            kids = child['Class ID'].tolist()\n",
    "            for i in kids:\n",
    "                child = df[df['Parents'].str.contains(i).fillna(False)]\n",
    "                dfgg = dfgg.append([child])\n",
    "                #dfgb = pd.concat([child, dfgb.loc[:]])\n",
    "                kids = child['Class ID'].tolist()\n",
    "                for i in kids:\n",
    "                    child = df[df['Parents'].str.contains(i).fillna(False)]\n",
    "                    dfgg = dfgg.append([child])\n",
    "                    #dfgb = pd.concat([child, dfgb.loc[:]])\n",
    "                    kids = child['Class ID'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfgg.loc[:].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chat_mesh_nutri#.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Create new empty dataframe\n",
    "dfg = pd.DataFrame(columns=df.columns)\n",
    "\n",
    "ds1 = df[df['Class ID'].str.contains(var3)]\n",
    "ds2 = df[df['Class ID'].str.contains(var4)]\n",
    "#dfg.append(arr, ignore_index=True)\n",
    "#dfg2 = dfg.append(ds1)\n",
    "dfg2 = pd.concat([ds2, dfg2.loc[:]])\n",
    "dfg2\n",
    "#USE CONCAT!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dd= df[['Preferred Label','MeSH Frequency']]\n",
    "ddf= dd.dropna(subset='Preferred Label')\n",
    "print(ddf.shape)\n",
    "print(dd.shape)\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#plt.scatter(dd['Preferred Label'], dd['MeSH Frequency'])\n",
    "ddf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#df[\"Preferred Label\"].__contains__(\"human\")\n",
    "df2=df[df[\"Preferred Label\"].str.contains('Polymorphism')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df3= df2[['Preferred Label','MeSH Frequency']]\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df2=df[df[\"Preferred Label\"].str.contains('Diabetes')]\n",
    "df3= df2[['Preferred Label','MeSH Frequency']]\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df2=df[df[\"Preferred Label\"].str.contains('Fabry')]\n",
    "df3= df2[['Preferred Label','MeSH Frequency']]\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# links\n",
    " #https://www.nlm.nih.gov/databases/download/data_distrib_main.html\n",
    " #https://www.nlm.nih.gov/databases/download/mesh.html\n",
    " #https://id.nlm.nih.gov/mesh/swagger/ui"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# previous trials"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison of specific MESH list with MESH big data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dn = pd.read_csv('ref-mesh-archive/nbdb1-mesh.csv')\n",
    "\n",
    "print(len(dn['mesh'].drop_duplicates()))\n",
    "dga= df[df['Preferred Label'].isin(dn['mesh'])]\n",
    "\n",
    "#dgb = dn[-df['Preferred Label'].isin(dn['mesh'])].drop_duplicates()\n",
    "\n",
    "#see not contained in all-MESH\n",
    "dgaa= dga['Preferred Label']\n",
    "dgb = dn['mesh'].drop_duplicates()\n",
    "dgg = dgb[dgb.isin(dgaa)==False]\n",
    "dgg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpolation with mesh selected form semantic type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dn = pd.read_csv('ref-mesh-archive/MeshEX-meshsorted.csv',sep=';')\n",
    "print(len(dn['Preferred Label'].drop_duplicates()))\n",
    "dnmask = dn.Column2.isin(['green'])\n",
    "dnfilter = dn[dnmask]\n",
    "dnfilter\n",
    "# moldulo 'Allineator'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'Allineator' module for list value\n",
    "dnallign = []\n",
    "for i in range(len(dnfilter)):\n",
    "    for pmid in dnfilter['Semantic Types'][i]: #dfrspost = mother table\n",
    "        #out = (dfrspost['rsid'][i], pmid)\n",
    "        out = dnfilter['Preferred Label'][i], pmid\n",
    "        dnallign.append(out)\n",
    "print(type(dnallign))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### table explosion Module:\n",
    "\n",
    "data.csv = mesh; 'Semantic Types'; ID\n",
    "Bone Density,\"T201,T081\",D015519\n",
    "Ideal Body Weight,\"T201,T074,T081\",D056865ù\n",
    "\n",
    "split the SEM column by comma and explode the values to create a new row for each SEM value\n",
    "df = df.assign(SEM=df['SEM'].str.split(',')).explode('SEM')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# table explosion Module:\n",
    "\n",
    "# split the SEM column by comma and explode the values to create a new row for each SEM value\n",
    "dnfilter = dnfilter.assign(SEM=dnfilter['Semantic Types'].str.split(',')).explode('SEM')\n",
    "\n",
    "# print the updated table\n",
    "dnfilerstimple = dnfilter[['Preferred Label','Class ID','Semantic Types','SEM']]\n",
    "dnfiltsupersimple = dnfilter[['Preferred Label','SEM']].drop_duplicates()\n",
    "dnfiltsupersimple.to_clipboard()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Modulo analisi groupby:\n",
    "dnfilercount= dnfiltsupersimple.groupby('SEM').describe().reset_index()\n",
    "dnfilercount.columns = dnfilercount.columns.to_flat_index()\n",
    "new_column_names = ['SEM', 'Labes_count', 'Labes_count_unique','Labes_top','Labels_freq']\n",
    "dnfilercount.columns = new_column_names\n",
    "dnfilercountsort = dnfilercount.sort_values(by= 'Labes_count' ,ascending = False)\n",
    "dnfilercountsortsmall= dnfilercountsort[['SEM','Labes_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Add semantic name column #Use Merge\n",
    "semcode = pd.read_csv('ref-mesh-archive/MeshSTY-code.csv',sep=';')\n",
    "new_column_names = ['Label', 'SEM']\n",
    "semcode.columns = new_column_names\n",
    "dnfilercountsortsmall2 = pd.merge(dnfilercountsortsmall, semcode, on='SEM', how='inner')\n",
    "print(len(dnfilercountsortsmall.SEM))\n",
    "dnfilercountsortsmall2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "w = dnfilercountsortsmall2.Labes_count[dnfilercountsortsmall2['Labes_count'] >1]\n",
    "u = dnfilercountsortsmall2.Label[dnfilercountsortsmall2['Labes_count'] >1]\n",
    "\n",
    "#plt.figure(figsize=(15, 12))\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.scatter(u,w)\n",
    "\n",
    "plt.title('Scatter Plot: \"nutritional physiology\" reference mesh plot')\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "plt.show()\n",
    "print(len(dnfilercountsortsmall2.Label[dnfilercountsortsmall2['Labes_count'] >1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dnfilercountsortsmall2[dnfilercountsortsmall2['Labes_count'] >1].head(8)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}