general_utilities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Py General Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import io\n",
    "import zipfile\n",
    "import requests\n",
    "\n",
    "def simple_bool(message):\n",
    "    choose = input(message+\" (y/n): \").lower()\n",
    "    your_bool = choose in [\"y\", \"yes\",\"yea\",\"sure\"]\n",
    "    return your_bool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# >> Downloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import io\n",
    "import requests\n",
    "import zipfile\n",
    "\n",
    "\n",
    "# git clone\n",
    "def git_clone(repo_url,\n",
    "              save_dir = os.getcwd()):\n",
    "    cmd = f'git clone {repo_url} {save_dir}'\n",
    "    os.system(cmd)\n",
    "    print('done')\n",
    "\n",
    "\n",
    "def git_clone_sub(repo_url,\n",
    "                  subfolder,\n",
    "                  temp_dir = os.getcwd()+'\\\\temp\\\\'):\n",
    "    if not os.path.exists(temp_dir):\n",
    "        os.makedirs(temp_dir)\n",
    "    cmd = f'git clone {repo_url} {temp_dir}'\n",
    "    os.system(cmd)\n",
    "    # specify original path and destination path\n",
    "    original_path = temp_dir + subfolder\n",
    "    destination_path = os.path.join(os.getcwd(), subfolder)\n",
    "    os.rename(original_path, destination_path)\n",
    "    delete = f'rmdir /s /q  {temp_dir}'\n",
    "    os.system(delete)\n",
    "    print('done')\n",
    "\n",
    "\n",
    "# Simple Downloader\n",
    "def get_file(url,\n",
    "             file_name,\n",
    "             dir = os.getcwd()):\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        content = response.content\n",
    "        file_path = os.path.join(dir, file_name)\n",
    "        with open(file_path, 'wb') as file:\n",
    "            file.write(content)\n",
    "        print(f\"File downloaded successfully. Saved as {file_name}\")\n",
    "    else:\n",
    "        print(\"Unable to download the file.\")\n",
    "\n",
    "\n",
    "# Download single GitHub file from repository\n",
    "def get_gitfile(url,\n",
    "                flag='',\n",
    "                dir = os.getcwd()):\n",
    "    url = url.replace('blob','raw')\n",
    "    response = requests.get(url)\n",
    "    file_name = flag + url.rsplit('/',1)[1]\n",
    "    file_path = os.path.join(dir, file_name)\n",
    "    if response.status_code == 200:\n",
    "        with open(file_path, 'wb') as file:\n",
    "            file.write(response.content)\n",
    "        print(f\"File downloaded successfully. Saved as {file_name}\")\n",
    "    else:\n",
    "        print(\"Unable to download the file.\")\n",
    "\n",
    "\n",
    "# Download and Exrtact zip file from Zenodo\n",
    "def get_and_extract_zenodo(file,\n",
    "                           zenodoid = 8205724,\n",
    "                           dir = os.getcwd(),\n",
    "                           ext = '.zip'):\n",
    "    url='https://zenodo.org/record/'+str(zenodoid)+'/files/'+file+'.zip?download=1'\n",
    "    zip_file_name = file+ext\n",
    "    extracted_folder_name = dir\n",
    "    # Download the ZIP file\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        # Extract the ZIP contents\n",
    "        with io.BytesIO(response.content) as zip_buffer:\n",
    "            with zipfile.ZipFile(zip_buffer, 'r') as zip_ref:\n",
    "                zip_ref.extractall(extracted_folder_name)\n",
    "        print(f\"ZIP file '{zip_file_name}' extracted to '{extracted_folder_name}' successfully.\")\n",
    "    else:\n",
    "        print(\"Failed to download the ZIP file.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "git_clone('https://github.com/johndef64/pychatgpt.git', save_dir = os.getcwd()+'\\\\temp\\\\')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "git_clone_sub('https://github.com/johndef64/pychatgpt.git', \n",
    "              'gpt-cli')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "git_clone_sub('https://github.com/spacetx/starfish.git','starfish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "git_clone('https://github.com/johndef64/pychatgpt.git', r'C:\\Users\\giova\\Documents\\GitHub\\pyutilities_datascience\\newrepo')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download Datasets.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-15T09:37:02.578581600Z",
     "start_time": "2023-11-15T09:36:56.922915100Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File downloaded successfully. Saved as base_pokemon.csv\n",
      "File downloaded successfully. Saved as go_pokemon.csv\n",
      "File downloaded successfully. Saved as H_GENES_proteincoding_genes.csv\n"
     ]
    }
   ],
   "source": [
    "# Download the file\n",
    "handle = \"https://github.com/\"\n",
    "file1 = handle+\"SeniorMars/pokemon-csv/blob/master/pokemon.csv\"\n",
    "file2 = handle+\"zehnzwanzig/PokemonGo_CSV/blob/master/pokemon.csv\"\n",
    "file3 = handle+\"johndef64/GRPM_system/blob/main/human_genes_repo/H_GENES_proteincoding_genes.csv\"\n",
    "\n",
    "get_gitfile(file1, 'base_')\n",
    "get_gitfile(file2, 'go_'  )\n",
    "get_gitfile(file3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# >> File Operartions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "# Check files in folder (with extension)\n",
    "def file_display(ext,\n",
    "                 contains='',\n",
    "                 path=os.getcwd()):\n",
    "    file_pattern = os.path.join(path, \"*.\"+ext)\n",
    "    files = glob.glob(file_pattern)\n",
    "    files_name = []\n",
    "    for file in files:\n",
    "        file_name = os.path.basename(file)\n",
    "        files_name.append(file_name)\n",
    "\n",
    "    print('Available .'+ext+' files:')\n",
    "    files_df = pd.Series(files_name)\n",
    "    file = files_df[files_df.str.contains(contains)]\n",
    "    print(file)\n",
    "\n",
    "\n",
    "def file_display_subfolders(folder_path=os.getcwd()):\n",
    "    subfolders = [f.name for f in os.scandir(folder_path) if f.is_dir()]\n",
    "    print(\"Subfolders in\", folder_path, \":\")\n",
    "    for subfolder in subfolders:\n",
    "        print(subfolder)\n",
    "\n",
    "\n",
    "def file_get_subfolders(folder_path=os.getcwd()):\n",
    "    subfolders = [f.name for f in os.scandir(folder_path) if f.is_dir()]\n",
    "    return subfolders\n",
    "\n",
    "\n",
    "def file_get_files(ext,\n",
    "                   contains='',\n",
    "                   path=os.getcwd()):\n",
    "    file_pattern = os.path.join(path, \"*.\"+ext)\n",
    "    files = glob.glob(file_pattern)\n",
    "    files_name = []\n",
    "    for file in files:\n",
    "        file_name = os.path.basename(file)\n",
    "        files_name.append(file_name)\n",
    "    filtered_file = [item for item in files_name if isinstance(item, str) and contains in item]\n",
    "    return filtered_file\n",
    "\n",
    "\n",
    "def file_get_files_pd(ext, contains='',\n",
    "                      path=os.getcwd()):\n",
    "    # Create a file path pattern to match 'ext' files\n",
    "    file_pattern = os.path.join(path, \"*.\"+ext)\n",
    "    # Use glob to get a list of file paths matching the pattern\n",
    "    files = glob.glob(file_pattern)\n",
    "    files_name = []\n",
    "    # Get the list of 'ext' files\n",
    "    for file in files:\n",
    "        file_name = os.path.basename(file)\n",
    "        files_name.append(file_name)\n",
    "    files_sr = pd.Series(files_name)\n",
    "    filtered_file = files_sr[files_sr.str.contains(contains)]\n",
    "    return pd.Series(filtered_file)\n",
    "\n",
    "\n",
    "def file_delete(filename,\n",
    "                path=os.getcwd()):\n",
    "    try:\n",
    "        os.remove(os.path.join(path, filename))\n",
    "        print(f\"File {filename} deleted successfully.\")\n",
    "    except FileNotFoundError:\n",
    "        print(f\"File {filename} not found.\")\n",
    "    except PermissionError:\n",
    "        print(f\"Permission denied.\")\n",
    "    except Exception as e:\n",
    "        print(f\"Unable to delete file {filename}. Error: {str(e)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load & Display CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "path =  r'C:\\Users\\yourpath\\dataset.csv'\n",
    "df = pd.read_csv(path, encoding='utf-8')\n",
    "\n",
    "poke_base = pd.read_csv('base_pokemon.csv')\n",
    "poke_go = pd.read_csv('go_pokemon.csv', encoding='latin-1')\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# >> Dataframe Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import ast\n",
    "\n",
    "\n",
    "def pd_choose(my_list):\n",
    "    i = int(input('choose index:\\n'+str(pd.Series(my_list))))\n",
    "    return my_list[i]\n",
    "\n",
    "\n",
    "def pd_choose_col(my_df):\n",
    "    i = int(input('choose column:\\n'+str(pd.Series(my_df.columns))))\n",
    "    return my_df.columns[i]\n",
    "\n",
    "\n",
    "# Df merger\n",
    "def pd_merge_base(df1, df2, column1, column2, how= ''):\n",
    "    merged_df = pd.merge(df1, df2, left_on=column1, right_on=column2, how=how)\n",
    "    return merged_df\n",
    "\n",
    "\n",
    "def pd_merge_select(df1, df2, how= 'inner'):\n",
    "    column1 = df1.columns[int(input(pd.Series(df1.columns)))]\n",
    "    column2 = df2.columns[int(input(pd.Series(df2.columns)))]\n",
    "    merged_df = pd.merge(df1, df2, left_on=column1, right_on=column2, how=how)\n",
    "    return merged_df\n",
    "\n",
    "\n",
    "def pd_merge_select_multi(df1, df2):\n",
    "    how = pd_choose(['inner', 'outer','left', 'right','cross' ])\n",
    "    column1 = df1.columns[int(input(pd.Series(df1.columns)))]\n",
    "    column2 = df2.columns[int(input(pd.Series(df2.columns)))]\n",
    "    merged_df = pd.merge(df1, df2, left_on=column1, right_on=column2, how=how)\n",
    "    return merged_df\n",
    "\n",
    "\n",
    "def pd_groupby_describe(df1):\n",
    "    var = ['all','number','object','bool']\n",
    "    include = pd_choose(var)\n",
    "    column1 = df1.columns[int(input(pd.Series(df1.columns)))]\n",
    "    df1_count = df1.groupby(column1).describe(include=include).dropna(axis=1,how='all').reset_index()\n",
    "    return df1_count\n",
    "\n",
    "\n",
    "def pd_groupby_describe_flat(df1):\n",
    "    include = pd_choose(['all','number','object','bool'])\n",
    "    column1 = df1.columns[int(input(pd.Series(df1.columns)))]\n",
    "    df1_count = df1.groupby(column1).describe(include=include).dropna(axis=1,how='all').reset_index()\n",
    "    df1_count.columns = df1_count.columns.to_flat_index()\n",
    "    #pattern = r\"([\\w]+)_([\\w]+)\"\n",
    "    list_of_strings = []\n",
    "    for tuple in df1_count.columns:\n",
    "        string = \"_\".join(tuple)\n",
    "        #string = re.sub(r\"\\s+\", \"\", string)\n",
    "        list_of_strings.append(string)\n",
    "    df1_count.columns = list_of_strings\n",
    "    return df1_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## simple operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create random df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "data = {'A': [], 'B': [], 'C': []}\n",
    "my_list = poke_go.NAME_ENGLISH\n",
    "\n",
    "for _ in range(20):  # This will create a dataframe with 5 rows\n",
    "    data['A'].append(random.randint(1, 10))\n",
    "    data['B'].append(random.uniform(0.0, 1.0))\n",
    "    data['C'].append(random.choice(my_list))\n",
    "\n",
    "simple_df = pd.DataFrame(data)\n",
    "simple_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "simple_df.columns[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# nuova colonna con aggiunta\n",
    "import math\n",
    "import random\n",
    "random = random.randint(1, 10)\n",
    "\n",
    "simple_df['new_col'] = [(int(i))*2.2 for i in range(len(simple_df))]\n",
    "simple_df['new_col_2'] = simple_df[simple_df.columns[0]].apply(lambda x: math.sqrt(x))\n",
    "simple_df['new_col_3'] = simple_df[simple_df.columns[1]].apply(lambda x: x*random)\n",
    "simple_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = poke_go\n",
    "df['Type'] = df['TYP1'] + ','+ df['TYP2']\n",
    "poke_go =df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add boolean column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#method: str contains\n",
    "df = pd.read_csv('df.csv', index_col=False)\n",
    "\n",
    "hook = ['linmor', 'vonn', 'janus','deast','gio','papa','gius','iavn','dean','def','giul','lorem','crest','hor','vita','van','gian','imtd']\n",
    "df['bool'] = df['core'].str.contains('|'.join(hook))\n",
    "df[['core','bool']]\n",
    "\n",
    "#df = df.drop('index', axis=1)\n",
    "if simple_bool('save csc?'):\n",
    "    df.to_csv('filtered_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('filtered_df.csv', index_col=False)\n",
    "column_names = df.columns.tolist()\n",
    "\n",
    "# Move the last column to the beginning of the list\n",
    "column_names = [column_names[-1]] + column_names[:-1]\n",
    "# Reorder the columns in the DataFrame using the updated column names\n",
    "df = df[column_names]\n",
    "\n",
    "if simple_bool('save csv?'):\n",
    "    df.to_csv('filtered_df.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### apply style to df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('filtered_df.csv', index_col=False)\n",
    "#df = df.drop('Unnamed: 0', axis=1)\n",
    "\n",
    "def highlight_true(value):\n",
    "    \"\"\"\n",
    "    Apply background color to cells containing True.\n",
    "    \"\"\"\n",
    "    if value is True:\n",
    "        return 'background-color: yellow'\n",
    "    else:\n",
    "        return ''\n",
    "\n",
    "df_style = df.style.applymap(highlight_true)\n",
    "df_style"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### remove bool column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#remove bool column\n",
    "df = df[df['bool']==True]\n",
    "df = df.drop('bool', axis=1)\n",
    "\n",
    "if simple_bool('save csc?'):\n",
    "    df.to_csv('log64_bool.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### misc op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_list = ['all','number','object','bool']\n",
    "pd_choose(my_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = {\n",
    "    'name': ['John', 'Alice', 'Bob', 'Alice'],\n",
    "    'age': [25, 30, 35, 27],\n",
    "    'salary': [5000, 6000, 5500, 7000],\n",
    "    'city': ['New York', 'Chicago', 'New York', 'Chicago']\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "df.select_dtypes(include='object').columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "simple_df = pd_merge_select(poke_go, poke_base, 'inner')[['NAME_ENGLISH','NAME_FRENCH','Ability1']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd_merge_select_multi(poke_go[['NAME_ENGLISH','NAME_FRENCH']], poke_base[['Name','Ability1']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Groupby Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# groupby describe\n",
    "group_0 = pd_groupby_describe(poke_go)\n",
    "group_0[group_0.TYP1.count]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# groupby describe flat\n",
    "df = poke_go\n",
    "df['Type'] = df['TYP1'] + ','+ df['TYP2']\n",
    "poke_go =df\n",
    "\n",
    "group_1 = pd_groupby_describe_flat(poke_go, 'number')#.T#, 'str')\n",
    "group_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "type(gr.columns[0])\n",
    "group_1.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Create a sample DataFrame\n",
    "data = {\n",
    "    'name': ['John', 'Alice', 'Bob', 'Alice'],\n",
    "    'age': [25, 30, 35, 27],\n",
    "    'salary': [5000, 6000, 5500, 7000],\n",
    "    'city': ['New York', 'Chicago', 'New York', 'Chicago']\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Group by multiple columns\n",
    "grouped = df.groupby(['city', 'name'])\n",
    "\n",
    "# Example usage\n",
    "print(grouped.size())  # Grouped size based on 'city' and 'name'\n",
    "print(grouped.sum()['salary'])  # Grouped sum of 'salary' based on 'city' and 'name'\n",
    "grouped.count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# >> Data Visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While Pandas provides some basic plotting functionality, you may still need to use Matplotlib for more advanced and customized plots. Here are a few reasons why you might consider using Matplotlib alongside Pandas for plotting:\n",
    "\n",
    "1. **Flexibility**: Matplotlib is a powerful visualization library that offers a wide range of plot types and customization options. It provides fine-grained control over plot elements such as labels, titles, color schemes, and annotations. If you need to create complex or specialized plots that go beyond the capabilities of Pandas, Matplotlib can be a valuable tool.\n",
    "\n",
    "2. **Additional Plot Types**: While Pandas offers several basic plot types (e.g., line, bar, scatter), Matplotlib provides a larger variety of plot types such as histograms, pie charts, box plots, heatmaps, and 3D plots. If you need to create these types of plots, Matplotlib is a great choice.\n",
    "\n",
    "3. **Integration with Pandas**: Matplotlib integrates well with Pandas, allowing you to directly plot Pandas DataFrames and Series objects. You can use the `plot()` function from Pandas to quickly create basic plots, and then use Matplotlib to customize them further if needed. Matplotlib provides fine-grained control over plot elements, allowing you to tweak the plots created using Pandas.\n",
    "\n",
    "4. **Publication-Quality Plots**: Matplotlib is widely used in scientific research and data analysis because it offers a high level of customization and can create publication-quality plots. If you need to create professional-looking plots for papers, reports, or presentations, Matplotlib provides the necessary tools and options."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create a sample DataFrame\n",
    "data = {\n",
    "    'name': ['John', 'Alice', 'Bob', 'Alice'],\n",
    "    'age': [25, 30, 35, 27],\n",
    "    'salary': [5000, 6000, 5500, 7000],\n",
    "    'city': ['New York', 'Chicago', 'New York', 'Chicago']\n",
    "}\n",
    "df = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#---------------------------------------------------------\n",
    "df = poke_go\n",
    "grouped_data = df.groupby(pd_choose_col(df)).describe()\n",
    "#grouped_data = pd_groupby_describe(df)\n",
    "grouped_data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grouped_data['MAX_ATT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Visualize string columns\n",
    "#include = pd_choose(['number','object','bool','category','datetime'])\n",
    "#string_columns = grouped_data.select_dtypes(include=include).columns\n",
    "\n",
    "column = 'MAX_ATT'\n",
    "grouped_data[column].boxplot()\n",
    "#grouped_data[column].plot(kind='bar')\n",
    "plt.title(f\"{column} distribution by chosen col\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Visualize string columns\n",
    "include = pd_choose(['str','int','float','number','object','bool','category','datetime'])\n",
    "string_columns = df.select_dtypes(include=[include]).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "string_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "column = pd_choose_col(df)\n",
    "fig = plt.figure(figsize = (20,5))\n",
    "ax = fig.add_subplot()\n",
    "grouped_data[column].plot(kind='bar', ax=ax)\n",
    "ax.set_title(f\"{column} distribution by chosen col\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "column = pd_choose_col(df)\n",
    "fig = plt.figure(figsize=(20, 5))\n",
    "ax = fig.add_subplot()\n",
    "\n",
    "ax.set_title(f\"{column} distribution by chosen col\")\n",
    "grouped_data[column].plot(kind='bar', ax=ax)\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "column = pd_choose_col(df)\n",
    "\n",
    "#grouped_data[column].boxplot()\n",
    "grouped_data[column].plot(kind='bar')\n",
    "plt.title(f\"{column} distribution by chosen col\")\n",
    "plt.show()\n",
    "\n",
    "grouped_data[column]['mean'].plot(kind='bar')\n",
    "plt.title(f\"{column} distribution by chosen col\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "column = pd_choose_col(df)\n",
    "grouped_data[column].plot(kind='bar')\n",
    "plt.title(f\"{column} distribution by chosen col\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if simple_bool('loop plot?'):\n",
    "    for column in string_columns:\n",
    "        if column in grouped_data.columns:\n",
    "            #grouped_data[column].plot(kind='bar')\n",
    "            grouped_data[column].boxplot()\n",
    "            plt.title(f\"{column} distribution by chosen col\")\n",
    "            plt.show()\n",
    "        else:\n",
    "            print(f\"Column '{column}' does not exist in the DataFrame.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "# Visualize integer columns\n",
    "int_columns = df.select_dtypes(include=['int', 'float']).columns\n",
    "int_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if simple_bool('loop plot?'):\n",
    "    for column in int_columns:\n",
    "        if column in grouped_data.columns:\n",
    "            grouped_data[column].boxplot()\n",
    "            plt.title(f\"{column} distribution by chosen col\")\n",
    "            plt.show()\n",
    "        else:\n",
    "            print(f\"Column '{column}' does not exist in the DataFrame.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "poke_go"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#import matplotlib.pyplot as plt\n",
    "\n",
    "# Create a sample DataFrame\n",
    "data = {\n",
    "    'name': ['John', 'Alice', 'Bob', 'Alice'],\n",
    "    'age': [25, 30, 35, 27],\n",
    "    'salary': [5000, 6000, 5500, 7000],\n",
    "    'city': ['New York', 'Chicago', 'New York', 'Chicago']\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "# Perform groupby and describe\n",
    "grouped_data = df.groupby(['name']).describe(include=['object'])\n",
    "\n",
    "# Visualize string columns\n",
    "string_columns = df.select_dtypes(include=['object']).columns\n",
    "for column in string_columns:\n",
    "    if column in grouped_data.columns:\n",
    "        grouped_data[column].plot(kind='bar')\n",
    "        plt.title(f\"{column} distribution by city\")\n",
    "        plt.show()\n",
    "    else:\n",
    "        print(f\"Column '{column}' does not exist in the DataFrame.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grouped_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(data)\n",
    "# Perform groupby and describe\n",
    "grouped_data = df.groupby(['name']).describe(include=['number'])\n",
    "# Visualize integer columns\n",
    "int_columns = df.select_dtypes(include=['int', 'float']).columns\n",
    "for column in int_columns:\n",
    "    if column in grouped_data.columns:\n",
    "        grouped_data[column].boxplot()\n",
    "        plt.title(f\"{column} distribution by city\")\n",
    "        plt.show()\n",
    "    else:\n",
    "        print(f\"Column '{column}' does not exist in the DataFrame.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# subdivision of a multiindex object\n",
    "doit = grouped_data['age']\n",
    "print(type(doit))\n",
    "print(type(grouped_data))\n",
    "doit.boxplot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "§grouped_data['age']['count'].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "sns.kdeplot(data=gr, x=gr.columns[1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## other operations\n",
    "\n",
    "Ci sono molte operazioni che puoi eseguire sui DataFrame di Pandas. Ecco alcuni esempi:\n",
    "\n",
    "1. **Selezione dei dati**: Puoi selezionare dati specifici utilizzando il nome della colonna o condizioni specifiche.\n",
    "\n",
    "   ```python\n",
    "   df['colonna']  # seleziona una colonna\n",
    "   df[df['colonna'] > 0]  # seleziona righe dove 'colonna' è maggiore di 0\n",
    "   ```\n",
    "\n",
    "2. **Manipolazione dei dati**: Puoi modificare i tuoi dati in molti modi, come ad esempio aggiungere nuove colonne, modificare valori esistenti, ecc.\n",
    "\n",
    "   ```python\n",
    "   df['nuova_colonna'] = df['colonna1'] + df['colonna2']  # aggiunge una nuova colonna\n",
    "   df['colonna'] = df['colonna'].apply(lambda x: x*2)  # modifica i valori in 'colonna'\n",
    "   ```\n",
    "\n",
    "3. **Ordinamento**: Puoi ordinare i tuoi dati in base ai valori di una o più colonne.\n",
    "\n",
    "   ```python\n",
    "   df.sort_values(by='colonna')  # ordina in base a 'colonna'\n",
    "   ```\n",
    "\n",
    "4. **Grouping**: Puoi raggruppare i tuoi dati in base ai valori di una o più colonne e calcolare statistiche aggregate.\n",
    "\n",
    "   ```python\n",
    "   df.groupby('colonna').mean()  # calcola la media per ogni gruppo in 'colonna'\n",
    "   ```\n",
    "\n",
    "5. **Pivot**: Puoi pivotare i tuoi dati per creare una tabella pivot.\n",
    "\n",
    "   ```python\n",
    "   df.pivot_table(values='colonna1', index='colonna2', columns='colonna3')\n",
    "   ```\n",
    "\n",
    "6. **Join**: Oltre al merge, puoi anche unire DataFrame utilizzando `join`.\n",
    "\n",
    "   ```python\n",
    "   df1.join(df2, on='colonna_comune')\n",
    "   ```\n",
    "\n",
    "7. **Reshaping**: Puoi modificare la forma del tuo DataFrame utilizzando operazioni come `melt`, `pivot`, ecc.\n",
    "\n",
    "8. **Handling Missing Values**: Puoi gestire i valori mancanti utilizzando metodi come `dropna`, `fillna`, ecc.\n",
    "\n",
    "Questi sono solo alcuni esempi delle operazioni che puoi eseguire sui DataFrame di Pandas. Pandas è una libreria molto potente e flessibile che offre molte altre funzionalità."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get pychatgpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "if 'google.colab' in sys.modules:\n",
    "    os.chdir('/content/pychatgpt') #google colab\n",
    "\n",
    "if simple_bool('Do you have an openai API-key?'):\n",
    "    # Get pychatgpt\n",
    "    url=\"https://raw.githubusercontent.com/johndef64/pychatgpt/main/pychatgpt.py\"\n",
    "    get_gitfile(url)\n",
    "\n",
    "    import pychatgpt as op\n",
    "    model = 'gpt-3.5-turbo-16k'\n",
    "    # Example usage\n",
    "    message = \"Describe the Cosmic Holographic Principle\"\n",
    "    response = op.send_message(message, model='gpt-4')\n",
    "\n",
    "else:\n",
    "    print('get your api-key at https://platform.openai.com/account/api-keys\\n'\n",
    "          'or simply use web playground at https://platform.openai.com/playground?model=gpt-3.5-turbo-16k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "op.chat_gpt=[]\n",
    "character = 'Friedrich Nietzsche'\n",
    "m = '''\n",
    "Please tell me about your thoughts about society.\n",
    "'''\n",
    "op.send_message(m,\n",
    "                persona=character,\n",
    "                model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "m = '''\n",
    "Please, tell me more.\n",
    "'''\n",
    "op.send_message(m,\n",
    "                persona=character,\n",
    "                model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "op.save_chat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "op.load_chat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cleatchat\n",
    "m = 'clearchat'\n",
    "op.send_message(m)\n",
    "print(op.chat_gpt)# get & import pychatgpt (openai based module)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# open with notepad, subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "def open_in_notepadpp(file_path):\n",
    "    notepadpp_path = r\"C:\\Program Files\\Notepad++\\notepad++.exe\"  # Path to Notepad++ executable\n",
    "    subprocess.Popen([notepadpp_path, file_path])\n",
    "\n",
    "# Usage\n",
    "file_path = r\"chat_log.txt\"  # Replace with the actual file path\n",
    "open_in_notepadpp(file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prompt for keyword-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define Biomedical topics\n",
    "nutritional_topic = [['diseases and disorders realted to nutrition and diet ', 'diet, food consuption, eating behaviour and nutrition']]\n",
    "infective_topic = [['infective agents, bacteria, virus and protozoan','infective diseases']]\n",
    "reproductive_topic = [['reproductive system physiology','reproductive system pathology', 'Assisted reproductive technology']]\n",
    "female_infertility_topic = [['female infertility, genetic imprinting and maternal effect']]\n",
    "special_issue = [['Diagnosis and Therapies for Genetic Diseases']]\n",
    "\n",
    "nutritional_topics = [\n",
    "    ['Obesity, overweight and body weight control', 'compulsive eating behavior'],\n",
    "    ['cardiovascular diseases','physiological processes realted to cardiovascular diseases','lipid metabolism in the context of cardiovascular diseases'],\n",
    "    ['Diabetes Melitus Type II and metabolic syndrome'],\n",
    "    ['Vitamin metabolism and Vitamins recommended intake levels','Micronutrients metabolism and Micronutrient recommended intake levels', 'disease related to vitamins and micronutrients deficiency'],\n",
    "    ['eating behaviour and taste sensation'],\n",
    "    ['food intolerances'],\n",
    "    ['food allergies'],\n",
    "    ['diet-induced oxidative stress'],\n",
    "    ['metabolism of xenobiotics'],\n",
    "]\n",
    "chosen_topic = special_issue\n",
    "pd.Series(chosen_topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# GPT prompts\n",
    "\n",
    "# parameters--------------------------------------\n",
    "object1 = 'Pubmed MeSH terms'\n",
    "object2 = 'Pubmed keywords'\n",
    "\n",
    "object= object1\n",
    "num_mesh = 100\n",
    "topics = chosen_topic\n",
    "topic_id = 0\n",
    "#-----------------------------------------------\n",
    "topic_01  = topics[topic_id][0]\n",
    "topic_02  = topics[topic_id][1] if len(topics[topic_id])>=2 else None\n",
    "topic_03  = topics[topic_id][2] if len(topics[topic_id])>=3 else None\n",
    "\n",
    "format = {'list': \". Create a python list format like this:\\n gpt_01 = [\\\"term1\\\",\\n \\\"term2\\\",\\n \\\"term3\\\",...]\",\n",
    "          'csv':  \". Create a CSV file like this:\\n gpt_terms,\\n \\\"term1\\\",\\n \\\"term2\\\",\\n \\\"term3\\\", ...\"}\n",
    "format = format['csv']\n",
    "\n",
    "prompt_01 = \"give me a comprehensive list of \"+str(num_mesh)+\" real \"+object+\" terms related to \"+ topic_01+format+\"\\n\"\n",
    "prompt_02 = \"give me a comprehensive list of \"+str(num_mesh)+\" real \"+object+\" terms related to \"+ topic_02 +format+\"\\n\" if len(topics[topic_id])>=2 else None\n",
    "prompt_03 = \"give me a comprehensive list of \"+str(num_mesh)+\" real \"+object+\" terms related to \"+ topic_03 +format+\"\\n\" if len(topics[topic_id])>=3 else None\n",
    "\n",
    "prompts = [prompt_01, prompt_02, prompt_03]\n",
    "# If you do not have an openai API key, paste these prompts at https://platform.openai.com/playground?model=gpt-3.5-turbo-16k\n",
    "\n",
    "import pyperclip\n",
    "pyperclip.copy(prompt_01)\n",
    "pyperclip.copy(prompt_01+prompt_02) if len(topics[topic_id])>=2 else None\n",
    "pyperclip.copy(prompt_01+prompt_02+prompt_03) if len(topics[topic_id])>=3 else None\n",
    "\n",
    "print('prompt_01:',prompt_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get GPT-terms\n",
    "import pychatgpt as op\n",
    "op.chat_gpt =[]\n",
    "response = op.send_message_gpt(prompt_01, model='gpt-4', maxtoken=2000)\n",
    "#response = op.send_message_gpt('clearchat')\n",
    "\n",
    "print('''\\n\\nGet the GPT terms from 'chat_log.txt'\n",
    "=> save them manually in csv format in \"ref-mesh-archive/gpt_terms/yourterms.csv\"''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "file_path = 'special_issue_2.csv'\n",
    "content = '''gpt_terms\n",
    "\"Genetic Diseases Diagnosis\",\n",
    "\"Genetic Testing\",\n",
    "\"Molecular Diagnostics\",\n",
    "\"Genetic Screenings\",\n",
    "\"DNA Sequencing\",\n",
    "\"Genome Mapping\",\n",
    "\"Chromosomal Abnormalities\",\n",
    "\"Prenatal Diagnosis\",\n",
    "\"Newborn Screening\",\n",
    "\"Personalized Medicine\",\n",
    "\"Genetic Counseling\",\n",
    "\"Carrier Testing\",\n",
    "\"Genomic Medicine\",\n",
    "\"Pharmacogenetics\",\n",
    "\"Predictive Testing\",\n",
    "\"Presymptomatic Testing\",\n",
    "\"Biochemical Testing\",\n",
    "\"Genetic Therapies\",\n",
    "\"Gene Therapy\",\n",
    "\"Gene Editing\",\n",
    "\"CRISPR-Cas9\",\n",
    "\"Stem Cell Therapy\",\n",
    "\"RNA Therapy\",\n",
    "\"Genetic Surgery\",\n",
    "\"Molecular Therapy\",\n",
    "\"Enzyme Replacement Therapy\",\n",
    "\"Antisense Therapy\",\n",
    "\"Gene Silencing\",\n",
    "\"Genetic Vaccine\",\n",
    "\"Pharmacological Chaperones\"\n",
    "'''\n",
    "def write_to_file(file_path, content):\n",
    "    with open(file_path, 'w') as file:\n",
    "        file.write(content)\n",
    "\n",
    "write_to_file(file_path, content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "op.send_message_gpt('how to say, if a apackahger is not instaled, intall it?')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Zenodo Nbib Dataset (full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import io\n",
    "import sys\n",
    "import glob\n",
    "import zipfile\n",
    "import requests\n",
    "from datetime import datetime\n",
    "\n",
    "def simple_bool(message):\n",
    "    choose = input(message+\" (y/n): \").lower()\n",
    "    your_bool = choose in [\"y\", \"yes\",\"yea\",\"sure\"]\n",
    "    return your_bool\n",
    "\n",
    "def get_and_extract_zenodo(file, dir = os.getcwd(), ext = '.zip'):\n",
    "    url='https://zenodo.org/record/8205724/files/'+file+'.zip?download=1'\n",
    "    zip_file_name = file+ext\n",
    "    extracted_folder_name = dir\n",
    "    # Download the ZIP file\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        # Extract the ZIP contents\n",
    "        with io.BytesIO(response.content) as zip_buffer:\n",
    "            with zipfile.ZipFile(zip_buffer, 'r') as zip_ref:\n",
    "                zip_ref.extractall(extracted_folder_name)\n",
    "        print(f\"ZIP file '{zip_file_name}' extracted to '{extracted_folder_name}' successfully.\")\n",
    "    else:\n",
    "        print(\"Failed to download the ZIP file.\")\n",
    "\n",
    "if simple_bool('Download nbib-data from Zenodo?\\n (careful! 5GB unpacked)'):\n",
    "    timea = datetime.now()\n",
    "    get_and_extract_zenodo('nbib_data')\n",
    "    print('Download and extraction time ',datetime.now()-timea)\n",
    "\n",
    "#set source dataset:-----------------------\n",
    "db_tag = 'pcg'\n",
    "db_name = 'grpm_db_' + db_tag\n",
    "db_path = 'grpm_dataset/'+db_name\n",
    "\n",
    "time1 = datetime.now()\n",
    "#import gene-fullnbib\n",
    "dummy_nbib = pd.read_csv(db_path+'/complete_nbibtable.csv', index_col=0)\n",
    "dummy_nbib['pubmed_id'] = dummy_nbib['pubmed_id'].astype(str)\n",
    "time2 = datetime.now()\n",
    "print('time import nbib: ', time2-time1)\n",
    "print(dummy_nbib.memory_usage().sum() / 1024 / 1024, 'MB')\n",
    "\n",
    "display(dummy_nbib)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R in Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can run R code in a Jupyter notebook. You need to install the rpy2 package and load the R extension in Jupyter using the following steps:\n",
    "\n",
    "1. Install the rpy2 package using pip (Python's package installer). In a command line window run:\n",
    "   ```bash\n",
    "   pip install rpy2\n",
    "   ```\n",
    "\n",
    "2. Then, in your Jupyter notebook, load the rpy2 extension by adding this to a cell and running it:\n",
    "   ```python\n",
    "   %load_ext rpy2.ipython\n",
    "   ```\n",
    "\n",
    "3. Now you can use R in any cell by starting the cell with `%%R`. For example:\n",
    "   ```python\n",
    "   %%R\n",
    "   x <- seq(0, 2*pi, length.out=50)\n",
    "   y <- sin(x)\n",
    "   plot(x, y, main=\"y = sin(x)\")\n",
    "   ```\n",
    "\n",
    "Remember, the rpy2 package requires that R is installed on your machine and it relies on dynamic libraries that R uses, so be sure you have installed R and it is findable by rpy2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!pip install rpy2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext rpy2.ipython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%R\n",
    "x <- seq(0, 2*pi, length.out=50)\n",
    "y <- sin(x)\n",
    "plot(x, y, main=\"y = sin(x)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Repo Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# quick open in colab\n",
    "\n",
    "handle = \"https://colab.research.google.com/github/\"\n",
    "path = \"johndef64/pychatgpt/blob/main/pychatgpt_trial.ipynb\"\n",
    "path1 = \"spacetx/starfish/blob/master/notebooks/DARTFISH.ipynb\"\n",
    "import pyperclip\n",
    "pyperclip.copy(handle+path1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}