bioexplorer/pythonsdk/notebooks/astrocytes/BioExplorer_import_sonata_to_db.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f992ef9f",
   "metadata": {},
   "source": [
    "# BioExplorer - Astrocytes\n",
    "![](../bioexplorer_neurons_banner.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "345ec844",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from sqlalchemy import create_engine\n",
    "from sqlalchemy.orm import Session\n",
    "from tqdm.notebook import tqdm\n",
    "import libsonata\n",
    "import numpy as np\n",
    "import morphio\n",
    "import glob\n",
    "import h5py\n",
    "\n",
    "db_host = os.getenv('DB_HOST')\n",
    "db_name = os.getenv('DB_NAME')\n",
    "db_user = os.getenv('DB_USER')\n",
    "db_password = os.getenv('DB_PASSWORD')\n",
    "db_schema = 'astrocytes'\n",
    "\n",
    "db_connection_string = 'postgresql+psycopg2://%s:%s@%s:5432/%s' % (db_user, db_password, db_host, db_name)\n",
    "print('Connection string: ' + db_connection_string + ', schema: ' + db_schema)\n",
    "\n",
    "engine = create_engine(db_connection_string)\n",
    "conn = engine.connect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e0f227d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = os.getenv('ASTROCYTES_DATA_FOLDER')\n",
    "print(data_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a834b1e2",
   "metadata": {},
   "source": [
    "## Circuit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eed1274",
   "metadata": {},
   "outputs": [],
   "source": [
    "circuit_config = os.path.join(data_folder, 'build', 'ngv_config.json')\n",
    "circuit = libsonata.CircuitConfig.from_file(circuit_config)\n",
    "population_guid = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8491bda5",
   "metadata": {},
   "outputs": [],
   "source": [
    "json = json.loads(circuit.expanded_json)\n",
    "json.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab3dae92",
   "metadata": {},
   "source": [
    "### Connectivities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5191a3dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "connectivities = json['connectivities']\n",
    "gliovascular = connectivities['gliovascular']\n",
    "endfeet_areas = gliovascular['endfeet_areas']\n",
    "\n",
    "endfeet_areas_filename = os.path.join(data_folder, 'build', endfeet_areas)\n",
    "print(endfeet_areas_filename)\n",
    "endfeets = h5py.File(endfeet_areas_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74dc0d0e",
   "metadata": {},
   "source": [
    "#### End-feet areas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04615b02",
   "metadata": {},
   "outputs": [],
   "source": [
    "gliovascular_filename = os.path.join(data_folder, 'build', 'sonata', 'edges', 'gliovascular.h5')\n",
    "gliovascular = libsonata.EdgeStorage(gliovascular_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7985f9b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "gliovascular_population = gliovascular.open_population('gliovascular')\n",
    "ids = np.arange(0, gliovascular_population.size)\n",
    "astrocyte_section_ids = gliovascular_population.get_attribute('astrocyte_section_id', list(ids))\n",
    "endfoot_ids = gliovascular_population.get_attribute('endfoot_id', list(ids))\n",
    "\n",
    "endfoot_section_ids = dict()\n",
    "for i in range(len(astrocyte_section_ids)):\n",
    "    endfoot_section_ids[endfoot_ids[i]] = int(astrocyte_section_ids[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "116084eb",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "with Session(engine) as session:\n",
    "    session.execute('DELETE FROM %s.end_foot' % db_schema)\n",
    "    session.commit()\n",
    "\n",
    "objects = endfeets['objects']\n",
    "for obj in objects:\n",
    "    guid = int(obj.replace('endfoot_', ''))\n",
    "    vertices = objects[obj]['points']\n",
    "    v = list()\n",
    "    for vertex in vertices:\n",
    "        v.append(vertex[0])\n",
    "        v.append(vertex[1])\n",
    "        v.append(vertex[2])\n",
    "    vertices_buffer = np.array(v, dtype=np.float32).tobytes()\n",
    "    indices = objects[obj]['triangles']\n",
    "    i = list()\n",
    "    for index in indices:\n",
    "        i.append(index[0])\n",
    "        i.append(index[1])\n",
    "        i.append(index[2])\n",
    "    indices_buffer = np.array(i, dtype=np.uint32).tobytes()\n",
    "    astrocyte_guid = gliovascular_population.target_nodes([guid])[0]\n",
    "    astrocyte_section_guid = int(astrocyte_section_ids[guid])\n",
    "\n",
    "    with Session(engine) as session:\n",
    "        session.execute(\n",
    "            'INSERT INTO %s.end_foot VALUES (:guid, :astrocyte_guid, :astrocyte_section_guid, :vertices, :indices)' % db_schema,\n",
    "            {\n",
    "                'guid': guid,\n",
    "                'astrocyte_guid': int(astrocyte_guid),\n",
    "                'astrocyte_section_guid': astrocyte_section_guid,\n",
    "                'vertices': vertices_buffer,\n",
    "                'indices': indices_buffer,\n",
    "            }\n",
    "        )\n",
    "        session.commit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c62ffad",
   "metadata": {},
   "source": [
    "### Components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c5406af",
   "metadata": {},
   "outputs": [],
   "source": [
    "components = json['components']\n",
    "print(components)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5da277e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for component in components:\n",
    "    with Session(engine) as session:\n",
    "        session.execute(\n",
    "            'DELETE FROM %s.configuration WHERE guid=:guid' % db_schema,\n",
    "            {\n",
    "                'guid': component\n",
    "            }\n",
    "        )\n",
    "        session.commit()\n",
    "\n",
    "    print('Populating components')\n",
    "    with Session(engine) as session:\n",
    "        session.execute(\n",
    "            'INSERT INTO %s.configuration VALUES (:guid, :value)' % db_schema,\n",
    "            {\n",
    "                'guid': component,\n",
    "                'value': components[component],\n",
    "            }\n",
    "        )\n",
    "        session.commit()    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09543792",
   "metadata": {},
   "source": [
    "### Nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c226eec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = circuit.node_population('astrocytes')\n",
    "ids = np.arange(0, nodes.size)\n",
    "\n",
    "model_templates = nodes.get_attribute('model_template', list(ids))\n",
    "model_types = nodes.get_attribute('model_type', list(ids))\n",
    "morphologies = nodes.get_attribute('morphology', list(ids))\n",
    "mtypes = nodes.get_attribute('mtype', list(ids))\n",
    "radii = nodes.get_attribute('radius', list(ids))\n",
    "xs = nodes.get_attribute('x', list(ids))\n",
    "ys = nodes.get_attribute('y', list(ids))\n",
    "zs = nodes.get_attribute('z', list(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad3dc306",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nodes\n",
    "print('Deleting existing nodes')\n",
    "with Session(engine) as session:\n",
    "    session.execute(\n",
    "        'DELETE FROM %s.node WHERE population_guid=:population_guid' % db_schema,\n",
    "        {\n",
    "            'population_guid': population_guid\n",
    "        }\n",
    "    )\n",
    "    session.commit()\n",
    "\n",
    "print('Populating nodes')\n",
    "with Session(engine) as session:\n",
    "    for i in tqdm(range(nodes.size)):\n",
    "        session.execute(\n",
    "            'INSERT INTO %s.node VALUES (:guid, :population_guid, :x, :y, :z, :radius, :model_template, :model_type, :morphology, :mtype)' % db_schema,\n",
    "            {\n",
    "                'guid': i,\n",
    "                'population_guid': population_guid,\n",
    "                'x': xs[i],\n",
    "                'y': ys[i],\n",
    "                'z': zs[i],\n",
    "                'radius': radii[i],\n",
    "                'model_template': model_templates[i],\n",
    "                'model_type': model_types[i],\n",
    "                'morphology': morphologies[i],\n",
    "                'mtype': mtypes[i],\n",
    "            }\n",
    "        )\n",
    "        session.commit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fe5b8bf",
   "metadata": {},
   "source": [
    "### Astrocytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "517b4bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "morphology_folder = components['morphologies_dir']\n",
    "print(morphology_folder)\n",
    "\n",
    "morphology_files = glob.glob(os.path.join(morphology_folder, '*.h5'))\n",
    "for morphology_file in morphology_files:\n",
    "    m = morphio.Morphology(morphology_file)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01ad7b11",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = morphio.Morphology(morphology_files[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "719b128a",
   "metadata": {},
   "outputs": [],
   "source": [
    "section_types = dict()\n",
    "section_types[morphio.SectionType.undefined] = 0\n",
    "section_types[morphio.SectionType.soma] = 1\n",
    "section_types[morphio.SectionType.axon] = 2\n",
    "section_types[morphio.SectionType.basal_dendrite] = 3\n",
    "section_types[morphio.SectionType.apical_dendrite] = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf16eed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "with Session(engine) as session:\n",
    "    session.execute('DELETE FROM %s.section' % db_schema)\n",
    "    session.commit()\n",
    "\n",
    "with Session(engine) as session:\n",
    "    for morphology_guid in tqdm(range(len(morphologies))):\n",
    "        morphology=morphologies[morphology_guid]\n",
    "\n",
    "        morphology_file = os.path.join(morphology_folder, morphology + '.h5')\n",
    "        m = morphio.Morphology(morphology_file)\n",
    "\n",
    "        for section in m.sections:\n",
    "            section_type_guid = section_types[section.type]\n",
    "            parent = -1\n",
    "            if not section.is_root:\n",
    "                parent =  section.parent.id\n",
    "\n",
    "            points = list()\n",
    "            for i in range(len(section.points)):\n",
    "                point = section.points[i]\n",
    "                points.append(point[0])\n",
    "                points.append(point[1])\n",
    "                points.append(point[2])\n",
    "                points.append(section.diameters[i])\n",
    "            buffer = np.array(points, dtype=np.float32).tobytes()\n",
    "\n",
    "            session.execute(\n",
    "                'INSERT INTO %s.section VALUES (:morphology_guid, :section_guid, :section_parent_guid, :section_type_guid, :points)' % db_schema,\n",
    "                {\n",
    "                    'morphology_guid': morphology_guid,\n",
    "                    'section_guid': section.id,\n",
    "                    'section_parent_guid': parent,\n",
    "                    'section_type_guid': section_type_guid,\n",
    "                    'points': buffer\n",
    "                }\n",
    "            )\n",
    "        if morphology_guid % 10 == 0:\n",
    "            session.commit()\n",
    "    session.commit()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9575c848",
   "metadata": {},
   "source": [
    "### Micro-domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bbd945f",
   "metadata": {},
   "outputs": [],
   "source": [
    "microdomains_file = json['networks']['nodes'][2]['populations']['astrocytes']['microdomains_file']\n",
    "microdomains_file = os.path.join(data_folder, 'build', microdomains_file)\n",
    "microdomains = h5py.File(microdomains_file)\n",
    "\n",
    "data = microdomains['data']\n",
    "offsets = microdomains['offsets']\n",
    "points = data['points']\n",
    "triangle_indices = data['triangle_data']\n",
    "\n",
    "offset_points = offsets['points']\n",
    "offset_triangle_indices = offsets['triangle_data']\n",
    "\n",
    "astrocyte_guid = 0\n",
    "nb_astrocytes = len(offset_points) \n",
    "\n",
    "with Session(engine) as session:\n",
    "    \n",
    "    session.execute('DELETE FROM %s.micro_domain' % db_schema)\n",
    "    \n",
    "    for i in tqdm(range(nb_astrocytes - 1)):\n",
    "        points_as_floats = list()\n",
    "        triangle_indices_as_ints = list()\n",
    "\n",
    "        point_offset = offset_points[i]\n",
    "        nb_points = offset_points[i + 1] - point_offset\n",
    "        for j in range(point_offset, point_offset + nb_points):\n",
    "            points_as_floats.append(float(points[j][0]))\n",
    "            points_as_floats.append(float(points[j][1]))\n",
    "            points_as_floats.append(float(points[j][2]))\n",
    "\n",
    "        triangle_offset = offset_triangle_indices[i]\n",
    "        nb_triangles = offset_triangle_indices[i + 1] - triangle_offset\n",
    "        for j in range(triangle_offset, triangle_offset + nb_triangles):\n",
    "            # What is the first value of the triangle indices?\n",
    "            triangle_indices_as_ints.append(triangle_indices[j][1])\n",
    "            triangle_indices_as_ints.append(triangle_indices[j][2])\n",
    "            triangle_indices_as_ints.append(triangle_indices[j][3])\n",
    "\n",
    "        vertices_buffer = np.array(points_as_floats, dtype=np.float32).tobytes()\n",
    "        indices_buffer = np.array(triangle_indices_as_ints, dtype=np.uint32).tobytes()\n",
    "    \n",
    "        session.execute(\n",
    "            'INSERT INTO %s.micro_domain VALUES (:guid, :vertices, :indices)' % db_schema,\n",
    "            {\n",
    "                'guid': astrocyte_guid,\n",
    "                'vertices': vertices_buffer,\n",
    "                'indices': indices_buffer,\n",
    "            }\n",
    "        )\n",
    "        session.commit()\n",
    "        astrocyte_guid += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}