nested_dtypes.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "id": "94cbd156",
   "metadata": {},
   "source": [
    "---\n",
    "title: Nested dtypes\n",
    "execute:\n",
    "  eval: true\n",
    "  warning: true\n",
    "  error: true\n",
    "  keep-ipynb: true\n",
    "  cache: true\n",
    "pdf-engine: lualatex\n",
    "html:\n",
    "  code-tools: true\n",
    "  fold-code: false\n",
    "  author: Jonathan D. Rosenblatt\n",
    "  data: 04-20-2024\n",
    "  toc: true\n",
    "  number-sections: true\n",
    "  number-depth: 3\n",
    "  embed-resources: true\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b654f3c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| echo: false\n",
    "# %pip install --upgrade pip\n",
    "# %pip install --upgrade polars\n",
    "# %pip install --upgrade pyarrow\n",
    "# %pip install --upgrade Pandas\n",
    "# %pip install --upgrade plotly\n",
    "# %pip freeze > requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "setup-env",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: setup-env\n",
    "\n",
    "# %pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "Polars-version",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: polars\r\n",
      "Version: 0.20.27\r\n",
      "Summary: Blazingly fast DataFrame library\r\n",
      "Home-page: \r\n",
      "Author: \r\n",
      "Author-email: Ritchie Vink <ritchie46@gmail.com>\r\n",
      "License: \r\n",
      "Location: /Users/johnros/workspace/polars_demo/.venv/lib/python3.11/site-packages\r\n",
      "Requires: \r\n",
      "Required-by: \r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#| label: Polars-version\n",
    "%pip show Polars # check you Polars version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "Pandas-version",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: pandas\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Version: 2.2.2\r\n",
      "Summary: Powerful data structures for data analysis, time series, and statistics\r\n",
      "Home-page: https://pandas.pydata.org\r\n",
      "Author: \r\n",
      "Author-email: The Pandas Development Team <pandas-dev@python.org>\r\n",
      "License: BSD 3-Clause License\r\n",
      "\r\n",
      "Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team\r\n",
      "All rights reserved.\r\n",
      "\r\n",
      "Copyright (c) 2011-2023, Open source contributors.\r\n",
      "\r\n",
      "Redistribution and use in source and binary forms, with or without\r\n",
      "modification, are permitted provided that the following conditions are met:\r\n",
      "\r\n",
      "* Redistributions of source code must retain the above copyright notice, this\r\n",
      "  list of conditions and the following disclaimer.\r\n",
      "\r\n",
      "* Redistributions in binary form must reproduce the above copyright notice,\r\n",
      "  this list of conditions and the following disclaimer in the documentation\r\n",
      "  and/or other materials provided with the distribution.\r\n",
      "\r\n",
      "* Neither the name of the copyright holder nor the names of its\r\n",
      "  contributors may be used to endorse or promote products derived from\r\n",
      "  this software without specific prior written permission.\r\n",
      "\r\n",
      "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\r\n",
      "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r\n",
      "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r\n",
      "DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\r\n",
      "FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r\n",
      "DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r\n",
      "SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r\n",
      "CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r\n",
      "OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\n",
      "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n",
      "Location: /Users/johnros/workspace/polars_demo/.venv/lib/python3.11/site-packages\r\n",
      "Requires: numpy, python-dateutil, pytz, tzdata\r\n",
      "Required-by: \r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#| label: Pandas-version\n",
    "%pip show Pandas # check you Pandas version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "preliminaries",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: preliminaries\n",
    "\n",
    "import polars as pl\n",
    "pl.Config(fmt_str_lengths=50)\n",
    "import polars.selectors as cs\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pyarrow as pa\n",
    "import plotly.express as px\n",
    "import string\n",
    "import random\n",
    "import os\n",
    "import sys\n",
    "%matplotlib inline \n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "\n",
    "# Following two lines only required to view plotly when rendering from VScode. \n",
    "import plotly.io as pio\n",
    "# pio.renderers.default = \"plotly_mimetype+notebook_connected+notebook\"\n",
    "pio.renderers.default = \"plotly_mimetype+notebook\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0adf6db",
   "metadata": {},
   "source": [
    "What Polars module and dependencies are installed?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "show-versions",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------Version info---------\n",
      "Polars:               0.20.27\n",
      "Index type:           UInt32\n",
      "Platform:             macOS-14.4.1-arm64-arm-64bit\n",
      "Python:               3.11.6 (main, Oct  2 2023, 13:45:54) [Clang 15.0.0 (clang-1500.0.40.1)]\n",
      "\n",
      "----Optional dependencies----\n",
      "adbc_driver_manager:  <not installed>\n",
      "cloudpickle:          <not installed>\n",
      "connectorx:           <not installed>\n",
      "deltalake:            <not installed>\n",
      "fastexcel:            <not installed>\n",
      "fsspec:               <not installed>\n",
      "gevent:               <not installed>\n",
      "hvplot:               <not installed>\n",
      "matplotlib:           3.9.0\n",
      "nest_asyncio:         1.6.0\n",
      "numpy:                1.26.4\n",
      "openpyxl:             <not installed>\n",
      "pandas:               2.2.2\n",
      "pyarrow:              16.1.0\n",
      "pydantic:             <not installed>\n",
      "pyiceberg:            <not installed>\n",
      "pyxlsb:               <not installed>\n",
      "sqlalchemy:           2.0.30\n",
      "torch:                <not installed>\n",
      "xlsx2csv:             <not installed>\n",
      "xlsxwriter:           <not installed>\n"
     ]
    }
   ],
   "source": [
    "#| label: show-versions\n",
    "pl.show_versions()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89d36196",
   "metadata": {},
   "source": [
    "How many cores are available for parallelism?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "show-cores",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: show-cores\n",
    "pl.thread_pool_size()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "812a8d2c",
   "metadata": {},
   "source": [
    "# Preliminaries\n",
    "\n",
    "\n",
    "## A Polars Frame Can Hold Anything\n",
    "\n",
    "Fun fact- Polars, like Pandas, can store anything within a cell. For instance, a Polars frame can hold a Polars frame.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "make-polars-frame",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape: (2, 2)\n",
      "┌───────────────┬─────┐\n",
      "│ a             ┆ b   │\n",
      "│ ---           ┆ --- │\n",
      "│ object        ┆ str │\n",
      "╞═══════════════╪═════╡\n",
      "│ shape: (3, 2) ┆ a   │\n",
      "│ ┌─────┬─────┐ ┆     │\n",
      "│ │ x   ┆ y   │ ┆     │\n",
      "│ │ --- ┆ …     ┆     │\n",
      "│ shape: (3, 2) ┆ b   │\n",
      "│ ┌─────┬─────┐ ┆     │\n",
      "│ │ x   ┆ y   │ ┆     │\n",
      "│ │ --- ┆ …     ┆     │\n",
      "└───────────────┴─────┘\n"
     ]
    }
   ],
   "source": [
    "#| label: make-polars-frame\n",
    "\n",
    "df = pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [\n",
    "            pl.DataFrame({\"x\": [1, 2, 3], \"y\": [4, 5, 6]}), \n",
    "            pl.DataFrame({\"x\": [7, 8, 9], \"y\": [10, 11, 12]})\n",
    "            ],\n",
    "        \"b\": [\"a\", \"b\"]\n",
    "    }\n",
    ")\n",
    "\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "523924d6",
   "metadata": {},
   "source": [
    "Things to note: \n",
    "\n",
    "- The dtype of the frame is `Polars Object`. \n",
    "\n",
    "\n",
    "## Motivation For Nested dtypes\n",
    "\n",
    "Consider the following scenarios:\n",
    "\n",
    "1. I want to compute with part-of-strings generated by a **split**. \n",
    "\n",
    "2. I want to store and compute with arrays of **varying lengths**. Examples include: paths of cars on the globe, paths of users in a website, \n",
    "\n",
    "3. I want to **group columns** together, and compute with them as a unit.\n",
    "   1. For convenience of access.\n",
    "   2. Because I am missing a `foo_horizontal()` method.\n",
    "\n",
    "4. I want to read a JSON/XML/YML with **nested structures**.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "How can I go about?\n",
    "\n",
    "1. Since a Polars object, like a Python object, can hold anything, I could store the nested data as a **Polars object**.\n",
    "1. I could generate **many columns**, and pad with nulls when lengths differ.\n",
    "2. I could store each unit as a **Polars Series**.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "The **Python object** option is always available, and I will try to **avoid it**. \n",
    "If I can avoid Polars (i.e. Rust) calling Python, I will.\n",
    "\n",
    "The second options could work. It is actually a good one, if I can figure out the **column access** (regex?).\n",
    "\n",
    "The third option would be great. Unlike the first, I would remain within Rus. Unlike the second, I will have a **named unit** to access. \n",
    "\n",
    "This is today's topic. \n",
    "\n",
    "::: {.callout-important}\n",
    "The Polars nested dtypes are inherited from the Arrow format. \n",
    "So these are actually \"Arrow Nested dtypes with Polars' functionality\". \n",
    ":::\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "## Nested dtypes in Polars\n",
    "\n",
    "The Polars nested dtypes, inherited from [PyArrow](https://wesm.github.io/arrow-site-test/format/Layout.html):\n",
    "\n",
    "\n",
    "1.  **Polars List** \n",
    "2.  **Polars Array** \n",
    "3.  **Polars Struct** \n",
    "\n",
    "A **Polars list** is a Polars Series within a cell: All elements in the cell must have the same dtype; the elements are unnamed so accessed by index or filter.\n",
    "Do not be confused by the name. A Polars list is not a Python list.  \n",
    "\n",
    "A **Polars array** is a Polars list with a fixed length: All elements in the cell must have the same dtype and the same length; the elements are unnamed so accessed by index or filter. Another mental model of a Polars array is a numpy 2D array within a Polars frame.\n",
    "\n",
    "\n",
    "A **Polars struct** is a Polars DataFrame within a cell: All elements in the cell must have the same fields; the elements are named so accessed by field name. \n",
    "Do not think of the struct as a Python dict within a cell, because all rows must have the same fields.\n",
    "\n",
    "\n",
    "\n",
    "# Polars List {#sec-list}\n",
    "\n",
    "\n",
    "\n",
    "## Making a Polars List\n",
    "\n",
    "Make a Polars list from a Python list.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "make-list",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>list[i64]</td><td>str</td></tr></thead><tbody><tr><td>[1, 2, 3]</td><td>&quot;a&quot;</td></tr><tr><td>[4, 5, 6]</td><td>&quot;b&quot;</td></tr><tr><td>[7, 8, 9]</td><td>&quot;c&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌───────────┬─────┐\n",
       "│ a         ┆ b   │\n",
       "│ ---       ┆ --- │\n",
       "│ list[i64] ┆ str │\n",
       "╞═══════════╪═════╡\n",
       "│ [1, 2, 3] ┆ a   │\n",
       "│ [4, 5, 6] ┆ b   │\n",
       "│ [7, 8, 9] ┆ c   │\n",
       "└───────────┴─────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: make-list\n",
    "\n",
    "pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [[1, 2, 3], [4, 5, 6], [7, 8, 9]],\n",
    "        \"b\": [\"a\", \"b\", \"c\"]\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f188bf55",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- `a` is not a Python list, rather, it is a Polars list. \n",
    "- Can a Polars list hold a Polars list? Yes. It can hold any Polars object, even nested ones (list of list of list, etc.); provided that all elements in the cell have the same dtype.\n",
    "\n",
    "Make a Polars list of Python lists of Python lists:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "make-list-of-list",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>list[list[i64]]</td><td>str</td></tr></thead><tbody><tr><td>[[1, 2, 3], [4, 5, 6], [7, 8, 9]]</td><td>&quot;a&quot;</td></tr><tr><td>[[10, 11, 12], [13, 14, 15], [16, 17, 18]]</td><td>&quot;b&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2, 2)\n",
       "┌────────────────────────────────────────────┬─────┐\n",
       "│ a                                          ┆ b   │\n",
       "│ ---                                        ┆ --- │\n",
       "│ list[list[i64]]                            ┆ str │\n",
       "╞════════════════════════════════════════════╪═════╡\n",
       "│ [[1, 2, 3], [4, 5, 6], [7, 8, 9]]          ┆ a   │\n",
       "│ [[10, 11, 12], [13, 14, 15], [16, 17, 18]] ┆ b   │\n",
       "└────────────────────────────────────────────┴─────┘"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: make-list-of-list\n",
    "\n",
    "pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [[[1, 2, 3], [4, 5, 6], [7, 8, 9]], [[10, 11, 12], [13, 14, 15], [16, 17, 18]]],\n",
    "        \"b\": [\"a\", \"b\"]\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f76ed993",
   "metadata": {},
   "source": [
    "More often you will not make a Polars list directly, but rather it will be the result of some operation:\n",
    "\n",
    "1. When grouping without aggregation. \n",
    "1. When splitting a string of unknown length (knonwn length will return a Polars struct, and I expect in the future a Polars array).\n",
    "2. When wrapping a bunch of columns with `pl.concat_list()`.\n",
    "3. When \"imploding\" (aka \"collapsing\") a columns.  \n",
    "\n",
    "\n",
    "\n",
    "### List from Aggregation\n",
    "\n",
    "A Polars group_by, will actually create a Polars list internally, before applying the `.agg()` method.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "list-from-aggregation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>i64</td><td>list[i64]</td></tr></thead><tbody><tr><td>2</td><td>[3, 4]</td></tr><tr><td>3</td><td>[5, 6]</td></tr><tr><td>1</td><td>[1, 2]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌─────┬───────────┐\n",
       "│ a   ┆ b         │\n",
       "│ --- ┆ ---       │\n",
       "│ i64 ┆ list[i64] │\n",
       "╞═════╪═══════════╡\n",
       "│ 2   ┆ [3, 4]    │\n",
       "│ 3   ┆ [5, 6]    │\n",
       "│ 1   ┆ [1, 2]    │\n",
       "└─────┴───────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-from-aggregation\n",
    "\n",
    "df = pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [1, 1, 2, 2, 3, 3],\n",
    "        \"b\": [1, 2, 3, 4, 5, 6]\n",
    "    }\n",
    ")\n",
    "\n",
    "df.group_by(\"a\").agg(pl.col(\"b\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22b2b2fc",
   "metadata": {},
   "source": [
    "### List From Splitting a String\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "make-frame-with-string-to-split",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;apple, banana, orange&quot;</td></tr><tr><td>&quot;apple, banana&quot;</td></tr><tr><td>&quot;apple&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌───────────────────────┐\n",
       "│ a                     │\n",
       "│ ---                   │\n",
       "│ str                   │\n",
       "╞═══════════════════════╡\n",
       "│ apple, banana, orange │\n",
       "│ apple, banana         │\n",
       "│ apple                 │\n",
       "└───────────────────────┘"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: make-frame-with-string-to-split\n",
    "\n",
    "df_strings = pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [\"apple, banana, orange\", \"apple, banana\", \"apple\"],\n",
    "    }\n",
    ")\n",
    "\n",
    "df_strings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad41ee02",
   "metadata": {},
   "source": [
    "Now split `a` into... whatever Polars decides to return. \n",
    "Hint: a Polars list of Polars strings.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e68b152e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>list[str]</td></tr></thead><tbody><tr><td>[&quot;apple&quot;, &quot;banana&quot;, &quot;orange&quot;]</td></tr><tr><td>[&quot;apple&quot;, &quot;banana&quot;]</td></tr><tr><td>[&quot;apple&quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌───────────────────────────────┐\n",
       "│ a                             │\n",
       "│ ---                           │\n",
       "│ list[str]                     │\n",
       "╞═══════════════════════════════╡\n",
       "│ [\"apple\", \"banana\", \"orange\"] │\n",
       "│ [\"apple\", \"banana\"]           │\n",
       "│ [\"apple\"]                     │\n",
       "└───────────────────────────────┘"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_strings\n",
    "    .select(\n",
    "        pl.col('a').str.split(\", \")\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85c73768",
   "metadata": {},
   "source": [
    "### List from Imploding a Column\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "list-from-imploding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>i64</td><td>list[i64]</td></tr></thead><tbody><tr><td>1</td><td>[1, 2, … 6]</td></tr><tr><td>1</td><td>[1, 2, … 6]</td></tr><tr><td>2</td><td>[1, 2, … 6]</td></tr><tr><td>2</td><td>[1, 2, … 6]</td></tr><tr><td>3</td><td>[1, 2, … 6]</td></tr><tr><td>3</td><td>[1, 2, … 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌─────┬─────────────┐\n",
       "│ a   ┆ b           │\n",
       "│ --- ┆ ---         │\n",
       "│ i64 ┆ list[i64]   │\n",
       "╞═════╪═════════════╡\n",
       "│ 1   ┆ [1, 2, … 6] │\n",
       "│ 1   ┆ [1, 2, … 6] │\n",
       "│ 2   ┆ [1, 2, … 6] │\n",
       "│ 2   ┆ [1, 2, … 6] │\n",
       "│ 3   ┆ [1, 2, … 6] │\n",
       "│ 3   ┆ [1, 2, … 6] │\n",
       "└─────┴─────────────┘"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-from-imploding\n",
    "\n",
    "df.with_columns(pl.col(\"b\").implode())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a0e50a2",
   "metadata": {},
   "source": [
    "Try replacing `.with_columns()` with `.select()` and see what happens.\n",
    "\n",
    "\n",
    "Implode `group_by()`:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "implode-within-group",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>i64</td><td>list[i64]</td></tr></thead><tbody><tr><td>3</td><td>[5, 6]</td></tr><tr><td>1</td><td>[1, 2]</td></tr><tr><td>2</td><td>[3, 4]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌─────┬───────────┐\n",
       "│ a   ┆ b         │\n",
       "│ --- ┆ ---       │\n",
       "│ i64 ┆ list[i64] │\n",
       "╞═════╪═══════════╡\n",
       "│ 3   ┆ [5, 6]    │\n",
       "│ 1   ┆ [1, 2]    │\n",
       "│ 2   ┆ [3, 4]    │\n",
       "└─────┴───────────┘"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: implode-within-group\n",
    "\n",
    "df.group_by(\"a\").agg(pl.col(\"b\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0674cb9",
   "metadata": {},
   "source": [
    "Implode `over()`:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "implode-over",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| label: implode-over\n",
    "#| eval: false\n",
    "\n",
    "df.with_columns(pl.col(\"b\").over(\"a\")) # no good\n",
    "\n",
    "df.with_columns(pl.col(\"b\").implode().over(\"a\")) # no good\n",
    "\n",
    "df.with_columns(pl.concat_list('b').over(\"a\")) # no good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2363e776",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th><th>over_join</th><th>over_explode</th><th>over_group_to_rows</th></tr><tr><td>i64</td><td>i64</td><td>list[i64]</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>1</td><td>[1, 2]</td><td>1</td><td>1</td></tr><tr><td>1</td><td>2</td><td>[1, 2]</td><td>2</td><td>2</td></tr><tr><td>2</td><td>3</td><td>[3, 4]</td><td>3</td><td>3</td></tr><tr><td>2</td><td>4</td><td>[3, 4]</td><td>4</td><td>4</td></tr><tr><td>3</td><td>5</td><td>[5, 6]</td><td>5</td><td>5</td></tr><tr><td>3</td><td>6</td><td>[5, 6]</td><td>6</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 5)\n",
       "┌─────┬─────┬───────────┬──────────────┬────────────────────┐\n",
       "│ a   ┆ b   ┆ over_join ┆ over_explode ┆ over_group_to_rows │\n",
       "│ --- ┆ --- ┆ ---       ┆ ---          ┆ ---                │\n",
       "│ i64 ┆ i64 ┆ list[i64] ┆ i64          ┆ i64                │\n",
       "╞═════╪═════╪═══════════╪══════════════╪════════════════════╡\n",
       "│ 1   ┆ 1   ┆ [1, 2]    ┆ 1            ┆ 1                  │\n",
       "│ 1   ┆ 2   ┆ [1, 2]    ┆ 2            ┆ 2                  │\n",
       "│ 2   ┆ 3   ┆ [3, 4]    ┆ 3            ┆ 3                  │\n",
       "│ 2   ┆ 4   ┆ [3, 4]    ┆ 4            ┆ 4                  │\n",
       "│ 3   ┆ 5   ┆ [5, 6]    ┆ 5            ┆ 5                  │\n",
       "│ 3   ┆ 6   ┆ [5, 6]    ┆ 6            ┆ 6                  │\n",
       "└─────┴─────┴───────────┴──────────────┴────────────────────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .with_columns(\n",
    "        pl.col(\"b\").over(\"a\", mapping_strategy=\"join\").alias('over_join'),\n",
    "        \n",
    "        pl.col(\"b\").over(\"a\", mapping_strategy=\"explode\").alias('over_explode'),\n",
    "        \n",
    "        pl.col(\"b\").over(\"a\", mapping_strategy=\"group_to_rows\").alias('over_group_to_rows'),\n",
    "    )\n",
    ")\n",
    "# good!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bed208e8",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- The `mapping_strategy` argument govern the way the result within group is propagate to each row. Trying chainging its value to see the effect.\n",
    "- See [here](https://github.com/pola-rs/polars/pull/6487) for more context on how `.over()` works and why.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "### List From `pl.concat_list()`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "list-from-concatenation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td></tr><tr><td>[1, 2]</td></tr><tr><td>[2, 3]</td></tr><tr><td>[2, 4]</td></tr><tr><td>[3, 5]</td></tr><tr><td>[3, 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌───────────┐\n",
       "│ a         │\n",
       "│ ---       │\n",
       "│ list[i64] │\n",
       "╞═══════════╡\n",
       "│ [1, 1]    │\n",
       "│ [1, 2]    │\n",
       "│ [2, 3]    │\n",
       "│ [2, 4]    │\n",
       "│ [3, 5]    │\n",
       "│ [3, 6]    │\n",
       "└───────────┘"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-from-concatenation\n",
    "\n",
    "df.select(pl.concat_list([pl.col(\"a\"), pl.col(\"b\")]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd0719e8",
   "metadata": {},
   "source": [
    "### List From `.list.concat()` \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3ad39f84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ab</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1, 1, 1, 1, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1, 2, 1, 1, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[2, 3, 2, 2, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[2, 4, 2, 2, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[3, 5, 3, 3, 3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[3, 6, 3, 3, 3]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                ab\n",
       "0  [1, 1, 1, 1, 1]\n",
       "1  [1, 2, 1, 1, 1]\n",
       "2  [2, 3, 2, 2, 2]\n",
       "3  [2, 4, 2, 2, 2]\n",
       "4  [3, 5, 3, 3, 3]\n",
       "5  [3, 6, 3, 3, 3]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .with_columns(\n",
    "        pl.concat_list([pl.col(\"a\"), pl.col(\"b\")]).alias('ab')\n",
    "        )\n",
    "    .select(pl.col('ab').list.concat(['a','a','a']))\n",
    "    .to_pandas()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0abd77e",
   "metadata": {},
   "source": [
    "## Operating on List Elements\n",
    "\n",
    "Arguably, the only motivation for using a Polars list, is for easy access to the list and its elements. \n",
    "You are thus welcome to think of equivalent operations, without the `.list.` API. \n",
    "\n",
    "\n",
    "An element can be accessed and trasformed with `list.eval(pl.element())`, which will expose (almost) all the methods of a `pl.Expr()` object.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "list-eval-element",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th></tr><tr><td>list[i64]</td></tr></thead><tbody><tr><td>[1001, 1001]</td></tr><tr><td>[1001, 1002]</td></tr><tr><td>[1002, 1003]</td></tr><tr><td>[1002, 1004]</td></tr><tr><td>[1003, 1005]</td></tr><tr><td>[1003, 1006]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌──────────────┐\n",
       "│ ab           │\n",
       "│ ---          │\n",
       "│ list[i64]    │\n",
       "╞══════════════╡\n",
       "│ [1001, 1001] │\n",
       "│ [1001, 1002] │\n",
       "│ [1002, 1003] │\n",
       "│ [1002, 1004] │\n",
       "│ [1003, 1005] │\n",
       "│ [1003, 1006] │\n",
       "└──────────────┘"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-eval-element\n",
    "\n",
    "df_with_list = (\n",
    "    df\n",
    "    .with_columns(\n",
    "        pl.concat_list(['a','b']).alias('ab')\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab').list.eval(\n",
    "            pl.element().add(1000)\n",
    "            )\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36961d86",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- `.eval()` belongs to the `.list` namespace. \n",
    "- `pl.element()` is a selector that selects the elements of the list. It has almost all the methods available to `pl.col()`.\n",
    "- `.eval()` currently cannot access other columns. You will probably discover this is a considerable limitation.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "list-eval-element-methods",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['__abs__', '__add__', '__and__', '__annotations__', '__array_ufunc__', '__bool__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__floordiv__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__invert__', '__le__', '__lt__', '__mod__', '__module__', '__mul__', '__ne__', '__neg__', '__new__', '__or__', '__pos__', '__pow__', '__radd__', '__rand__', '__reduce__', '__reduce_ex__', '__repr__', '__rfloordiv__', '__rmod__', '__rmul__', '__ror__', '__rpow__', '__rsub__', '__rtruediv__', '__rxor__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__sub__', '__subclasshook__', '__truediv__', '__weakref__', '__xor__', '_accessors', '_from_pyexpr', '_map_batches_wrapper', '_pyexpr', '_register_plugin', '_repr_html_', 'abs', 'add', 'agg_groups', 'alias', 'all', 'and_', 'any', 'append', 'apply', 'approx_n_unique', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh', 'arg_max', 'arg_min', 'arg_sort', 'arg_true', 'arg_unique', 'arr', 'backward_fill', 'bin', 'bottom_k', 'bottom_k_by', 'cast', 'cat', 'cbrt', 'ceil', 'clip', 'clip_max', 'clip_min', 'cos', 'cosh', 'cot', 'count', 'cum_count', 'cum_max', 'cum_min', 'cum_prod', 'cum_sum', 'cumcount', 'cummax', 'cummin', 'cumprod', 'cumsum', 'cumulative_eval', 'cut', 'degrees', 'deserialize', 'diff', 'dot', 'drop_nans', 'drop_nulls', 'dt', 'entropy', 'eq', 'eq_missing', 'ewm_mean', 'ewm_mean_by', 'ewm_std', 'ewm_var', 'exclude', 'exp', 'explode', 'extend_constant', 'fill_nan', 'fill_null', 'filter', 'first', 'flatten', 'floor', 'floordiv', 'forward_fill', 'from_json', 'gather', 'gather_every', 'ge', 'get', 'gt', 'hash', 'head', 'hist', 'implode', 'inspect', 'interpolate', 'is_between', 'is_duplicated', 'is_finite', 'is_first', 'is_first_distinct', 'is_in', 'is_infinite', 'is_last', 'is_last_distinct', 'is_nan', 'is_not', 'is_not_nan', 'is_not_null', 'is_null', 'is_unique', 'keep_name', 'kurtosis', 'last', 'le', 'len', 'limit', 'list', 'log', 'log10', 'log1p', 'lower_bound', 'lt', 'map', 'map_alias', 'map_batches', 'map_dict', 'map_elements', 'max', 'mean', 'median', 'meta', 'min', 'mod', 'mode', 'mul', 'n_unique', 'name', 'nan_max', 'nan_min', 'ne', 'ne_missing', 'neg', 'not_', 'null_count', 'or_', 'over', 'pct_change', 'peak_max', 'peak_min', 'pipe', 'pow', 'prefix', 'product', 'qcut', 'quantile', 'radians', 'rank', 'rechunk', 'register_plugin', 'reinterpret', 'repeat_by', 'replace', 'reshape', 'reverse', 'rle', 'rle_id', 'rolling', 'rolling_apply', 'rolling_map', 'rolling_max', 'rolling_max_by', 'rolling_mean', 'rolling_mean_by', 'rolling_median', 'rolling_median_by', 'rolling_min', 'rolling_min_by', 'rolling_quantile', 'rolling_quantile_by', 'rolling_skew', 'rolling_std', 'rolling_std_by', 'rolling_sum', 'rolling_sum_by', 'rolling_var', 'rolling_var_by', 'round', 'round_sig_figs', 'sample', 'search_sorted', 'set_sorted', 'shift', 'shift_and_fill', 'shrink_dtype', 'shuffle', 'sign', 'sin', 'sinh', 'skew', 'slice', 'sort', 'sort_by', 'sqrt', 'std', 'str', 'struct', 'sub', 'suffix', 'sum', 'tail', 'take', 'take_every', 'tan', 'tanh', 'to_physical', 'top_k', 'top_k_by', 'truediv', 'unique', 'unique_counts', 'upper_bound', 'value_counts', 'var', 'where', 'xor']\n"
     ]
    }
   ],
   "source": [
    "#| label: list-eval-element-methods\n",
    "print(dir(pl.element()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d068a66a",
   "metadata": {},
   "source": [
    "`list.eval(pl.element())`, however, does not see to add much that cannot be done with `pl.col()` methods.\n",
    "The true power of the `.list` namespace, is in is pre-defined methods.\n",
    "\n",
    "\n",
    "## List Methods\n",
    "\n",
    "\n",
    "\n",
    "### Accessing Elements\n",
    "\n",
    "Recall; list elements are unnamed. They are thus accessed using indices (`iloc[]` style).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "list-get-and-gather",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>get_1</th><th>gather_1</th><th>slice_1</th><th>gather_2</th><th>slice_2</th><th>gather_every_2</th></tr><tr><td>list[i64]</td><td>i64</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>1</td><td>[1]</td><td>[1]</td><td>[1, 1]</td><td>[1, 1]</td><td>[1]</td></tr><tr><td>[1, 2]</td><td>1</td><td>[1]</td><td>[1]</td><td>[1, 2]</td><td>[1, 2]</td><td>[1]</td></tr><tr><td>[2, 3]</td><td>2</td><td>[2]</td><td>[2]</td><td>[2, 3]</td><td>[2, 3]</td><td>[2]</td></tr><tr><td>[2, 4]</td><td>2</td><td>[2]</td><td>[2]</td><td>[2, 4]</td><td>[2, 4]</td><td>[2]</td></tr><tr><td>[3, 5]</td><td>3</td><td>[3]</td><td>[3]</td><td>[3, 5]</td><td>[3, 5]</td><td>[3]</td></tr><tr><td>[3, 6]</td><td>3</td><td>[3]</td><td>[3]</td><td>[3, 6]</td><td>[3, 6]</td><td>[3]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 7)\n",
       "┌───────────┬───────┬───────────┬───────────┬───────────┬───────────┬────────────────┐\n",
       "│ ab        ┆ get_1 ┆ gather_1  ┆ slice_1   ┆ gather_2  ┆ slice_2   ┆ gather_every_2 │\n",
       "│ ---       ┆ ---   ┆ ---       ┆ ---       ┆ ---       ┆ ---       ┆ ---            │\n",
       "│ list[i64] ┆ i64   ┆ list[i64] ┆ list[i64] ┆ list[i64] ┆ list[i64] ┆ list[i64]      │\n",
       "╞═══════════╪═══════╪═══════════╪═══════════╪═══════════╪═══════════╪════════════════╡\n",
       "│ [1, 1]    ┆ 1     ┆ [1]       ┆ [1]       ┆ [1, 1]    ┆ [1, 1]    ┆ [1]            │\n",
       "│ [1, 2]    ┆ 1     ┆ [1]       ┆ [1]       ┆ [1, 2]    ┆ [1, 2]    ┆ [1]            │\n",
       "│ [2, 3]    ┆ 2     ┆ [2]       ┆ [2]       ┆ [2, 3]    ┆ [2, 3]    ┆ [2]            │\n",
       "│ [2, 4]    ┆ 2     ┆ [2]       ┆ [2]       ┆ [2, 4]    ┆ [2, 4]    ┆ [2]            │\n",
       "│ [3, 5]    ┆ 3     ┆ [3]       ┆ [3]       ┆ [3, 5]    ┆ [3, 5]    ┆ [3]            │\n",
       "│ [3, 6]    ┆ 3     ┆ [3]       ┆ [3]       ┆ [3, 6]    ┆ [3, 6]    ┆ [3]            │\n",
       "└───────────┴───────┴───────────┴───────────┴───────────┴───────────┴────────────────┘"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-get-and-gather\n",
    "\n",
    "(\n",
    "    df_with_list.select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.get(0).alias('get_1'), \n",
    "        pl.col('ab').list.gather([0]).alias('gather_1'), # returns a list\n",
    "        pl.col('ab').list.slice(0,1).alias('slice_1'), # returns a list\n",
    "        pl.col('ab').list.gather([0, 1]).alias('gather_2'),\n",
    "        pl.col('ab').list.slice(0,2).alias('slice_2'),\n",
    "        pl.col('ab').list.gather_every(2).alias('gather_every_2'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5138596",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- What returns a list, and what returns a single element?\n",
    "\n",
    "\n",
    "First, Head, Last, Tail\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "list-methods",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>first_1</th><th>first_2</th><th>first_5</th><th>last_2</th><th>last_1</th><th>last_3</th></tr><tr><td>list[i64]</td><td>i64</td><td>i64</td><td>list[i64]</td><td>i64</td><td>list[i64]</td><td>i64</td></tr></thead><tbody><tr><td>[1, 1]</td><td>1</td><td>1</td><td>[1]</td><td>1</td><td>[1]</td><td>1</td></tr><tr><td>[1, 2]</td><td>2</td><td>1</td><td>[1]</td><td>2</td><td>[2]</td><td>2</td></tr><tr><td>[2, 3]</td><td>3</td><td>2</td><td>[2]</td><td>3</td><td>[3]</td><td>3</td></tr><tr><td>[2, 4]</td><td>4</td><td>2</td><td>[2]</td><td>4</td><td>[4]</td><td>4</td></tr><tr><td>[3, 5]</td><td>5</td><td>3</td><td>[3]</td><td>5</td><td>[5]</td><td>5</td></tr><tr><td>[3, 6]</td><td>6</td><td>3</td><td>[3]</td><td>6</td><td>[6]</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 7)\n",
       "┌───────────┬─────────┬─────────┬───────────┬────────┬───────────┬────────┐\n",
       "│ ab        ┆ first_1 ┆ first_2 ┆ first_5   ┆ last_2 ┆ last_1    ┆ last_3 │\n",
       "│ ---       ┆ ---     ┆ ---     ┆ ---       ┆ ---    ┆ ---       ┆ ---    │\n",
       "│ list[i64] ┆ i64     ┆ i64     ┆ list[i64] ┆ i64    ┆ list[i64] ┆ i64    │\n",
       "╞═══════════╪═════════╪═════════╪═══════════╪════════╪═══════════╪════════╡\n",
       "│ [1, 1]    ┆ 1       ┆ 1       ┆ [1]       ┆ 1      ┆ [1]       ┆ 1      │\n",
       "│ [1, 2]    ┆ 2       ┆ 1       ┆ [1]       ┆ 2      ┆ [2]       ┆ 2      │\n",
       "│ [2, 3]    ┆ 3       ┆ 2       ┆ [2]       ┆ 3      ┆ [3]       ┆ 3      │\n",
       "│ [2, 4]    ┆ 4       ┆ 2       ┆ [2]       ┆ 4      ┆ [4]       ┆ 4      │\n",
       "│ [3, 5]    ┆ 5       ┆ 3       ┆ [3]       ┆ 5      ┆ [5]       ┆ 5      │\n",
       "│ [3, 6]    ┆ 6       ┆ 3       ┆ [3]       ┆ 6      ┆ [6]       ┆ 6      │\n",
       "└───────────┴─────────┴─────────┴───────────┴────────┴───────────┴────────┘"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-methods\n",
    "\n",
    "(\n",
    "    df_with_list.select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.get(1).alias('first_1'),\n",
    "        pl.col('ab').list.first().alias('first_2'),\n",
    "        pl.col('ab').list.head(1).alias('first_5'), # returns a list\n",
    "        pl.col('ab').list.last().alias('last_2'), \n",
    "        pl.col('ab').list.tail(1).alias('last_1'), # returns a list\n",
    "        pl.col('ab').list.tail(1).list.first().alias('last_3'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1d4da1e",
   "metadata": {},
   "source": [
    "Shift\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "list-shift",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>shift_1</th><th>shift_-1</th></tr><tr><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>[null, 1]</td><td>[1, null]</td></tr><tr><td>[1, 2]</td><td>[null, 1]</td><td>[2, null]</td></tr><tr><td>[2, 3]</td><td>[null, 2]</td><td>[3, null]</td></tr><tr><td>[2, 4]</td><td>[null, 2]</td><td>[4, null]</td></tr><tr><td>[3, 5]</td><td>[null, 3]</td><td>[5, null]</td></tr><tr><td>[3, 6]</td><td>[null, 3]</td><td>[6, null]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 3)\n",
       "┌───────────┬───────────┬───────────┐\n",
       "│ ab        ┆ shift_1   ┆ shift_-1  │\n",
       "│ ---       ┆ ---       ┆ ---       │\n",
       "│ list[i64] ┆ list[i64] ┆ list[i64] │\n",
       "╞═══════════╪═══════════╪═══════════╡\n",
       "│ [1, 1]    ┆ [null, 1] ┆ [1, null] │\n",
       "│ [1, 2]    ┆ [null, 1] ┆ [2, null] │\n",
       "│ [2, 3]    ┆ [null, 2] ┆ [3, null] │\n",
       "│ [2, 4]    ┆ [null, 2] ┆ [4, null] │\n",
       "│ [3, 5]    ┆ [null, 3] ┆ [5, null] │\n",
       "│ [3, 6]    ┆ [null, 3] ┆ [6, null] │\n",
       "└───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-shift\n",
    "\n",
    "(\n",
    "    df_with_list.select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.shift(1).alias('shift_1'),\n",
    "        pl.col('ab').list.shift(-1).alias('shift_-1'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d1d73df",
   "metadata": {},
   "source": [
    "Sample\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "list-sample",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ab</th>\n",
       "      <th>sample</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>[1, 1, 1, 1, 2, 1, 1, 1, 2, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[2, 3]</td>\n",
       "      <td>[2, 2, 3, 3, 2, 3, 2, 2, 2, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[2, 4]</td>\n",
       "      <td>[4, 4, 2, 2, 2, 4, 2, 2, 4, 4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[3, 5]</td>\n",
       "      <td>[5, 3, 5, 5, 3, 3, 5, 3, 3, 5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[3, 6]</td>\n",
       "      <td>[6, 3, 3, 3, 3, 3, 6, 3, 3, 3]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ab                          sample\n",
       "0  [1, 1]  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
       "1  [1, 2]  [1, 1, 1, 1, 2, 1, 1, 1, 2, 1]\n",
       "2  [2, 3]  [2, 2, 3, 3, 2, 3, 2, 2, 2, 2]\n",
       "3  [2, 4]  [4, 4, 2, 2, 2, 4, 2, 2, 4, 4]\n",
       "4  [3, 5]  [5, 3, 5, 5, 3, 3, 5, 3, 3, 5]\n",
       "5  [3, 6]  [6, 3, 3, 3, 3, 3, 6, 3, 3, 3]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-sample\n",
    "\n",
    "(\n",
    "    df_with_list.select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.sample(10, with_replacement=True).alias('sample'),\n",
    "        )\n",
    "    .to_pandas()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ede06bd",
   "metadata": {},
   "source": [
    "Careful where you put the `.sample()`!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3ad2d5ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ab</th>\n",
       "      <th>within_row</th>\n",
       "      <th>within_column</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]</td>\n",
       "      <td>[3, 6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>[2, 1, 1, 2, 1, 2, 1, 1, 2, 2]</td>\n",
       "      <td>[1, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[2, 3]</td>\n",
       "      <td>[3, 2, 2, 2, 3, 2, 3, 3, 2, 2]</td>\n",
       "      <td>[3, 6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[2, 4]</td>\n",
       "      <td>[4, 2, 2, 4, 2, 2, 4, 2, 2, 2]</td>\n",
       "      <td>[2, 4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[3, 5]</td>\n",
       "      <td>[3, 5, 5, 3, 5, 5, 5, 3, 3, 5]</td>\n",
       "      <td>[3, 6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[3, 6]</td>\n",
       "      <td>[3, 3, 6, 6, 3, 3, 3, 3, 3, 3]</td>\n",
       "      <td>[1, 2]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ab                      within_row within_column\n",
       "0  [1, 1]  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]        [3, 6]\n",
       "1  [1, 2]  [2, 1, 1, 2, 1, 2, 1, 1, 2, 2]        [1, 2]\n",
       "2  [2, 3]  [3, 2, 2, 2, 3, 2, 3, 3, 2, 2]        [3, 6]\n",
       "3  [2, 4]  [4, 2, 2, 4, 2, 2, 4, 2, 2, 2]        [2, 4]\n",
       "4  [3, 5]  [3, 5, 5, 3, 5, 5, 5, 3, 3, 5]        [3, 6]\n",
       "5  [3, 6]  [3, 3, 6, 6, 3, 3, 3, 3, 3, 3]        [1, 2]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        \n",
    "        pl.col('ab').list.sample(10, with_replacement=True).alias('within_row'),\n",
    "        \n",
    "        pl.col('ab').sample(6, with_replacement=True).alias('within_column'),\n",
    "    )\n",
    "    .to_pandas()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78a3d884",
   "metadata": {},
   "source": [
    "### Statistical Aggregations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "list-statistical-aggregations",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 11)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>min</th><th>max</th><th>sum</th><th>mean</th><th>median</th><th>var</th><th>std</th><th>n_unique</th><th>unique</th><th>len</th></tr><tr><td>list[i64]</td><td>i64</td><td>i64</td><td>i64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>u32</td><td>list[i64]</td><td>u32</td></tr></thead><tbody><tr><td>[1, 1]</td><td>1</td><td>1</td><td>2</td><td>1.0</td><td>1.0</td><td>0.0</td><td>0.0</td><td>1</td><td>[1]</td><td>2</td></tr><tr><td>[1, 2]</td><td>1</td><td>2</td><td>3</td><td>1.5</td><td>1.5</td><td>0.5</td><td>0.707107</td><td>2</td><td>[1, 2]</td><td>2</td></tr><tr><td>[2, 3]</td><td>2</td><td>3</td><td>5</td><td>2.5</td><td>2.5</td><td>0.5</td><td>0.707107</td><td>2</td><td>[2, 3]</td><td>2</td></tr><tr><td>[2, 4]</td><td>2</td><td>4</td><td>6</td><td>3.0</td><td>3.0</td><td>2.0</td><td>1.414214</td><td>2</td><td>[2, 4]</td><td>2</td></tr><tr><td>[3, 5]</td><td>3</td><td>5</td><td>8</td><td>4.0</td><td>4.0</td><td>2.0</td><td>1.414214</td><td>2</td><td>[3, 5]</td><td>2</td></tr><tr><td>[3, 6]</td><td>3</td><td>6</td><td>9</td><td>4.5</td><td>4.5</td><td>4.5</td><td>2.12132</td><td>2</td><td>[3, 6]</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 11)\n",
       "┌───────────┬─────┬─────┬─────┬───┬──────────┬──────────┬───────────┬─────┐\n",
       "│ ab        ┆ min ┆ max ┆ sum ┆ … ┆ std      ┆ n_unique ┆ unique    ┆ len │\n",
       "│ ---       ┆ --- ┆ --- ┆ --- ┆   ┆ ---      ┆ ---      ┆ ---       ┆ --- │\n",
       "│ list[i64] ┆ i64 ┆ i64 ┆ i64 ┆   ┆ f64      ┆ u32      ┆ list[i64] ┆ u32 │\n",
       "╞═══════════╪═════╪═════╪═════╪═══╪══════════╪══════════╪═══════════╪═════╡\n",
       "│ [1, 1]    ┆ 1   ┆ 1   ┆ 2   ┆ … ┆ 0.0      ┆ 1        ┆ [1]       ┆ 2   │\n",
       "│ [1, 2]    ┆ 1   ┆ 2   ┆ 3   ┆ … ┆ 0.707107 ┆ 2        ┆ [1, 2]    ┆ 2   │\n",
       "│ [2, 3]    ┆ 2   ┆ 3   ┆ 5   ┆ … ┆ 0.707107 ┆ 2        ┆ [2, 3]    ┆ 2   │\n",
       "│ [2, 4]    ┆ 2   ┆ 4   ┆ 6   ┆ … ┆ 1.414214 ┆ 2        ┆ [2, 4]    ┆ 2   │\n",
       "│ [3, 5]    ┆ 3   ┆ 5   ┆ 8   ┆ … ┆ 1.414214 ┆ 2        ┆ [3, 5]    ┆ 2   │\n",
       "│ [3, 6]    ┆ 3   ┆ 6   ┆ 9   ┆ … ┆ 2.12132  ┆ 2        ┆ [3, 6]    ┆ 2   │\n",
       "└───────────┴─────┴─────┴─────┴───┴──────────┴──────────┴───────────┴─────┘"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-statistical-aggregations\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.min().alias('min'),\n",
    "        pl.col('ab').list.max().alias('max'),\n",
    "        pl.col('ab').list.sum().alias('sum'),\n",
    "        pl.col('ab').list.mean().alias('mean'),\n",
    "        pl.col('ab').list.median().alias('median'),\n",
    "        pl.col('ab').list.var().alias('var'),\n",
    "        pl.col('ab').list.std().alias('std'),\n",
    "        pl.col('ab').list.n_unique().alias('n_unique'),\n",
    "        pl.col('ab').list.unique().alias('unique'),\n",
    "        pl.col('ab').list.len().alias('len'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aee353d0",
   "metadata": {},
   "source": [
    "### Ordering and Ranking\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "list-ordering-and-ranking",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>sort</th><th>reverse</th></tr><tr><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>[1, 1]</td><td>[1, 1]</td></tr><tr><td>[1, 2]</td><td>[1, 2]</td><td>[2, 1]</td></tr><tr><td>[2, 3]</td><td>[2, 3]</td><td>[3, 2]</td></tr><tr><td>[2, 4]</td><td>[2, 4]</td><td>[4, 2]</td></tr><tr><td>[3, 5]</td><td>[3, 5]</td><td>[5, 3]</td></tr><tr><td>[3, 6]</td><td>[3, 6]</td><td>[6, 3]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 3)\n",
       "┌───────────┬───────────┬───────────┐\n",
       "│ ab        ┆ sort      ┆ reverse   │\n",
       "│ ---       ┆ ---       ┆ ---       │\n",
       "│ list[i64] ┆ list[i64] ┆ list[i64] │\n",
       "╞═══════════╪═══════════╪═══════════╡\n",
       "│ [1, 1]    ┆ [1, 1]    ┆ [1, 1]    │\n",
       "│ [1, 2]    ┆ [1, 2]    ┆ [2, 1]    │\n",
       "│ [2, 3]    ┆ [2, 3]    ┆ [3, 2]    │\n",
       "│ [2, 4]    ┆ [2, 4]    ┆ [4, 2]    │\n",
       "│ [3, 5]    ┆ [3, 5]    ┆ [5, 3]    │\n",
       "│ [3, 6]    ┆ [3, 6]    ┆ [6, 3]    │\n",
       "└───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-ordering-and-ranking\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        # pl.col('ab').list.rank().alias('rank'),\n",
    "        pl.col('ab').list.sort().alias('sort'),\n",
    "        pl.col('ab').list.reverse().alias('reverse'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44968f26",
   "metadata": {},
   "source": [
    "### Sequence Operations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "list-diff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>diff</th></tr><tr><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>[null, 0]</td></tr><tr><td>[1, 2]</td><td>[null, 1]</td></tr><tr><td>[2, 3]</td><td>[null, 1]</td></tr><tr><td>[2, 4]</td><td>[null, 2]</td></tr><tr><td>[3, 5]</td><td>[null, 2]</td></tr><tr><td>[3, 6]</td><td>[null, 3]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌───────────┬───────────┐\n",
       "│ ab        ┆ diff      │\n",
       "│ ---       ┆ ---       │\n",
       "│ list[i64] ┆ list[i64] │\n",
       "╞═══════════╪═══════════╡\n",
       "│ [1, 1]    ┆ [null, 0] │\n",
       "│ [1, 2]    ┆ [null, 1] │\n",
       "│ [2, 3]    ┆ [null, 1] │\n",
       "│ [2, 4]    ┆ [null, 2] │\n",
       "│ [3, 5]    ┆ [null, 2] │\n",
       "│ [3, 6]    ┆ [null, 3] │\n",
       "└───────────┴───────────┘"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-diff\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.diff().alias('diff'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f838482",
   "metadata": {},
   "source": [
    "What about `.pct_change()`?\n",
    "The following will not work, but see @sec-pct-change for a workaround.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3b34eddf",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'ExprListNameSpace' object has no attribute 'pct_change'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[29], line 6\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#eval: false\u001b[39;00m\n\u001b[1;32m      2\u001b[0m (\n\u001b[1;32m      3\u001b[0m     df_with_list\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;241m.\u001b[39mselect(\n\u001b[1;32m      5\u001b[0m         pl\u001b[38;5;241m.\u001b[39mcol(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mab\u001b[39m\u001b[38;5;124m'\u001b[39m),\n\u001b[0;32m----> 6\u001b[0m         \u001b[43mpl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcol\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mab\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlist\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpct_change\u001b[49m()\u001b[38;5;241m.\u001b[39malias(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpct_change\u001b[39m\u001b[38;5;124m'\u001b[39m),\n\u001b[1;32m      7\u001b[0m         )\n\u001b[1;32m      8\u001b[0m \n\u001b[1;32m      9\u001b[0m )\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'ExprListNameSpace' object has no attribute 'pct_change'"
     ]
    }
   ],
   "source": [
    "#eval: false\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.pct_change().alias('pct_change'),\n",
    "        )\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aacfc4e6",
   "metadata": {},
   "source": [
    "### Logical Aggregations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "list-logical-aggregations",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>any</th><th>all</th></tr><tr><td>list[i64]</td><td>bool</td><td>bool</td></tr></thead><tbody><tr><td>[1, 1]</td><td>true</td><td>true</td></tr><tr><td>[1, 2]</td><td>true</td><td>false</td></tr><tr><td>[2, 3]</td><td>false</td><td>false</td></tr><tr><td>[2, 4]</td><td>false</td><td>false</td></tr><tr><td>[3, 5]</td><td>false</td><td>false</td></tr><tr><td>[3, 6]</td><td>false</td><td>false</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 3)\n",
       "┌───────────┬───────┬───────┐\n",
       "│ ab        ┆ any   ┆ all   │\n",
       "│ ---       ┆ ---   ┆ ---   │\n",
       "│ list[i64] ┆ bool  ┆ bool  │\n",
       "╞═══════════╪═══════╪═══════╡\n",
       "│ [1, 1]    ┆ true  ┆ true  │\n",
       "│ [1, 2]    ┆ true  ┆ false │\n",
       "│ [2, 3]    ┆ false ┆ false │\n",
       "│ [2, 4]    ┆ false ┆ false │\n",
       "│ [3, 5]    ┆ false ┆ false │\n",
       "│ [3, 6]    ┆ false ┆ false │\n",
       "└───────────┴───────┴───────┘"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-logical-aggregations\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.eval(pl.element().eq(1)).list.any().alias('any'),\n",
    "        pl.col('ab').list.eval(pl.element().eq(1)).list.all().alias('all'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5202b783",
   "metadata": {},
   "source": [
    "### String Operations\n",
    "\n",
    "- `list.join()` will join the strings in the list. I admit, I would have expected a `.list.str.concat()` instead. \n",
    "- `list.contains()` will check if the list contains a string.\n",
    "- `list.count_matches()` will count the number of times a string appears in the list.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "list-strings",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>sep</th><th>joined</th><th>contains_a</th><th>count_a</th></tr><tr><td>list[str]</td><td>str</td><td>str</td><td>bool</td><td>u32</td></tr></thead><tbody><tr><td>[&quot;a&quot;, &quot;b&quot;, &quot;c&quot;]</td><td>&quot;@&quot;</td><td>&quot;a@b@c&quot;</td><td>true</td><td>1</td></tr><tr><td>[&quot;d&quot;, &quot;e&quot;, &quot;f&quot;]</td><td>&quot;#&quot;</td><td>&quot;d#e#f&quot;</td><td>false</td><td>0</td></tr><tr><td>[&quot;g&quot;, &quot;h&quot;, &quot;i&quot;]</td><td>&quot;$&quot;</td><td>&quot;g$h$i&quot;</td><td>false</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 5)\n",
       "┌─────────────────┬─────┬────────┬────────────┬─────────┐\n",
       "│ ab              ┆ sep ┆ joined ┆ contains_a ┆ count_a │\n",
       "│ ---             ┆ --- ┆ ---    ┆ ---        ┆ ---     │\n",
       "│ list[str]       ┆ str ┆ str    ┆ bool       ┆ u32     │\n",
       "╞═════════════════╪═════╪════════╪════════════╪═════════╡\n",
       "│ [\"a\", \"b\", \"c\"] ┆ @   ┆ a@b@c  ┆ true       ┆ 1       │\n",
       "│ [\"d\", \"e\", \"f\"] ┆ #   ┆ d#e#f  ┆ false      ┆ 0       │\n",
       "│ [\"g\", \"h\", \"i\"] ┆ $   ┆ g$h$i  ┆ false      ┆ 0       │\n",
       "└─────────────────┴─────┴────────┴────────────┴─────────┘"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-strings\n",
    "\n",
    "df_2_with_list = pl.DataFrame(\n",
    "    {\n",
    "        \"ab\": [[\"a\", \"b\", \"c\"], [\"d\", \"e\", \"f\"], [\"g\", \"h\", \"i\"]],\n",
    "        'sep': ['@', '#', '$']\n",
    "    }\n",
    ")\n",
    "\n",
    "(\n",
    "    df_2_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('sep'),\n",
    "        pl.col('ab').list.join(pl.col('sep')).alias('joined'),\n",
    "        pl.col('ab').list.contains('a').alias('contains_a'),\n",
    "        pl.col('ab').list.count_matches('a').alias('count_a'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84658b10",
   "metadata": {},
   "source": [
    "### Filtering\n",
    "\n",
    "Currently there is no `.list.filter()` method (see [issue](https://github.com/pola-rs/polars/issues/9189)).\n",
    "There are, however, many workarounds. \n",
    "\n",
    "#### Option 1: Using `pl.element().filter()`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "list-filter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>gt_2</th></tr><tr><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>[]</td></tr><tr><td>[1, 2]</td><td>[]</td></tr><tr><td>[2, 3]</td><td>[3]</td></tr><tr><td>[2, 4]</td><td>[4]</td></tr><tr><td>[3, 5]</td><td>[3, 5]</td></tr><tr><td>[3, 6]</td><td>[3, 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌───────────┬───────────┐\n",
       "│ ab        ┆ gt_2      │\n",
       "│ ---       ┆ ---       │\n",
       "│ list[i64] ┆ list[i64] │\n",
       "╞═══════════╪═══════════╡\n",
       "│ [1, 1]    ┆ []        │\n",
       "│ [1, 2]    ┆ []        │\n",
       "│ [2, 3]    ┆ [3]       │\n",
       "│ [2, 4]    ┆ [4]       │\n",
       "│ [3, 5]    ┆ [3, 5]    │\n",
       "│ [3, 6]    ┆ [3, 6]    │\n",
       "└───────────┴───────────┘"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-filter\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        \n",
    "        pl.col('ab').list.eval(\n",
    "            pl.element()\n",
    "            .filter(pl.element().gt(2)),\n",
    "        )\n",
    "        .alias('gt_2'),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf05804e",
   "metadata": {},
   "source": [
    "#### Option 2: Using `.list.gather()` + `pl.arg_where()`\n",
    "\n",
    "Let's try to filter elements of a list based on the length of the string.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d8e8df81",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample data\n",
    "data = pl.DataFrame({\n",
    "    'string_list': [\n",
    "        [\"apple\", \"banana\", \"orange\", \"grape\", \"pear\"],\n",
    "        [\"apple\", \"banana\", \"orange\", \"grape\"],\n",
    "        [\"apple\", \"banana\", \"orange\"],\n",
    "        [\"apple\", \"banana\"],\n",
    "        [\"apple\"],\n",
    "        [\"apple\", \"banana\", \"orange\", \"grape\", \"pear\"],\n",
    "        [\"apple\", \"banana\", \"orange\", \"grape\"],\n",
    "    ],\n",
    "    }\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c662924",
   "metadata": {},
   "source": [
    "In each list, keep only strings of length 4.\n",
    "\n",
    "Using `.list.gather()` and `pl.arg_where()` and explode to access `pl.col().str` methods. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "84dc19ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (7, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>string_list</th></tr><tr><td>list[str]</td></tr></thead><tbody><tr><td>[&quot;pear&quot;]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[&quot;pear&quot;]</td></tr><tr><td>[]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (7, 1)\n",
       "┌─────────────┐\n",
       "│ string_list │\n",
       "│ ---         │\n",
       "│ list[str]   │\n",
       "╞═════════════╡\n",
       "│ [\"pear\"]    │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ [\"pear\"]    │\n",
       "│ []          │\n",
       "└─────────────┘"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    data\n",
    "    .with_row_index('i')\n",
    "    .with_columns(\n",
    "        pl.col('string_list').list.gather(\n",
    "            pl.arg_where(\n",
    "                pl.col('string_list').explode().str.len_chars().eq(4))\n",
    "                )\n",
    "                .over('i')\n",
    "            )\n",
    "    .drop('i')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0b5382e",
   "metadata": {},
   "source": [
    "Without exploding: Using `list.gather()` to filter, and `list.eval()` to access `pl.element().str` methods.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e041298c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (7, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>string_list</th></tr><tr><td>list[str]</td></tr></thead><tbody><tr><td>[&quot;pear&quot;]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[&quot;pear&quot;]</td></tr><tr><td>[]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (7, 1)\n",
       "┌─────────────┐\n",
       "│ string_list │\n",
       "│ ---         │\n",
       "│ list[str]   │\n",
       "╞═════════════╡\n",
       "│ [\"pear\"]    │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ [\"pear\"]    │\n",
       "│ []          │\n",
       "└─────────────┘"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    data\n",
    "    .select(\n",
    "        pl.col('string_list').list.gather(\n",
    "            pl.col('string_list').list.eval(\n",
    "                pl.arg_where(\n",
    "                    pl.element().str.len_chars().eq(4)\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa62ea98",
   "metadata": {},
   "source": [
    "Explode once, and call the output twice using the Walrus operator; once to access `pl.col().filter()` and then to access `pl.col().str`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7dc76e71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (7, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>string_list</th></tr><tr><td>list[str]</td></tr></thead><tbody><tr><td>[&quot;pear&quot;]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[]</td></tr><tr><td>[&quot;pear&quot;]</td></tr><tr><td>[]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (7, 1)\n",
       "┌─────────────┐\n",
       "│ string_list │\n",
       "│ ---         │\n",
       "│ list[str]   │\n",
       "╞═════════════╡\n",
       "│ [\"pear\"]    │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ []          │\n",
       "│ [\"pear\"]    │\n",
       "│ []          │\n",
       "└─────────────┘"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    data\n",
    "    .with_row_index('i')\n",
    "    .with_columns(\n",
    "        (\n",
    "            (cntr := pl.col('string_list').explode())\n",
    "            .filter(cntr.str.len_chars().eq(4))\n",
    "            )\n",
    "        .implode()\n",
    "        .over('i')\n",
    "        )\n",
    "    .drop('i')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ff0afec",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- If you are unfamilar with Python's (not Polars') Walrus operator `:=`, see [here](https://realpython.com/python-walrus-operator/).\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "### Missing\n",
    "\n",
    "I believe everything can be done with `.list.eval(pl.element().method())`.\n",
    "\n",
    "\n",
    "\n",
    "### Set Operations\n",
    "\n",
    ".list.set_intersection\n",
    "\n",
    ".list.set_union\n",
    "\n",
    ".list.set_difference\n",
    "\n",
    ".list.set_symmetric_difference\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "dc4ac92d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>intersection</th><th>union</th><th>union_2</th><th>difference</th><th>symmetric_difference</th></tr><tr><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>[1]</td><td>[1, 11]</td><td>[1, null]</td><td>[1]</td><td>[2, 3]</td></tr><tr><td>[1, 2]</td><td>[1, 2]</td><td>[1, 2, 11]</td><td>[1, 2, null]</td><td>[1]</td><td>[3]</td></tr><tr><td>[2, 3]</td><td>[2, 3]</td><td>[2, 3, 11]</td><td>[2, 3, null]</td><td>[]</td><td>[1]</td></tr><tr><td>[2, 4]</td><td>[2]</td><td>[2, 4, 11]</td><td>[2, 4, null]</td><td>[4]</td><td>[4, 1, 3]</td></tr><tr><td>[3, 5]</td><td>[3]</td><td>[3, 5, 11]</td><td>[3, 5, null]</td><td>[5]</td><td>[5, 1, 2]</td></tr><tr><td>[3, 6]</td><td>[3]</td><td>[3, 6, 11]</td><td>[3, 6, null]</td><td>[6]</td><td>[6, 1, 2]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 6)\n",
       "┌───────────┬──────────────┬────────────┬──────────────┬────────────┬──────────────────────┐\n",
       "│ ab        ┆ intersection ┆ union      ┆ union_2      ┆ difference ┆ symmetric_difference │\n",
       "│ ---       ┆ ---          ┆ ---        ┆ ---          ┆ ---        ┆ ---                  │\n",
       "│ list[i64] ┆ list[i64]    ┆ list[i64]  ┆ list[i64]    ┆ list[i64]  ┆ list[i64]            │\n",
       "╞═══════════╪══════════════╪════════════╪══════════════╪════════════╪══════════════════════╡\n",
       "│ [1, 1]    ┆ [1]          ┆ [1, 11]    ┆ [1, null]    ┆ [1]        ┆ [2, 3]               │\n",
       "│ [1, 2]    ┆ [1, 2]       ┆ [1, 2, 11] ┆ [1, 2, null] ┆ [1]        ┆ [3]                  │\n",
       "│ [2, 3]    ┆ [2, 3]       ┆ [2, 3, 11] ┆ [2, 3, null] ┆ []         ┆ [1]                  │\n",
       "│ [2, 4]    ┆ [2]          ┆ [2, 4, 11] ┆ [2, 4, null] ┆ [4]        ┆ [4, 1, 3]            │\n",
       "│ [3, 5]    ┆ [3]          ┆ [3, 5, 11] ┆ [3, 5, null] ┆ [5]        ┆ [5, 1, 2]            │\n",
       "│ [3, 6]    ┆ [3]          ┆ [3, 6, 11] ┆ [3, 6, null] ┆ [6]        ┆ [6, 1, 2]            │\n",
       "└───────────┴──────────────┴────────────┴──────────────┴────────────┴──────────────────────┘"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        pl.col('ab').list.set_intersection([1,2,3]).alias('intersection'),\n",
    "        \n",
    "        pl.col('ab').list.set_union(['11']).alias('union'),\n",
    "\n",
    "        pl.col('ab').list.set_union(['a']).alias('union_2'),\n",
    "        \n",
    "        pl.col('ab').list.set_difference([2,3]).alias('difference'),\n",
    "        \n",
    "        pl.col('ab').list.set_symmetric_difference([1,2,3]).alias('symmetric_difference'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fcd4c9d",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- We can pass a Python list and not a Python set. \n",
    "- Set union with an appropriate dtype will return Null. Is that what you would expect? \n",
    "\n",
    "\n",
    "### Polars List to Array or Struct\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "list-to-struct",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>ab_struct</th><th>ab_array</th></tr><tr><td>list[i64]</td><td>struct[2]</td><td>array[i64, 2]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>{1,1}</td><td>[1, 1]</td></tr><tr><td>[1, 2]</td><td>{1,2}</td><td>[1, 2]</td></tr><tr><td>[2, 3]</td><td>{2,3}</td><td>[2, 3]</td></tr><tr><td>[2, 4]</td><td>{2,4}</td><td>[2, 4]</td></tr><tr><td>[3, 5]</td><td>{3,5}</td><td>[3, 5]</td></tr><tr><td>[3, 6]</td><td>{3,6}</td><td>[3, 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 3)\n",
       "┌───────────┬───────────┬───────────────┐\n",
       "│ ab        ┆ ab_struct ┆ ab_array      │\n",
       "│ ---       ┆ ---       ┆ ---           │\n",
       "│ list[i64] ┆ struct[2] ┆ array[i64, 2] │\n",
       "╞═══════════╪═══════════╪═══════════════╡\n",
       "│ [1, 1]    ┆ {1,1}     ┆ [1, 1]        │\n",
       "│ [1, 2]    ┆ {1,2}     ┆ [1, 2]        │\n",
       "│ [2, 3]    ┆ {2,3}     ┆ [2, 3]        │\n",
       "│ [2, 4]    ┆ {2,4}     ┆ [2, 4]        │\n",
       "│ [3, 5]    ┆ {3,5}     ┆ [3, 5]        │\n",
       "│ [3, 6]    ┆ {3,6}     ┆ [3, 6]        │\n",
       "└───────────┴───────────┴───────────────┘"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-to-struct\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "\n",
    "        pl.col('ab').list.to_struct().alias('ab_struct'),\n",
    "\n",
    "        pl.col('ab').list.to_array(width=2).alias('ab_array'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "482425c6",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- An array has a fixed length; it is not inferred, you must specify it.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "## Exploding and Imploding {#sec-list-explode}\n",
    "\n",
    "**Explode**: from list onto a column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "21e7dbcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (12, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab_exploded</th></tr><tr><td>i64</td></tr></thead><tbody><tr><td>1</td></tr><tr><td>1</td></tr><tr><td>1</td></tr><tr><td>2</td></tr><tr><td>2</td></tr><tr><td>&hellip;</td></tr><tr><td>4</td></tr><tr><td>3</td></tr><tr><td>5</td></tr><tr><td>3</td></tr><tr><td>6</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (12, 1)\n",
       "┌─────────────┐\n",
       "│ ab_exploded │\n",
       "│ ---         │\n",
       "│ i64         │\n",
       "╞═════════════╡\n",
       "│ 1           │\n",
       "│ 1           │\n",
       "│ 1           │\n",
       "│ 2           │\n",
       "│ 2           │\n",
       "│ …           │\n",
       "│ 4           │\n",
       "│ 3           │\n",
       "│ 5           │\n",
       "│ 3           │\n",
       "│ 6           │\n",
       "└─────────────┘"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab').list.explode().alias('ab_exploded'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c49cfaa",
   "metadata": {},
   "source": [
    "**Implode**: from column to list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1b5ae8a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab_imploded</th><th>ab_exploded_imploded</th></tr><tr><td>list[list[i64]]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[[1, 1], [1, 2], … [3, 6]]</td><td>[1, 1, … 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 2)\n",
       "┌────────────────────────────┬──────────────────────┐\n",
       "│ ab_imploded                ┆ ab_exploded_imploded │\n",
       "│ ---                        ┆ ---                  │\n",
       "│ list[list[i64]]            ┆ list[i64]            │\n",
       "╞════════════════════════════╪══════════════════════╡\n",
       "│ [[1, 1], [1, 2], … [3, 6]] ┆ [1, 1, … 6]          │\n",
       "└────────────────────────────┴──────────────────────┘"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab').implode().alias('ab_imploded'),\n",
    "        \n",
    "        pl.col('ab').list.explode().implode().alias('ab_exploded_imploded'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c748b8b0",
   "metadata": {},
   "source": [
    "::: {.callout-important}\n",
    "The following is a great trick to use when the `.list` namespace does not have the method you need.\n",
    ":::\n",
    "\n",
    "Explode within a group will convert the list to a column, on which you can operate using usual `pl.col()` methods.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1bc7fc7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>this_is_actually_a_column</th></tr><tr><td>u32</td><td>list[i64]</td></tr></thead><tbody><tr><td>0</td><td>[1, 1]</td></tr><tr><td>1</td><td>[1, 2]</td></tr><tr><td>2</td><td>[2, 3]</td></tr><tr><td>3</td><td>[2, 4]</td></tr><tr><td>4</td><td>[3, 5]</td></tr><tr><td>5</td><td>[3, 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌───────┬───────────────────────────┐\n",
       "│ index ┆ this_is_actually_a_column │\n",
       "│ ---   ┆ ---                       │\n",
       "│ u32   ┆ list[i64]                 │\n",
       "╞═══════╪═══════════════════════════╡\n",
       "│ 0     ┆ [1, 1]                    │\n",
       "│ 1     ┆ [1, 2]                    │\n",
       "│ 2     ┆ [2, 3]                    │\n",
       "│ 3     ┆ [2, 4]                    │\n",
       "│ 4     ┆ [3, 5]                    │\n",
       "│ 5     ┆ [3, 6]                    │\n",
       "└───────┴───────────────────────────┘"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .with_row_index()\n",
    "    .group_by('index')\n",
    "    .agg(\n",
    "        pl.col('ab').list.explode() \n",
    "        .alias('this_is_actually_a_column'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "282e07fe",
   "metadata": {},
   "source": [
    "There is also  `df.explode()`, alongside the `pl.col().explode()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "df-explode",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (12, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th><th>ab</th></tr><tr><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>1</td><td>1</td></tr><tr><td>1</td><td>1</td><td>1</td></tr><tr><td>1</td><td>2</td><td>1</td></tr><tr><td>1</td><td>2</td><td>2</td></tr><tr><td>2</td><td>3</td><td>2</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>2</td><td>4</td><td>4</td></tr><tr><td>3</td><td>5</td><td>3</td></tr><tr><td>3</td><td>5</td><td>5</td></tr><tr><td>3</td><td>6</td><td>3</td></tr><tr><td>3</td><td>6</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (12, 3)\n",
       "┌─────┬─────┬─────┐\n",
       "│ a   ┆ b   ┆ ab  │\n",
       "│ --- ┆ --- ┆ --- │\n",
       "│ i64 ┆ i64 ┆ i64 │\n",
       "╞═════╪═════╪═════╡\n",
       "│ 1   ┆ 1   ┆ 1   │\n",
       "│ 1   ┆ 1   ┆ 1   │\n",
       "│ 1   ┆ 2   ┆ 1   │\n",
       "│ 1   ┆ 2   ┆ 2   │\n",
       "│ 2   ┆ 3   ┆ 2   │\n",
       "│ …   ┆ …   ┆ …   │\n",
       "│ 2   ┆ 4   ┆ 4   │\n",
       "│ 3   ┆ 5   ┆ 3   │\n",
       "│ 3   ┆ 5   ┆ 5   │\n",
       "│ 3   ┆ 6   ┆ 3   │\n",
       "│ 3   ┆ 6   ┆ 6   │\n",
       "└─────┴─────┴─────┘"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: df-explode\n",
    "df_with_list.explode('ab')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5e41150",
   "metadata": {},
   "source": [
    "## Examples \n",
    "\n",
    "\n",
    "#### pct_change {#sec-pct-change}\n",
    "\n",
    "In the absence of a `.list.pct_change()` method, we can use the following workaround.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "8594735e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>pct_change</th></tr><tr><td>u32</td><td>list[f64]</td></tr></thead><tbody><tr><td>0</td><td>[null, 0.0]</td></tr><tr><td>1</td><td>[null, 1.0]</td></tr><tr><td>2</td><td>[null, 0.5]</td></tr><tr><td>3</td><td>[null, 1.0]</td></tr><tr><td>4</td><td>[null, 0.666667]</td></tr><tr><td>5</td><td>[null, 1.0]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌───────┬──────────────────┐\n",
       "│ index ┆ pct_change       │\n",
       "│ ---   ┆ ---              │\n",
       "│ u32   ┆ list[f64]        │\n",
       "╞═══════╪══════════════════╡\n",
       "│ 0     ┆ [null, 0.0]      │\n",
       "│ 1     ┆ [null, 1.0]      │\n",
       "│ 2     ┆ [null, 0.5]      │\n",
       "│ 3     ┆ [null, 1.0]      │\n",
       "│ 4     ┆ [null, 0.666667] │\n",
       "│ 5     ┆ [null, 1.0]      │\n",
       "└───────┴──────────────────┘"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .with_row_index()\n",
    "    .group_by('index')\n",
    "    .agg(\n",
    "        pl.col('ab').list.explode().pct_change().alias('pct_change'),\n",
    "        ) \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d5e65cb",
   "metadata": {},
   "source": [
    "Could I have done that with the multiple column approach?\n",
    "Would have needed `pl.col.pct_change_horizontal()`.\n",
    "\n",
    "\n",
    "\n",
    "#### ECDF {#sec-ecdf}\n",
    "\n",
    "How to compute within a column?\n",
    "In the abcense of `.list.ecdf()` and `pl.order_horizontal()`, we can use the following.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "list-ecdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th><th>ranks</th><th>ecdf_1</th></tr><tr><td>list[i64]</td><td>list[f64]</td><td>list[f64]</td></tr></thead><tbody><tr><td>[1, 1]</td><td>[1.5, 1.5]</td><td>[0.75, 0.75]</td></tr><tr><td>[1, 2]</td><td>[1.0, 2.0]</td><td>[0.5, 1.0]</td></tr><tr><td>[2, 3]</td><td>[1.0, 2.0]</td><td>[0.5, 1.0]</td></tr><tr><td>[2, 4]</td><td>[1.0, 2.0]</td><td>[0.5, 1.0]</td></tr><tr><td>[3, 5]</td><td>[1.0, 2.0]</td><td>[0.5, 1.0]</td></tr><tr><td>[3, 6]</td><td>[1.0, 2.0]</td><td>[0.5, 1.0]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 3)\n",
       "┌───────────┬────────────┬──────────────┐\n",
       "│ ab        ┆ ranks      ┆ ecdf_1       │\n",
       "│ ---       ┆ ---        ┆ ---          │\n",
       "│ list[i64] ┆ list[f64]  ┆ list[f64]    │\n",
       "╞═══════════╪════════════╪══════════════╡\n",
       "│ [1, 1]    ┆ [1.5, 1.5] ┆ [0.75, 0.75] │\n",
       "│ [1, 2]    ┆ [1.0, 2.0] ┆ [0.5, 1.0]   │\n",
       "│ [2, 3]    ┆ [1.0, 2.0] ┆ [0.5, 1.0]   │\n",
       "│ [2, 4]    ┆ [1.0, 2.0] ┆ [0.5, 1.0]   │\n",
       "│ [3, 5]    ┆ [1.0, 2.0] ┆ [0.5, 1.0]   │\n",
       "│ [3, 6]    ┆ [1.0, 2.0] ┆ [0.5, 1.0]   │\n",
       "└───────────┴────────────┴──────────────┘"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: list-ecdf\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab'),\n",
    "        \n",
    "        # Ranks\n",
    "        pl.col('ab').list.eval(pl.element().rank()).alias('ranks'),\n",
    "        \n",
    "        # Option 1:ECDF with hard coded length (makes sense for an array)\n",
    "        pl.col('ab').list.eval(pl.element().rank().truediv(2)).alias('ecdf_1'),\n",
    "        \n",
    "        # Option 2: ECDF with length of list. Will not work. \n",
    "        # pl.col('ab').list.eval(pl.element().rank().truediv(pl.col('ab').list.len())).alias('ecdf_2'),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6057fac1",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- `.list.eval(pl.element())` can do more than point_wise operations. Think about the `rank()` example.\n",
    "\n",
    "- Option 2 will fail because `.list.eval()` cannot reference another column. See [issue](https://github.com/pola-rs/polars/issues/7210).\n",
    "\n",
    "- `.rank()` has a `method` argument that can be used to specify how to deal with ties.\n",
    "\n",
    "\n",
    "Here is another attempt, which does not assume the length of the list is known (not fixed). \n",
    "The following will not work, but may be fixed. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c92cc423",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>ab</th></tr><tr><td>u32</td><td>list[f64]</td></tr></thead><tbody><tr><td>0</td><td>[0.75, 0.75]</td></tr><tr><td>1</td><td>[0.5, 1.0]</td></tr><tr><td>2</td><td>[0.5, 1.0]</td></tr><tr><td>3</td><td>[0.5, 1.0]</td></tr><tr><td>4</td><td>[0.5, 1.0]</td></tr><tr><td>5</td><td>[0.5, 1.0]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌───────┬──────────────┐\n",
       "│ index ┆ ab           │\n",
       "│ ---   ┆ ---          │\n",
       "│ u32   ┆ list[f64]    │\n",
       "╞═══════╪══════════════╡\n",
       "│ 0     ┆ [0.75, 0.75] │\n",
       "│ 1     ┆ [0.5, 1.0]   │\n",
       "│ 2     ┆ [0.5, 1.0]   │\n",
       "│ 3     ┆ [0.5, 1.0]   │\n",
       "│ 4     ┆ [0.5, 1.0]   │\n",
       "│ 5     ┆ [0.5, 1.0]   │\n",
       "└───────┴──────────────┘"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_list\n",
    "    .with_row_index()\n",
    "    .group_by('index')\n",
    "    .agg(\n",
    "        (abrank:=pl.col('ab').explode().rank()).truediv(abrank.len()),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c30644b",
   "metadata": {},
   "source": [
    "#### ECDF wrt Other Column\n",
    "\n",
    "@sec-ecdf showed how to compute the ECDF of a list. Say we want to evaluate the ECDF of column `a` wrt column `b`.\n",
    "\n",
    "TODO\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "696f8009",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| eval: false\n",
    "\n",
    "(\n",
    "    df_with_list\n",
    "    .with_columns(pl.col('a').implode())    \n",
    "    .with_row_index()\n",
    "    .group_by('index')\n",
    "    .agg(\n",
    "        (\n",
    "            a_ge_b_ind := pl.col('a').explode().ge(pl.col('b'))\n",
    "        )\n",
    "    )\n",
    "    .with_columns(\n",
    "        a_ge_b_ind.list.sum().truediv(a_ge_b_ind.list.len())\n",
    "    )\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07a1ae68",
   "metadata": {},
   "source": [
    "#### arg_max_horizontal\n",
    "\n",
    "Note, there currently is no `pl.arg_max_horizontal()` method.\n",
    "We will try to make one by concatenating columns to list, and then using the `.list.arg_max()`.\n",
    "In particular, we will want the arg_max col name, and not the index. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "arg-max-horizontal",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th><th>arg_max_col_name</th></tr><tr><td>i64</td><td>i64</td><td>str</td></tr></thead><tbody><tr><td>1</td><td>1</td><td>&quot;a&quot;</td></tr><tr><td>1</td><td>2</td><td>&quot;b&quot;</td></tr><tr><td>2</td><td>3</td><td>&quot;b&quot;</td></tr><tr><td>2</td><td>4</td><td>&quot;b&quot;</td></tr><tr><td>3</td><td>5</td><td>&quot;b&quot;</td></tr><tr><td>3</td><td>6</td><td>&quot;b&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 3)\n",
       "┌─────┬─────┬──────────────────┐\n",
       "│ a   ┆ b   ┆ arg_max_col_name │\n",
       "│ --- ┆ --- ┆ ---              │\n",
       "│ i64 ┆ i64 ┆ str              │\n",
       "╞═════╪═════╪══════════════════╡\n",
       "│ 1   ┆ 1   ┆ a                │\n",
       "│ 1   ┆ 2   ┆ b                │\n",
       "│ 2   ┆ 3   ┆ b                │\n",
       "│ 2   ┆ 4   ┆ b                │\n",
       "│ 3   ┆ 5   ┆ b                │\n",
       "│ 3   ┆ 6   ┆ b                │\n",
       "└─────┴─────┴──────────────────┘"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: arg-max-horizontal\n",
    "\n",
    "(\n",
    "    df\n",
    "    # Identify the arg_max\n",
    "    .with_columns(\n",
    "        pl.concat_list([pl.col(\"a\"), pl.col(\"b\")])\n",
    "        .list.arg_max()\n",
    "        .alias('arg_max'),\n",
    "        )\n",
    "    # Return the name of the argmax\n",
    "    .select(\n",
    "        'a',\n",
    "        'b',\n",
    "        (\n",
    "            pl.col('arg_max')\n",
    "            .map_elements(lambda i: df.columns[i], return_dtype=pl.Utf8)\n",
    "            .alias('arg_max_col_name')\n",
    "        )\n",
    "        \n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd4ef3b1",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- Can you find a more efficient way to do this? Replacing the lambda?\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#### Quantile\n",
    "\n",
    "Recall there is no `.list.quantile()` method; so we need to brew our own. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "quantile-example",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ab</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[2, 1, 2, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[2, 2, 2, 3, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[2, 4, 4, 4, 4, 4, 4, 2, 2, 2, 4, 4, 2, 4, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[3, 5, 3, 5, 5, 3, 5, 5, 3, 5, 3, 3, 5, 5, 5, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[3, 3, 6, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  ab\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
       "1  [2, 1, 2, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 1, ...\n",
       "2  [2, 2, 2, 3, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, ...\n",
       "3  [2, 4, 4, 4, 4, 4, 4, 2, 2, 2, 4, 4, 2, 4, 2, ...\n",
       "4  [3, 5, 3, 5, 5, 3, 5, 5, 3, 5, 3, 3, 5, 5, 5, ...\n",
       "5  [3, 3, 6, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ..."
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: quantile-example\n",
    "\n",
    "df_for_quantile = (\n",
    "    df_with_list\n",
    "    .select(\n",
    "        pl.col('ab').list.sample(100, with_replacement=True),\n",
    "    )\n",
    ")\n",
    "\n",
    "df_for_quantile.to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f4ff18a",
   "metadata": {},
   "source": [
    "Option 1: `group_by` then `explode`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "2edd513d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>index</th><th>ab</th></tr><tr><td>u32</td><td>f64</td></tr></thead><tbody><tr><td>0</td><td>1.0</td></tr><tr><td>1</td><td>1.0</td></tr><tr><td>2</td><td>2.0</td></tr><tr><td>3</td><td>2.0</td></tr><tr><td>4</td><td>3.0</td></tr><tr><td>5</td><td>3.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌───────┬─────┐\n",
       "│ index ┆ ab  │\n",
       "│ ---   ┆ --- │\n",
       "│ u32   ┆ f64 │\n",
       "╞═══════╪═════╡\n",
       "│ 0     ┆ 1.0 │\n",
       "│ 1     ┆ 1.0 │\n",
       "│ 2     ┆ 2.0 │\n",
       "│ 3     ┆ 2.0 │\n",
       "│ 4     ┆ 3.0 │\n",
       "│ 5     ┆ 3.0 │\n",
       "└───────┴─────┘"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_for_quantile\n",
    "    .with_row_index()\n",
    "    .group_by('index')\n",
    "    .agg(\n",
    "        pl.col('ab').explode().quantile(0.2)\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29f69c97",
   "metadata": {},
   "source": [
    "Option 2: `list.eval()`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "05e7b945",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>quantile_0.2</th></tr><tr><td>f64</td></tr></thead><tbody><tr><td>1.0</td></tr><tr><td>1.0</td></tr><tr><td>2.0</td></tr><tr><td>2.0</td></tr><tr><td>3.0</td></tr><tr><td>3.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌──────────────┐\n",
       "│ quantile_0.2 │\n",
       "│ ---          │\n",
       "│ f64          │\n",
       "╞══════════════╡\n",
       "│ 1.0          │\n",
       "│ 1.0          │\n",
       "│ 2.0          │\n",
       "│ 2.0          │\n",
       "│ 3.0          │\n",
       "│ 3.0          │\n",
       "└──────────────┘"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_for_quantile\n",
    "    .select(\n",
    "        pl.col('ab').list.eval(pl.element().quantile(0.2))\n",
    "        .list.first() # to extract a float from a list\n",
    "        .alias('quantile_0.2'),\n",
    "    )\n",
    "\n",
    "\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4203a554",
   "metadata": {},
   "source": [
    "# Polars Array {#sec-array}\n",
    "\n",
    "Polars Arrays as Polars lists with a fixed length.\n",
    "\n",
    "## Making a Polars Array\n",
    "\n",
    "Make a Polars array from a Python list.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "make-array",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Array_1</th><th>Array_2</th></tr><tr><td>array[i64, 2]</td><td>array[i64, 3]</td></tr></thead><tbody><tr><td>[1, 3]</td><td>[1, 7, 3]</td></tr><tr><td>[2, 5]</td><td>[8, 1, 0]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2, 2)\n",
       "┌───────────────┬───────────────┐\n",
       "│ Array_1       ┆ Array_2       │\n",
       "│ ---           ┆ ---           │\n",
       "│ array[i64, 2] ┆ array[i64, 3] │\n",
       "╞═══════════════╪═══════════════╡\n",
       "│ [1, 3]        ┆ [1, 7, 3]     │\n",
       "│ [2, 5]        ┆ [8, 1, 0]     │\n",
       "└───────────────┴───────────────┘"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: make-array\n",
    "\n",
    "pl.DataFrame(\n",
    "    [\n",
    "        pl.Series(\"Array_1\", [[1, 3], [2, 5]]),\n",
    "        pl.Series(\"Array_2\", [[1, 7, 3], [8, 1, 0]]),\n",
    "    ],\n",
    "    schema={\n",
    "        \"Array_1\": pl.Array(pl.Int64, 2),\n",
    "        \"Array_2\": pl.Array(pl.Int64, 3),\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14078bd2",
   "metadata": {},
   "source": [
    "Things to note:\n",
    "\n",
    "- The dtype of the array is specified in the schema. Otherwise, a Polars list would have been inferred. \n",
    "\n",
    "\n",
    "## Array Methods\n",
    "\n",
    "Currently, list methods and array methods seem to be the same. \n",
    "I expect these lists to diverge in the future, because more operations can be defined when assuming a fixed length.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Polars Struct {#sec-struct}\n",
    "\n",
    "Quoting [RhoSignal](https://www.rhosignal.com/posts/nested-dtypes/):\n",
    "> pl.Struct type is a nested collection of columns. The pl.Struct is really just a way of having a nested namespace for columns. The underlying columns are just normal Polars Series.\n",
    "\n",
    "Also from the [Arrow documentation](https://wesm.github.io/arrow-site-test/format/Layout.html):\n",
    "> A struct is a nested type parameterized by an ordered sequence of relative types (which can all be distinct), called its fields.\n",
    "\n",
    "So in summary:\n",
    "\n",
    "2. The Python dict analogy is not a good one.\n",
    "3. A Polars struct is a column which consists of another Polars dataframe.\n",
    "4. There no difference in performance between a Polars struct and a set of columns (unless an entire field is null. See the [Arrow documentation](https://wesm.github.io/arrow-site-test/format/Layout.html)).\n",
    "5. \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "##  Making a Polars Struct\n",
    "\n",
    "Make a Polars struct from a Python dict.\n",
    "But recall- there is nothing \"Dicti\" about the struct!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "make-struct",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>struct[2]</td><td>str</td></tr></thead><tbody><tr><td>{1,2}</td><td>&quot;a&quot;</td></tr><tr><td>{3,4}</td><td>&quot;b&quot;</td></tr><tr><td>{5,6}</td><td>&quot;c&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌───────────┬─────┐\n",
       "│ a         ┆ b   │\n",
       "│ ---       ┆ --- │\n",
       "│ struct[2] ┆ str │\n",
       "╞═══════════╪═════╡\n",
       "│ {1,2}     ┆ a   │\n",
       "│ {3,4}     ┆ b   │\n",
       "│ {5,6}     ┆ c   │\n",
       "└───────────┴─────┘"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: make-struct\n",
    "\n",
    "pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [{\"x\": 1, \"y\": 2}, {\"x\": 3, \"y\": 4}, {\"x\": 5, \"y\": 6}],\n",
    "        \"b\": [\"a\", \"b\", \"c\"]\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6018d485",
   "metadata": {},
   "source": [
    "More often you will not make a Polars struct directly, but rather you will create one from a Polars DataFrame:\n",
    "\n",
    "1. By directly creating a struct.\n",
    "2. As the output of some operation, like `pl.Expr().value_counts()`, or all the horizontal cumulators like `pl.cum_sum_horizontal()`, `pl.cum_reduce()`, `pl.cum_fold()`, ...\n",
    "3. From a Polars list. \n",
    "4. By splitting a string to a known length. \n",
    "\n",
    "\n",
    "\n",
    "### Making a Polars Struct Directly\n",
    "\n",
    "`pl.struct()` with existing names:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "make-struct-directly",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>struct</th></tr><tr><td>struct[2]</td></tr></thead><tbody><tr><td>{1,1}</td></tr><tr><td>{1,2}</td></tr><tr><td>{2,3}</td></tr><tr><td>{2,4}</td></tr><tr><td>{3,5}</td></tr><tr><td>{3,6}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌───────────┐\n",
       "│ struct    │\n",
       "│ ---       │\n",
       "│ struct[2] │\n",
       "╞═══════════╡\n",
       "│ {1,1}     │\n",
       "│ {1,2}     │\n",
       "│ {2,3}     │\n",
       "│ {2,4}     │\n",
       "│ {3,5}     │\n",
       "│ {3,6}     │\n",
       "└───────────┘"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: make-struct-directly\n",
    "df_with_struct = df.select(\n",
    "    pl.struct(['a','b'])\n",
    "    .alias('struct')\n",
    "    )\n",
    "\n",
    "df_with_struct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f78d750",
   "metadata": {},
   "source": [
    "Verify that the column is a struct:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "verify-struct",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('struct', Struct({'a': Int64, 'b': Int64}))])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: verify-struct\n",
    "df_with_struct.schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81e2e913",
   "metadata": {},
   "source": [
    "`pl.struct()` with field naming:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "f82242f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('aaa', Struct({'aaa': Int64, 'bbb': Int64}))])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(pl.struct(aaa=pl.col('a'), bbb=pl.col('b'))).schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "686a300a",
   "metadata": {},
   "source": [
    "### Struct As Output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ed51dd24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>struct[2]</td></tr></thead><tbody><tr><td>{3,2}</td></tr><tr><td>{1,2}</td></tr><tr><td>{2,2}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌───────────┐\n",
       "│ a         │\n",
       "│ ---       │\n",
       "│ struct[2] │\n",
       "╞═══════════╡\n",
       "│ {3,2}     │\n",
       "│ {1,2}     │\n",
       "│ {2,2}     │\n",
       "└───────────┘"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(pl.col('a').value_counts())\n",
    "# df.select(pl.col('a').value_counts().struct.json_encode())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96aac027",
   "metadata": {},
   "source": [
    "### Struct From List\n",
    "\n",
    "When a list has constant length, which you want to access by name, it makes sense to convert it to a struct.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "struct-from-list",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ab</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;{&quot;field_0&quot;:1,&quot;field_1&quot;:1}&quot;</td></tr><tr><td>&quot;{&quot;field_0&quot;:1,&quot;field_1&quot;:2}&quot;</td></tr><tr><td>&quot;{&quot;field_0&quot;:2,&quot;field_1&quot;:3}&quot;</td></tr><tr><td>&quot;{&quot;field_0&quot;:2,&quot;field_1&quot;:4}&quot;</td></tr><tr><td>&quot;{&quot;field_0&quot;:3,&quot;field_1&quot;:5}&quot;</td></tr><tr><td>&quot;{&quot;field_0&quot;:3,&quot;field_1&quot;:6}&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌───────────────────────────┐\n",
       "│ ab                        │\n",
       "│ ---                       │\n",
       "│ str                       │\n",
       "╞═══════════════════════════╡\n",
       "│ {\"field_0\":1,\"field_1\":1} │\n",
       "│ {\"field_0\":1,\"field_1\":2} │\n",
       "│ {\"field_0\":2,\"field_1\":3} │\n",
       "│ {\"field_0\":2,\"field_1\":4} │\n",
       "│ {\"field_0\":3,\"field_1\":5} │\n",
       "│ {\"field_0\":3,\"field_1\":6} │\n",
       "└───────────────────────────┘"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-from-list\n",
    "df_with_list.select(pl.col('ab').list.to_struct().struct.json_encode())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "038e85e0",
   "metadata": {},
   "source": [
    "### Split a String\n",
    "\n",
    "If you want to govern the length of the output of a split, use `.str.splitn()` or `.str.split_exact()` instead of `.str.split()`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "struct-from-splitn",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>struct[4]</td></tr></thead><tbody><tr><td>{&quot;apple&quot;,&quot; banana&quot;,&quot; orange&quot;,null}</td></tr><tr><td>{&quot;apple&quot;,&quot; banana&quot;,null,null}</td></tr><tr><td>{&quot;apple&quot;,null,null,null}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌────────────────────────────────────┐\n",
       "│ a                                  │\n",
       "│ ---                                │\n",
       "│ struct[4]                          │\n",
       "╞════════════════════════════════════╡\n",
       "│ {\"apple\",\" banana\",\" orange\",null} │\n",
       "│ {\"apple\",\" banana\",null,null}      │\n",
       "│ {\"apple\",null,null,null}           │\n",
       "└────────────────────────────────────┘"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-from-splitn\n",
    "\n",
    "(\n",
    "    df_strings\n",
    "    .select(\n",
    "        pl.col('a').str.splitn(\",\",4)\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "struct-from-split-exact",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>struct[5]</td></tr></thead><tbody><tr><td>{&quot;apple&quot;,&quot; banana&quot;,&quot; orange&quot;,null,null}</td></tr><tr><td>{&quot;apple&quot;,&quot; banana&quot;,null,null,null}</td></tr><tr><td>{&quot;apple&quot;,null,null,null,null}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 1)\n",
       "┌─────────────────────────────────────────┐\n",
       "│ a                                       │\n",
       "│ ---                                     │\n",
       "│ struct[5]                               │\n",
       "╞═════════════════════════════════════════╡\n",
       "│ {\"apple\",\" banana\",\" orange\",null,null} │\n",
       "│ {\"apple\",\" banana\",null,null,null}      │\n",
       "│ {\"apple\",null,null,null,null}           │\n",
       "└─────────────────────────────────────────┘"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-from-split-exact\n",
    "\n",
    "(\n",
    "    df_strings\n",
    "    .select(\n",
    "        pl.col('a').str.split_exact(\",\",4)\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfb474b8",
   "metadata": {},
   "source": [
    "## Struct Methods\n",
    "\n",
    "\n",
    "### Extracting Elements\n",
    "\n",
    "Getting a field is actually getting a column from a sub-frame.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "struct-get",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th><th>b</th></tr><tr><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>1</td></tr><tr><td>1</td><td>2</td></tr><tr><td>2</td><td>3</td></tr><tr><td>2</td><td>4</td></tr><tr><td>3</td><td>5</td></tr><tr><td>3</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌─────┬─────┐\n",
       "│ a   ┆ b   │\n",
       "│ --- ┆ --- │\n",
       "│ i64 ┆ i64 │\n",
       "╞═════╪═════╡\n",
       "│ 1   ┆ 1   │\n",
       "│ 1   ┆ 2   │\n",
       "│ 2   ┆ 3   │\n",
       "│ 2   ┆ 4   │\n",
       "│ 3   ┆ 5   │\n",
       "│ 3   ┆ 6   │\n",
       "└─────┴─────┘"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-get\n",
    "(\n",
    "    df_with_struct\n",
    "    .select(\n",
    "        pl.col('struct').struct.field('a'),\n",
    "        pl.col('struct').struct.field('b'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f37b1dea",
   "metadata": {},
   "source": [
    "To text in JSON format:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "struct-to-json",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>struct</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;{&quot;a&quot;:1,&quot;b&quot;:1}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:1,&quot;b&quot;:2}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:2,&quot;b&quot;:3}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:2,&quot;b&quot;:4}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:3,&quot;b&quot;:5}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:3,&quot;b&quot;:6}&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌───────────────┐\n",
       "│ struct        │\n",
       "│ ---           │\n",
       "│ str           │\n",
       "╞═══════════════╡\n",
       "│ {\"a\":1,\"b\":1} │\n",
       "│ {\"a\":1,\"b\":2} │\n",
       "│ {\"a\":2,\"b\":3} │\n",
       "│ {\"a\":2,\"b\":4} │\n",
       "│ {\"a\":3,\"b\":5} │\n",
       "│ {\"a\":3,\"b\":6} │\n",
       "└───────────────┘"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-to-json\n",
    "(\n",
    "    df_with_struct\n",
    "    .select(\n",
    "        pl.col('struct').struct.json_encode()\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8c93d3b",
   "metadata": {},
   "source": [
    "### The New (>= 0.20.27) Way to Access Fields\n",
    "\n",
    "As of version 0.20.27, you can access fields using a `.with_fields()` context, akin to the `.with_columns()` context.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f39186bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>struct</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;{&quot;a&quot;:1,&quot;b&quot;:1,&quot;new_field&quot;:1}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:1,&quot;b&quot;:2,&quot;new_field&quot;:2}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:2,&quot;b&quot;:3,&quot;new_field&quot;:6}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:2,&quot;b&quot;:4,&quot;new_field&quot;:8}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:3,&quot;b&quot;:5,&quot;new_field&quot;:15}&quot;</td></tr><tr><td>&quot;{&quot;a&quot;:3,&quot;b&quot;:6,&quot;new_field&quot;:18}&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌──────────────────────────────┐\n",
       "│ struct                       │\n",
       "│ ---                          │\n",
       "│ str                          │\n",
       "╞══════════════════════════════╡\n",
       "│ {\"a\":1,\"b\":1,\"new_field\":1}  │\n",
       "│ {\"a\":1,\"b\":2,\"new_field\":2}  │\n",
       "│ {\"a\":2,\"b\":3,\"new_field\":6}  │\n",
       "│ {\"a\":2,\"b\":4,\"new_field\":8}  │\n",
       "│ {\"a\":3,\"b\":5,\"new_field\":15} │\n",
       "│ {\"a\":3,\"b\":6,\"new_field\":18} │\n",
       "└──────────────────────────────┘"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df_with_struct\n",
    "    .select(\n",
    "        pl.col('struct').struct.with_fields(\n",
    "            new_field = pl.col('struct').struct.field('a') * \n",
    "            pl.col('struct').struct.field('b')\n",
    "            )\n",
    "        .struct.json_encode()\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be0ecacd",
   "metadata": {},
   "source": [
    "## Struct to List\n",
    "\n",
    "There is no `.struct.to_list()` method. \n",
    "This makes sense, since a struct does not require all fields to have the same dtype. \n",
    "In the case where all fields have the same dtype, you can use the following.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "struct-to-list",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td></tr><tr><td>[1, 2]</td></tr><tr><td>[2, 3]</td></tr><tr><td>[2, 4]</td></tr><tr><td>[3, 5]</td></tr><tr><td>[3, 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌───────────┐\n",
       "│ a         │\n",
       "│ ---       │\n",
       "│ list[i64] │\n",
       "╞═══════════╡\n",
       "│ [1, 1]    │\n",
       "│ [1, 2]    │\n",
       "│ [2, 3]    │\n",
       "│ [2, 4]    │\n",
       "│ [3, 5]    │\n",
       "│ [3, 6]    │\n",
       "└───────────┘"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-to-list\n",
    "(\n",
    "    df_with_struct\n",
    "    .unnest('struct') #from struct to columns\n",
    "    .select(pl.concat_list(pl.all())) # from columns to list\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afa75485",
   "metadata": {},
   "source": [
    "You can also consider element-by-element extractions.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "struct-to-list-element-by-element",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>a</th></tr><tr><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 1]</td></tr><tr><td>[1, 2]</td></tr><tr><td>[2, 3]</td></tr><tr><td>[2, 4]</td></tr><tr><td>[3, 5]</td></tr><tr><td>[3, 6]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 1)\n",
       "┌───────────┐\n",
       "│ a         │\n",
       "│ ---       │\n",
       "│ list[i64] │\n",
       "╞═══════════╡\n",
       "│ [1, 1]    │\n",
       "│ [1, 2]    │\n",
       "│ [2, 3]    │\n",
       "│ [2, 4]    │\n",
       "│ [3, 5]    │\n",
       "│ [3, 6]    │\n",
       "└───────────┘"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: struct-to-list-element-by-element\n",
    "(\n",
    "    df_with_struct\n",
    "    .select(\n",
    "        pl.concat_list([\n",
    "            pl.col('struct').struct.field('a'), \n",
    "            pl.col('struct').struct.field('b')\n",
    "            ])\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00530a6d",
   "metadata": {},
   "source": [
    "## Examples\n",
    "\n",
    "\n",
    "\n",
    "### Verifying Multi-Column Uniques (hashing)\n",
    "\n",
    "The `.expr().unique()` method does not accept multiple columns.\n",
    "We can use a struct to group the columns, and then apply the single-column `.unique()` method.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "multi-column-uniques",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (6, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>unique_ind</th><th>hash</th></tr><tr><td>bool</td><td>u64</td></tr></thead><tbody><tr><td>true</td><td>7275562222371371866</td></tr><tr><td>true</td><td>5001730839500661466</td></tr><tr><td>true</td><td>214433202675950699</td></tr><tr><td>true</td><td>3276922812139959675</td></tr><tr><td>true</td><td>594759630009917312</td></tr><tr><td>true</td><td>12464129093563214397</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (6, 2)\n",
       "┌────────────┬──────────────────────┐\n",
       "│ unique_ind ┆ hash                 │\n",
       "│ ---        ┆ ---                  │\n",
       "│ bool       ┆ u64                  │\n",
       "╞════════════╪══════════════════════╡\n",
       "│ true       ┆ 7275562222371371866  │\n",
       "│ true       ┆ 5001730839500661466  │\n",
       "│ true       ┆ 214433202675950699   │\n",
       "│ true       ┆ 3276922812139959675  │\n",
       "│ true       ┆ 594759630009917312   │\n",
       "│ true       ┆ 12464129093563214397 │\n",
       "└────────────┴──────────────────────┘"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| label: multi-column-uniques\n",
    "\n",
    "(\n",
    "    df\n",
    "    .select(\n",
    "        pl.struct(['a','b']).alias('struct')\n",
    "        )\n",
    "    .select(\n",
    "        pl.col('struct').is_unique().alias('unique_ind'),\n",
    "        pl.col('struct').hash().alias('hash'),\n",
    "        )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41803fbd",
   "metadata": {},
   "source": [
    "# Discussion\n",
    "\n",
    "\n",
    "### Q: Can I have a list of list of lists (i.e. more than 2 layers of nesting)?  \n",
    "\n",
    "Yes. \n",
    "\n",
    "### Can I use list methods on a struct?(e.g. argmax)  \n",
    "\n",
    "No. But you can extract the struct to a list, and then use list methods.\n",
    "\n",
    "\n",
    "### Q: When a Polars List and when a set of columns?\n",
    "\n",
    "The difference is mostly syntactic. So a matter of preference and convenience. See the following example. The same goes for Polars arrays. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "daeaec01",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a frame with lists of random length\n",
    "\n",
    "max_length = int(1e6)\n",
    "\n",
    "def make_list():\n",
    "    \n",
    "    result = pl.Series(list(string.ascii_letters)).sample(random.randint(1, max_length), with_replacement=True)\n",
    "    \n",
    "    return result\n",
    "\n",
    "# make_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "9bdc20c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pl.DataFrame(\n",
    "    {\n",
    "        \"a\": [make_list(), make_list(), make_list()],\n",
    "        \"b\": [\"a\", \"b\", \"c\"]\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "a27c3e6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3414945602416992"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.estimated_size(unit='mb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "3a800a25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3414688110351562"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2 = df.select(pl.col('a').list.to_struct(n_field_strategy='max_width')).unnest(\"a\")\n",
    "df_2.estimated_size(unit='mb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "7e01bebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %timeit df_2.select(pl.col('a').str.len_chars())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a9caf92",
   "metadata": {},
   "source": [
    "### Can I export to CSV? To Parquet? \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "f7a76d16",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| eval: false\n",
    "# get tmp file name from operating system\n",
    "import tempfile\n",
    "temp_file = os.path.join(tempfile._get_default_tempdir(), 'something.csv')\n",
    "\n",
    "df_with_list.write_csv('df_with_list.csv')\n",
    "df_with_list.write_parquet('df_with_list.parquet')\n",
    "pl.read_parquet('df_with_list.parquet').schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7784e74",
   "metadata": {},
   "source": [
    "What is the dtype when I export to Pandas?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "export-to-pandas",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6 entries, 0 to 5\n",
      "Data columns (total 3 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   a       6 non-null      int64 \n",
      " 1   b       6 non-null      int64 \n",
      " 2   ab      6 non-null      object\n",
      "dtypes: int64(2), object(1)\n",
      "memory usage: 276.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "#| label: export-to-pandas\n",
    "\n",
    "df_with_list_pandas = df_with_list.to_pandas()\n",
    "df_with_list_pandas.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b173f75",
   "metadata": {},
   "source": [
    "### Q: What is the namespace for a list of lists?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}