example_usage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef9a6f86",
   "metadata": {},
   "source": [
    "# Example usage of the Feature Catalog API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b32cf810",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "from feature_catalog.main import compute_features\n",
    "from feature_catalog.feature_groups.zone import Zone\n",
    "from feature_catalog.feature_groups.zone_likelyhood import ZoneLikelyhood\n",
    "\n",
    "spark = SparkSession.builder.getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8700d01",
   "metadata": {},
   "source": [
    "# Basic usage\n",
    "\n",
    "Define the features you want to compute and call the `compute_features` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ddadef38",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (91064, 4)\n",
      "   avatarId  darkshore_count  total_count  darkshore_likelyhood\n",
      "0         0                0           76                   0.0\n",
      "1         1                0           19                   0.0\n",
      "2         2                0          192                   0.0\n",
      "3         3                0          109                   0.0\n",
      "4         4                0           76                   0.0\n"
     ]
    }
   ],
   "source": [
    "aggregation_level = \"avatarId\"\n",
    "scope = spark.read.parquet(\"data/wow.parquet\").select(aggregation_level).distinct()\n",
    "\n",
    "features = compute_features(\n",
    "    spark=spark,\n",
    "    scope=scope, \n",
    "    feature_groups=[\n",
    "        Zone(features_of_interest=[\"darkshore_count\", \"total_count\"], aggregation_level=aggregation_level),\n",
    "        ZoneLikelyhood(features_of_interest=[\"darkshore_likelyhood\"], aggregation_level=aggregation_level),\n",
    "    ]).toPandas()\n",
    "    \n",
    "print(\"shape: \", features.shape)\n",
    "print(features.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06da437d",
   "metadata": {},
   "source": [
    "### You can compute features on different aggregation levels\n",
    "\n",
    "As long as the feature group supports it you can compute features on different aggregation levels.\n",
    "One call to the feature catalog (with `compute_features`) will always only be on one level since it makes sense to compute your features on a specific level.\n",
    "\n",
    "See example below.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ddadef38",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (91064, 2)\n",
      "   avatarId  darkshore_likelyhood\n",
      "0         0                   0.0\n",
      "1         1                   0.0\n",
      "2         2                   0.0\n",
      "3         3                   0.0\n",
      "4         4                   0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 26:===================================================>     (9 + 1) / 10]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (514, 2)\n",
      "   guild  darkshore_likelyhood\n",
      "0     -1              0.000226\n",
      "1      0              0.000084\n",
      "2      1              0.000209\n",
      "3      2              0.000166\n",
      "4      3              0.000163\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "for aggregation_level in [\"avatarId\", \"guild\"]:\n",
    "    scope = spark.read.parquet(\"data/wow.parquet\").select(aggregation_level).distinct()\n",
    "\n",
    "    features = compute_features(\n",
    "        spark=spark,\n",
    "        scope=scope, \n",
    "        feature_groups=[\n",
    "            ZoneLikelyhood(features_of_interest=[\"darkshore_likelyhood\"], aggregation_level=aggregation_level),\n",
    "        ]).toPandas()\n",
    "        \n",
    "    print(\"shape: \", features.shape)\n",
    "    print(features.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d21496b9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "37de517f4bc755360273b7f4e9f819081f746be3118b413e197b9349e0e02bb8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}