pandas_dataframe_descriptive_stats.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:1f38542507632b48e0a9fdab8810d0c34f3c692ba1a577ecad2d29d4398dd285"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Descriptive Statistics For pandas Dataframe\n",
      "\n",
      "This snippit was written by [Chris R. Albon](http://www.chrisralbon.com/) and is part of his collection of [well-documented Python snippits](https://github.com/chrisalbon/code_py). All code is written in Python 3 in iPython notebook and offered under the [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### import modules"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 40
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Create dataframe"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "data = {'name': ['Jason', 'Molly', 'Tina', 'Jake', 'Amy'], \n",
      "        'age': [42, 52, 36, 24, 73], \n",
      "        'preTestScore': [4, 24, 31, 2, 3],\n",
      "        'postTestScore': [25, 94, 57, 62, 70]}\n",
      "df = pd.DataFrame(data, columns = ['name', 'age', 'preTestScore', 'postTestScore'])\n",
      "df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>name</th>\n",
        "      <th>age</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> Jason</td>\n",
        "      <td> 42</td>\n",
        "      <td>  4</td>\n",
        "      <td> 25</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Molly</td>\n",
        "      <td> 52</td>\n",
        "      <td> 24</td>\n",
        "      <td> 94</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  Tina</td>\n",
        "      <td> 36</td>\n",
        "      <td> 31</td>\n",
        "      <td> 57</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  Jake</td>\n",
        "      <td> 24</td>\n",
        "      <td>  2</td>\n",
        "      <td> 62</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>   Amy</td>\n",
        "      <td> 73</td>\n",
        "      <td>  3</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 4 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 41,
       "text": [
        "    name  age  preTestScore  postTestScore\n",
        "0  Jason   42             4             25\n",
        "1  Molly   52            24             94\n",
        "2   Tina   36            31             57\n",
        "3   Jake   24             2             62\n",
        "4    Amy   73             3             70\n",
        "\n",
        "[5 rows x 4 columns]"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### The sum of all the ages"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['age'].sum()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 42,
       "text": [
        "227"
       ]
      }
     ],
     "prompt_number": 42
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Mean preTestScore"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 43,
       "text": [
        "12.800000000000001"
       ]
      }
     ],
     "prompt_number": 43
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Cumulative sum of preTestScores, moving from the rows from the top"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].cumsum()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 44,
       "text": [
        "0     4\n",
        "1    28\n",
        "2    59\n",
        "3    61\n",
        "4    64\n",
        "Name: preTestScore, dtype: int64"
       ]
      }
     ],
     "prompt_number": 44
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Summary statistics on preTestScore"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].describe()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 45,
       "text": [
        "count     5.000000\n",
        "mean     12.800000\n",
        "std      13.663821\n",
        "min       2.000000\n",
        "25%       3.000000\n",
        "50%       4.000000\n",
        "75%      24.000000\n",
        "max      31.000000\n",
        "Name: preTestScore, dtype: float64"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Count the number of non-NA values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].count()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 46,
       "text": [
        "5"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Minimum value of preTestScore"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].min()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 47,
       "text": [
        "2"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Maximum value of preTestScore"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].max()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 48,
       "text": [
        "31"
       ]
      }
     ],
     "prompt_number": 48
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Median value of preTestScore"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].median()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 49,
       "text": [
        "4.0"
       ]
      }
     ],
     "prompt_number": 49
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Sample variance of preTestScore values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].var()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 50,
       "text": [
        "186.69999999999999"
       ]
      }
     ],
     "prompt_number": 50
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Sample standard deviation of preTestScore values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].std()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 51,
       "text": [
        "13.663820841916802"
       ]
      }
     ],
     "prompt_number": 51
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Skewness of preTestScore values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].skew()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 52,
       "text": [
        "0.74334524573267591"
       ]
      }
     ],
     "prompt_number": 52
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Kurtosis of preTestScore values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['preTestScore'].kurt()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 53,
       "text": [
        "-2.4673543738411525"
       ]
      }
     ],
     "prompt_number": 53
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Correlation Matrix Of Values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.corr()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>age</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>age</th>\n",
        "      <td> 1.000000</td>\n",
        "      <td>-0.105651</td>\n",
        "      <td> 0.328852</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>preTestScore</th>\n",
        "      <td>-0.105651</td>\n",
        "      <td> 1.000000</td>\n",
        "      <td> 0.378039</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>postTestScore</th>\n",
        "      <td> 0.328852</td>\n",
        "      <td> 0.378039</td>\n",
        "      <td> 1.000000</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>3 rows \u00d7 3 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 54,
       "text": [
        "                    age  preTestScore  postTestScore\n",
        "age            1.000000     -0.105651       0.328852\n",
        "preTestScore  -0.105651      1.000000       0.378039\n",
        "postTestScore  0.328852      0.378039       1.000000\n",
        "\n",
        "[3 rows x 3 columns]"
       ]
      }
     ],
     "prompt_number": 54
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Covariance Matrix Of Values"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.cov()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>age</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>age</th>\n",
        "      <td> 340.80</td>\n",
        "      <td> -26.65</td>\n",
        "      <td> 151.20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>preTestScore</th>\n",
        "      <td> -26.65</td>\n",
        "      <td> 186.70</td>\n",
        "      <td> 128.65</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>postTestScore</th>\n",
        "      <td> 151.20</td>\n",
        "      <td> 128.65</td>\n",
        "      <td> 620.30</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>3 rows \u00d7 3 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 55,
       "text": [
        "                  age  preTestScore  postTestScore\n",
        "age            340.80        -26.65         151.20\n",
        "preTestScore   -26.65        186.70         128.65\n",
        "postTestScore  151.20        128.65         620.30\n",
        "\n",
        "[3 rows x 3 columns]"
       ]
      }
     ],
     "prompt_number": 55
    }
   ],
   "metadata": {}
  }
 ]
}