YSDA_Dimitri_Code-03.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:035a74e7cd708012b85bd4585fcdc8f28a8faa7bf000a6fb773ac77b3a35f6f3"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Based on code published by Dimitri, YSDA\n",
      "@ morph, for the YSDA ML Trainings 18 October, 2014"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Big Data -- Sample Data!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train = pd.read_csv('train.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#print train\n",
      "train.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "(1700000, 146)"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sample_size = 500000\n",
      "ratio = train.shape[0] / sample_size\n",
      "\n",
      "train_sample = train[\n",
      "    [hash(id) % ratio == 0 for id in train['id']]\n",
      "]\n",
      "\n",
      "train_sample.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "(566666, 146)"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_sample.to_csv('train_sample.csv', index = False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Free memory\n",
      "\n",
      "del train"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 25
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Try to make something useful"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_sample = pd.read_csv('train_sample.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "labels = pd.read_csv('trainLabels.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "labels.columns"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "Index([u'id', u'y1', u'y2', u'y3', u'y4', u'y5', u'y6', u'y7', u'y8', u'y9', u'y10', u'y11', u'y12', u'y13', u'y14', u'y15', u'y16', u'y17', u'y18', u'y19', u'y20', u'y21', u'y22', u'y23', u'y24', u'y25', u'y26', u'y27', u'y28', u'y29', u'y30', u'y31', u'y32', u'y33'], dtype='object')"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_with_labels = pd.merge(train_sample, labels, on = 'id')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_with_labels.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "(566666, 179)"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import Counter\n",
      "\n",
      "Counter([name[0] for name in train_with_labels.columns])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "Counter({'x': 145, 'y': 33, 'i': 1})"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "del labels\n",
      "del train_sample"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test = pd.read_csv('test.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Categorical values encoding"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.feature_extraction import DictVectorizer\n",
      "import numpy as np\n",
      "\n",
      "X_numerical = []\n",
      "X_test_numerical = []\n",
      "\n",
      "vec = DictVectorizer()\n",
      "\n",
      "names_categorical = []\n",
      "\n",
      "train_with_labels.replace('YES', 1, inplace = True)\n",
      "train_with_labels.replace('NO', 0, inplace = True)\n",
      "train_with_labels.replace('nan', np.NaN, inplace = True)\n",
      "\n",
      "test.replace('YES', 1, inplace = True)\n",
      "test.replace('NO', 0, inplace = True)\n",
      "test.replace('nan', np.NaN, inplace = True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "column_type, _ =max(Counter(map(lambda x: str(type(x)), train_with_labels[train_with_labels.columns[1]])).items(),key=lambda x:x[1])\n",
      "print column_type, _\n",
      "#print train_with_labels"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<type 'numpy.float64'> 566666\n"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for name in train_with_labels.columns :    \n",
      "    if name.startswith('x') :\n",
      "        #Data type guessing (using max type)\n",
      "        column_type, _ = max(Counter(map(lambda x: str(type(x)), train_with_labels[name])).items(), key = lambda x: x[1])\n",
      "        \n",
      "        #print name,column_type, _\n",
      "        \n",
      "        # LOL expression\n",
      "        if column_type == str(str) :\n",
      "            train_with_labels[name] = map(str, train_with_labels[name])\n",
      "            test[name] = map(str, test[name])\n",
      "\n",
      "            names_categorical.append(name)\n",
      "            print name, len(np.unique(train_with_labels[name]))\n",
      "        else :\n",
      "            #May have to something other than -999\n",
      "            X_numerical.append(train_with_labels[name].fillna(-999))\n",
      "            X_test_numerical.append(test[name].fillna(-999))\n",
      "        \n",
      "X_numerical = np.column_stack(X_numerical)\n",
      "X_test_numerical = np.column_stack(X_test_numerical)\n",
      "\n",
      "X_sparse = vec.fit_transform(train_with_labels[names_categorical].T.to_dict().values())\n",
      "X_test_sparse = vec.transform(test[names_categorical].T.to_dict().values())\n",
      "\n",
      "print X_numerical.shape, X_sparse.shape, X_test_numerical.shape, X_test_sparse.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "x3 "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "96393\n",
        "x4"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 11854\n",
        "x34"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 114253\n",
        "x35"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 16277\n",
        "x61"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 201727\n",
        "x64"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 117192\n",
        "x65"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 16847\n",
        "x91"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 64314\n",
        "x94"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 87950\n",
        "x95"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 12086\n",
        "(566666L, 135L)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " (566666, 738893) (545082L, 135L) (545082, 738893)\n"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X_numerical = np.nan_to_num(X_numerical)\n",
      "X_test_numerical = np.nan_to_num(X_test_numerical)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.externals import joblib\n",
      "\n",
      "joblib.dump(\n",
      "    (X_numerical, X_sparse, X_test_numerical, X_test_sparse),\n",
      "    'X03.dump',\n",
      "    compress = 1,\n",
      ")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 39,
       "text": [
        "['X03.dump', 'X03.dump_01.npy.z', 'X03.dump_02.npy.z']"
       ]
      }
     ],
     "prompt_number": 39
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Trying to predict something"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Build two level classifier, first train base level"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.metrics import roc_auc_score, f1_score, log_loss, make_scorer\n",
      "from sklearn.svm import LinearSVC\n",
      "from sklearn.cross_validation import cross_val_score, train_test_split\n",
      "from sklearn.ensemble import RandomForestClassifier\n",
      "\n",
      "log_loss_scorer = make_scorer(log_loss, needs_proba = True)\n",
      "\n",
      "y_columns = [name for name in train_with_labels.columns if name.startswith('y')]\n",
      "\n",
      "X_numerical_base, X_numerical_meta, X_sparse_base, X_sparse_meta, y_base, y_meta = train_test_split(\n",
      "    X_numerical, \n",
      "    X_sparse, \n",
      "    train_with_labels[y_columns].values,\n",
      "    test_size = 0.5\n",
      ")\n",
      "\n",
      "X_meta = [] \n",
      "X_test_meta = []\n",
      "\n",
      "print \"Build meta\"\n",
      "\n",
      "for i in range(y_base.shape[1]) :\n",
      "    print i\n",
      "    \n",
      "    y = y_base[:, i]\n",
      "    if len(np.unique(y)) == 2 : \n",
      "        rf = RandomForestClassifier(n_estimators = 10, n_jobs = 1,random_state=0)\n",
      "        rf.fit(X_numerical_base, y)\n",
      "        X_meta.append(rf.predict_proba(X_numerical_meta))\n",
      "        X_test_meta.append(rf.predict_proba(X_test_numerical))\n",
      "\n",
      "        svm = LinearSVC(random_state=1)\n",
      "        svm.fit(X_sparse_base, y)\n",
      "        X_meta.append(svm.decision_function(X_sparse_meta))\n",
      "        X_test_meta.append(svm.decision_function(X_test_sparse))\n",
      "        \n",
      "X_meta = np.column_stack(X_meta)\n",
      "X_test_meta = np.column_stack(X_test_meta)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Build meta\n",
        "0\n",
        "1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "2"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "3"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "4"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "5"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "6"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "7"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "8"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "9"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "10"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "11"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "12"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "13"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "14\n",
        "15"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "16"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "17"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "18"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "19"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "20"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "21"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "22"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "23"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "24"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "25"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "26"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "27"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "28"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "29"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "30"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "31"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "32"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print X_meta.shape, X_test_meta.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(283333L, 96L) (545082L, 96L)\n"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Here train meta level and get predictions for test set"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "p_test = []\n",
      "\n",
      "for i in range(y_base.shape[1]) :\n",
      "    y = y_meta[:, i]\n",
      "\n",
      "    constant = Counter(y)\n",
      "    constant = constant[0] < 4 or constant[1] < 4\n",
      "    \n",
      "    predicted = None\n",
      "    \n",
      "    if constant :\n",
      "        # Best constant\n",
      "        constant_pred = np.mean(list(y_base[:, i]) + list(y_meta[:, i]))\n",
      "        \n",
      "        predicted = np.ones(X_test_meta.shape[0]) * constant_pred\n",
      "        print \"%d is constant like: %f\" % (i, constant_pred)\n",
      "    else :\n",
      "        rf = RandomForestClassifier(n_estimators=50, n_jobs = 1,random_state=2)\n",
      "        rf.fit(np.hstack([X_meta, X_numerical_meta]), y)\n",
      "\n",
      "        predicted = rf.predict_proba(np.hstack([X_test_meta, X_test_numerical]))\n",
      "\n",
      "        predicted = predicted[:, 1]\n",
      "        \n",
      "        rf = RandomForestClassifier(n_estimators=50, n_jobs = 1,random_state=3)\n",
      "        scores = cross_val_score(rf, np.hstack([X_meta, X_numerical_meta]), y, cv = 4, n_jobs = 1, scoring = log_loss_scorer)\n",
      "\n",
      "        #predicted = rf.predict_proba(np.hstack([X_test_meta, X_test_numerical]))\n",
      "\n",
      "        #predicted = predicted[:, 1]        \n",
      "        \n",
      "        print i, 'RF log-loss: %.4f \u00b1 %.4f, mean = %.6f' %(np.mean(scores), np.std(scores), np.mean(predicted))\n",
      "\n",
      "    \n",
      "    p_test.append(\n",
      "        predicted\n",
      "    )\n",
      "    \n",
      "p_test = np.column_stack(p_test)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0 RF log-loss: 0.0055 \u00b1 0.0006, mean = 0.006271\n",
        "1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0020 \u00b1 0.0006, mean = 0.000676\n",
        "2"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0016 \u00b1 0.0002, mean = 0.020808\n",
        "3"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0048 \u00b1 0.0011, mean = 0.013247\n",
        "4"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0016 \u00b1 0.0002, mean = 0.000103\n",
        "5"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0437 \u00b1 0.0023, mean = 0.074442\n",
        "6"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0379 \u00b1 0.0022, mean = 0.038092\n",
        "7"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0107 \u00b1 0.0008, mean = 0.000927\n",
        "8"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0393 \u00b1 0.0006, mean = 0.078572\n",
        "9"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0270 \u00b1 0.0009, mean = 0.017610\n",
        "10"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0004 \u00b1 0.0002, mean = 0.000930\n",
        "11"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0528 \u00b1 0.0011, mean = 0.073221\n",
        "12"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0084 \u00b1 0.0014, mean = 0.014653\n",
        "13 is constant like: 0.000000"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "14"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0021 \u00b1 0.0005, mean = 0.002458\n",
        "15"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0028 \u00b1 0.0005, mean = 0.011000\n",
        "16"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0009 \u00b1 0.0002, mean = 0.000077\n",
        "17"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0006 \u00b1 0.0004, mean = 0.000192\n",
        "18"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0004 \u00b1 0.0002, mean = 0.000664\n",
        "19"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0005 \u00b1 0.0002, mean = 0.002965\n",
        "20"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0021 \u00b1 0.0001, mean = 0.007619\n",
        "21"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0017 \u00b1 0.0002, mean = 0.006374\n",
        "22"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0004 \u00b1 0.0003, mean = 0.000998\n",
        "23"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0066 \u00b1 0.0006, mean = 0.016922\n",
        "24"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0006 \u00b1 0.0003, mean = 0.002447\n",
        "25"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0062 \u00b1 0.0007, mean = 0.011258\n",
        "26"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0022 \u00b1 0.0004, mean = 0.009589\n",
        "27"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0052 \u00b1 0.0016, mean = 0.009922\n",
        "28"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0202 \u00b1 0.0011, mean = 0.031871\n",
        "29"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0126 \u00b1 0.0010, mean = 0.023713\n",
        "30"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0162 \u00b1 0.0005, mean = 0.029738\n",
        "31"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.0181 \u00b1 0.0022, mean = 0.054784\n",
        "32"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " RF log-loss: 0.1180 \u00b1 0.0036, mean = 0.559729\n"
       ]
      }
     ],
     "prompt_number": 42
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Save predictions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "p_test.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 43,
       "text": [
        "(545082L, 33L)"
       ]
      }
     ],
     "prompt_number": 43
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import gzip\n",
      "\n",
      "def save_predictions(name, ids, predictions) :\n",
      "    out = gzip.open(name, 'w')\n",
      "    print >>out, 'id_label,pred'\n",
      "    for id, id_predictions in zip(test['id'], p_test) :\n",
      "        for y_id, pred in enumerate(id_predictions) :\n",
      "            if pred == 0 or pred == 1 :\n",
      "                pred = str(int(pred))\n",
      "            else :\n",
      "                pred = '%.6f' % pred\n",
      "            print >>out, '%d_y%d,%s' % (id, y_id + 1, pred)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "save_predictions('Test_YSDA_03.csv.gz', test['id'].values, p_test)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 45
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Public result"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "566666 training points - 0.0132658"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 45
    }
   ],
   "metadata": {}
  }
 ]
}