The good and bad of Economy analysis (post ICA).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preperation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure IPython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# activate plots\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pandas'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-a7c3cf0182a3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mscripts\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0manalyze\u001b[0m \u001b[0;31m# some analysis helper functions for this paper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/code/economic_sentiment/scripts/analyze.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msavReaderWriter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pandas'"
     ]
    }
   ],
   "source": [
    "from scripts import analyze # some analysis helper functions for this paper\n",
    "import pandas \n",
    "import datetime\n",
    "import os\n",
    "import numpy\n",
    "import statsmodels.formula.api as sm\n",
    "import seaborn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prepare some visualization stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "corplot = lambda cortable, title: seaborn.heatmap(cortable, linewidth=.1, annot=True, cmap=\"RdBu_r\", vmin=-1, vmax=1).set_title(title)\n",
    "tool_order = [ \"recessie\" , \"boukes\",\"LIWC\",\"sentistrength\",\"pattern\",\"polyglot\",\"DANEW\"]\n",
    "text_order = [\"text_\"+tool for tool in tool_order]\n",
    "title_order = [\"title_\"+tool for tool in tool_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_files = \"./data\"\n",
    "coded_files   = \"./results/data_with_sentiment.csv\"\n",
    "os.makedirs('results',exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do or Load Sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If sentiments have not yet been computed\n",
    "if not os.path.exists(coded_files):\n",
    "    print(\"Starting from scratch\")\n",
    "    raw_data = analyze.load_files(path_to_files)\n",
    "    print(\"Recoding\")\n",
    "    rec_data = analyze.recode_annotations(raw_data) \n",
    "    print(\"Adding sentiment\")\n",
    "    sen_data = analyze.add_sentiments(analyze.add_sentiments(rec_data,'text'),'title')\n",
    "    print(\"Adding linguistic features for error analysis\")\n",
    "    tex_data = analyze.add_text_properties(analyze.add_text_properties(sen_data,'text'),'title')\n",
    "    print(\"Writing results to disk\")\n",
    "    tex_data.to_csv(coded_files)\n",
    "    data = tex_data\n",
    "# If sentiments have been computed\n",
    "else:\n",
    "    print(\"Using previously stored computations\")\n",
    "    data = pandas.read_csv(coded_files)\n",
    "\n",
    "data.index = data.ID\n",
    "\n",
    "# Add LIWC results if available\n",
    "if \"LIWC2015 Results (ID_Text.csv).csv\" in os.listdir(path_to_files):\n",
    "    text_liwc = pandas.read_csv(os.path.join(path_to_files,\"LIWC2015 Results (ID_Text.csv).csv\"))\n",
    "    text_liwc = pandas.DataFrame({'ID':text_liwc.A, 'text_LIWC':text_liwc.Posemo - text_liwc.Negemo}).loc[2:,:]\n",
    "    text_liwc.index = text_liwc.ID.astype(\"float64\")\n",
    "    data = data.join(text_liwc.drop('ID',1))\n",
    "    \n",
    "if \"LIWC2015 Results (ID_Title.csv).csv\" in os.listdir(path_to_files):\n",
    "    title_liwc = pandas.read_csv(os.path.join(path_to_files,\"LIWC2015 Results (ID_Title.csv).csv\"))\n",
    "    title_liwc = pandas.DataFrame({'ID':title_liwc.A, 'title_LIWC':title_liwc.Posemo - title_liwc.Negemo}).loc[2:,:]\n",
    "    title_liwc.index = title_liwc.ID.astype(\"float64\")\n",
    "    data = data.join(title_liwc.drop('ID',1))\n",
    "    \n",
    "# Set time-based index\n",
    "todate = lambda d: datetime.datetime.strptime(d.replace('T',' ').split('.')[0],'%Y-%m-%d %H:%M:%S')\n",
    "data.index = data.date_y.map(todate) # saving as CSV breaks the timeindex, so we reconstruct it here\n",
    "    \n",
    "\n",
    "# Filter out timepoints after which data becomes too sparse\n",
    "filtered = (data.index >= datetime.datetime(year=2015,month=7,day=7)).sum()\n",
    "data = data[data.index < datetime.datetime(year=2015,month=7,day=7)] \n",
    "print(\"filtered out\",filtered, \"observations\")\n",
    "\n",
    "data = data[~data.index.dayofweek.isin([0,6])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## order columns appropriately"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.index = range(len(data))\n",
    "red = data.copy()\n",
    "data = data.drop(text_order,axis=1)\n",
    "data = data.drop(title_order,axis=1)\n",
    "data = data.join(red[text_order])\n",
    "data = data.join(red[title_order])\n",
    "del(red)\n",
    "data.index = data.index = data.date_y.map(todate) # saving as CSV breaks the timeindex, so we reconstruct it here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option to toggle non-relevant human annotations to '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data.text_gold = data.text_gold.replace({numpy.nan:0})\n",
    "#data.title_gold = data.title_gold.replace({numpy.nan:0})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resulting crosstabs for gold annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Fulltext crosstab\")\n",
    "pandas.DataFrame(\n",
    "    {\n",
    "        'online':data.text_gold[data.online].value_counts(),\n",
    "        'online_%': round(data.text_gold[data.online].value_counts()/data.text_gold[data.online].value_counts().sum(),2),\n",
    "        'offline':data.text_gold[~data.online].value_counts(),\n",
    "        'offline_%': round(data.text_gold[~data.online].value_counts()/data.text_gold[~data.online].value_counts().sum(),2),\n",
    "        'total': data.text_gold.value_counts()\n",
    "    }\n",
    ").sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Headline crosstab\")\n",
    "pandas.DataFrame(\n",
    "    {\n",
    "        'online':data.title_gold[data.online].value_counts(),\n",
    "        'online_%': round(data.title_gold[data.online].value_counts()/data.title_gold[data.online].value_counts().sum(),2),\n",
    "        'offline':data.title_gold[~data.online].value_counts(),\n",
    "        'offline_%': round(data.title_gold[~data.online].value_counts()/data.title_gold[~data.online].value_counts().sum(),2),\n",
    "        'total': data.title_gold.value_counts()\n",
    "    }\n",
    ").sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing fleiss-scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations = pandas.read_csv(os.path.join(path_to_files, \"Inhoudsanalyse_AllesMerged_noICR_toneOnly.csv\"),delimiter=\";\")\n",
    "annotations = annotations.replace({\" \":numpy.nan})\n",
    "#annotations[[\"Toon_Kop\",\"Posit_Nega\"]] = annotations[[\"Toon_Kop\",\"Posit_Nega\"]].astype(float)\n",
    "\n",
    "\n",
    "print(\"Headline Fleiss score, N: \", analyze.calculate_intercoder_reliability(annotations, \"ID\",\"Codeur\",\"Toon_Kop\"))\n",
    "print(\"Fulltext Fleiss score, N: \", analyze.calculate_intercoder_reliability(annotations, \"ID\",\"Codeur\",\"Posit_Nega\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add 'the best of the best' models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = analyze.z_best(data, 'title', list(reversed([\"LIWC\",\"polyglot\",\"sentistrength\",\"DANEW\"])))\n",
    "data = analyze.z_best(data, 'text', list(reversed([\"LIWC\",\"polyglot\",\"sentistrength\",\"DANEW\"])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agregations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_data = data.copy()\n",
    "daily_data   = data.copy().resample(\"1D\").mean()\n",
    "weekly_data  = data.copy().resample(\"1W\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_data.resample(\"1D\").count().date_y.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# article level errors\n",
    "article_text_errors  = analyze.calculate_errors(article_data,'text')\n",
    "article_title_errors = analyze.calculate_errors(article_data,'title')\n",
    "# daily level errors\n",
    "daily_text_errors  = analyze.calculate_errors(daily_data, 'text')\n",
    "daily_title_errors = analyze.calculate_errors(daily_data, 'title')\n",
    "# weekly level errors\n",
    "weekly_text_errors  = analyze.calculate_errors(weekly_data, 'text')\n",
    "weekly_title_errors = analyze.calculate_errors(weekly_data, 'title')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictors of title absolute errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.analyze_errors(article_title_errors,'title')[tool_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictors of text absolute errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.analyze_errors(article_data,'text')[tool_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Article-level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(article_data.loc[:,[\"title_gold\"]+title_order],field='title')\n",
    "plot = standardized.drop('ID',1).plot(kind='box',figsize=(10,15),title=\"Boxplot of headline sentiment scores at the Article level\")\n",
    "plot.set_xticklabels(['Human\\nAnnotation','Recession','Damstra &\\nBoukes','LIWC','Sentistrength','Pattern','Polyglot','DANEW'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_data.loc[:,[\"title_gold\"]+title_order].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import kstest,kurtosis,skew\n",
    "print(\"Online\")\n",
    "for col in text_order:\n",
    "    k,pval = kstest(article_data[article_data.online==True][col].dropna(),cdf='norm')\n",
    "    kurt   = kurtosis(article_data[article_data.online==True][col].dropna())\n",
    "    sk     = skew(article_data[article_data.online==True][col].dropna())\n",
    "    print(\"{:20.20s}: D={:0.3f} pval={:0.3f} kurtosis={:6.2f} skew={:6.2f}\".format(col,k,pval,kurt,sk))\n",
    "print(\"Offline\")\n",
    "for col in text_order:\n",
    "    k,pval = kstest(article_data[article_data.online==False][col].dropna(),cdf='norm')\n",
    "    kurt   = kurtosis(article_data[article_data.online==False][col].dropna())\n",
    "    sk     = skew(article_data[article_data.online==False][col].dropna())\n",
    "    print(\"{:20.20s}: D={:0.3f} pval={:0.3f} kurtosis={:6.2f} skew={:6.2f}\".format(col,k,pval,kurt,sk))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.violinplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt())\n",
    "seaborn.stripplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt(),color=\".3\",jitter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(article_data.loc[:,[\"text_gold\"]+text_order],field='text')\n",
    "plot=standardized.drop('ID',1).plot(\n",
    "    kind='box',\n",
    "    figsize=(10,15),\n",
    "    title=\"Boxplot of fulltext sentiment scores at the Article level\"\n",
    "    )\n",
    "plot.set_xticklabels(['Human\\nAnnotation','Recession','Damstra &\\nBoukes','LIWC','Sentistrength','Pattern','Polyglot','DANEW'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(article_data[article_data.online].loc[:,[\"text_gold\"]+text_order],field='text')\n",
    "plot=standardized.drop('ID',1).plot(\n",
    "    kind='box',\n",
    "    figsize=(10,10),\n",
    "    title=\"Boxplot of online text sentiment scores at the Article level\",\n",
    "    )\n",
    "plot.set_xticklabels(['Manual\\nAnnotation','Recession','Damstra &\\nBoukes','LIWC','Sentistrength','Pattern','Polyglot','DANEW'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(article_data[~article_data.online].loc[:,[\"text_gold\"]+text_order],field='text')\n",
    "plot=standardized.drop('ID',1).plot(\n",
    "    kind='box',\n",
    "    figsize=(10,10),\n",
    "    title=\"Boxplot of offline text sentiment scores at the Article level\"\n",
    "    )\n",
    "plot.set_xticklabels(['Manual\\nAnnotation','Recession','Damstra &\\nBoukes','LIWC','Sentistrength','Pattern','Polyglot','DANEW'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.violinplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt())\n",
    "seaborn.stripplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt(),color=\".3\",jitter=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality of models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.check_quality(article_data,'title').loc[title_order,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.check_quality(article_data,'text').loc[text_order,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlations of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data,'title').loc[['title_gold']+title_order,['title_gold']+title_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data,'text').loc[['text_gold']+text_order,['text_gold']+text_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### article-level correlations in online publications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data[article_data.online],'title').loc[['title_gold']+title_order,['title_gold']+title_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_data.title_recessie[article_data.online].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### article-level correlations in offline publications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data[~article_data.online],'title').loc[['title_gold']+title_order,['title_gold']+title_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(article_data.drop(\"title_recessie\",axis=1),'title'),\"Correlation heatmap of title sentiment scores at the article level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data,'text').loc[['text_gold']+text_order,['text_gold']+text_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Article-level body online correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data[article_data.online],'text').loc[['text_gold']+text_order,['text_gold']+text_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Article-level body offline correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data[~article_data.online],'text').loc[['text_gold']+text_order,['text_gold']+text_order]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "online_full  = analyze.correlation_tests(article_data[article_data.online],'text').loc[['text_gold']+text_order,['text_gold']+text_order].text_gold\n",
    "online_head  = analyze.correlation_tests(article_data[article_data.online],'title').loc[['title_gold']+title_order,['title_gold']+title_order].title_gold\n",
    "offline_full = analyze.correlation_tests(article_data[~article_data.online],'text').loc[['text_gold']+text_order,['text_gold']+text_order].text_gold\n",
    "offline_head = analyze.correlation_tests(article_data[~article_data.online],'title').loc[['title_gold']+title_order,['title_gold']+title_order].title_gold\n",
    "overall_full = analyze.correlation_tests(article_data,'text').loc[['text_gold']+text_order,['text_gold']+text_order].text_gold\n",
    "overall_head = analyze.correlation_tests(article_data,'title').loc[['title_gold']+title_order,['title_gold']+title_order].title_gold\n",
    "\n",
    "online_full.index = [\"Human\"] + tool_order\n",
    "offline_full.index = [\"Human\"] + tool_order\n",
    "offline_head.index = [\"Human\"] + tool_order\n",
    "online_head.index = [\"Human\"] + tool_order\n",
    "overall_head.index = [\"Human\"] + tool_order\n",
    "overall_full.index = [\"Human\"] + tool_order\n",
    "\n",
    "compared_cors = pandas.DataFrame({\n",
    "        \"online_fulltext\" : online_full,\n",
    "        \"online_headline\" : online_head,\n",
    "        \"offline_fulltext\": offline_full,\n",
    "        \"offline_headline\": offline_head,\n",
    "        \"all_fulltext\"    : overall_full,\n",
    "        \"all_headline\"    : overall_head\n",
    "    })\n",
    "compared_cors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_h, av_f = [], []\n",
    "for classifier, headline, fulltext in zip(compared_cors.index, compared_cors.all_headline,compared_cors.all_fulltext):\n",
    "    to_num = lambda x: float(x.replace('*','').strip())\n",
    "    c_compare = analyze.cor_compare(to_num(headline), to_num(fulltext), len(data[article_data.text_gold.isnull()]), len(data[article_data.title_gold.isnull()]))\n",
    "    av_h.append(to_num(headline))\n",
    "    av_f.append(to_num(fulltext))\n",
    "    print(classifier, round(c_compare['cordiff'],2), c_compare['p_value'])\n",
    "print(\"average\", numpy.nanmean(av_h),numpy.nanmean(av_f),analyze.cor_compare(numpy.nanmean(av_h),numpy.nanmean(av_f), len(data[article_data.text_gold.isnull()]), len(data[article_data.title_gold.isnull()]))['p_value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.heatmap(analyze.correlate_results(article_data,'text'), linewidth=.1, annot=True,cmap='RdBu_r',vmin=-1).set_title(\n",
    "\"Correlation heatmap of text sentiment scores at the article level\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlations of errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data, 'title', errors=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(article_data,'title',errors=True),\"Heatmap of Article-level title error correlation coefficients\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(article_data, 'text',errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(article_data,'text',errors=True),\"Heatmap of Article-level text error correlation coefficients\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Daily-level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(daily_data,field='title')\n",
    "standardized.drop('ID',1).plot(kind='box',figsize=(10,10),title=\"Boxplot of title sentiment scores at the Daily level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.violinplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt())\n",
    "seaborn.stripplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt(),color=\".3\",jitter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(daily_data,field='text')\n",
    "standardized.drop('ID',1).plot(kind='box',figsize=(10,10),title=\"Boxplot of text sentiment scores at the Daily level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.violinplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt())\n",
    "seaborn.stripplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt(),color=\".3\",jitter=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality of models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.check_quality(daily_data,'title').loc[title_order,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.check_quality(daily_data,'text').loc[text_order,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlations of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(daily_data,'title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(daily_data.drop(\"title_recessie\",axis=1),'title'),\"Correlation heatmap of title sentiment scores at the daily level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(daily_data,'text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(daily_data,'text'),\"Correlation heatmap of text sentiment scores at the article level\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlations of errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(daily_data, 'title', errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(daily_data,'title',errors=True),\"Heatmap of Daily-level title error correlation coefficients\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(daily_data, 'text',errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(daily_data,'text',errors=True),\"Heatmap of Daily-level text error correlation coefficients\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weekly-level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(weekly_data,field='title')\n",
    "standardized.drop('ID',1).plot(kind='box',figsize=(10,10),title=\"Boxplot of title sentiment scores at the week level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.violinplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt())\n",
    "seaborn.stripplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt(),color=\".3\",jitter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "standardized = analyze.compare_sentiment_means(weekly_data,field='text')\n",
    "standardized.drop('ID',1).plot(kind='box',figsize=(10,10),title=\"Boxplot of text sentiment scores at the Week level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.violinplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt())\n",
    "seaborn.stripplot(x=\"variable\", y=\"value\",data=standardized.drop('ID',1).melt(),color=\".3\",jitter=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality of models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.check_quality(weekly_data,'title').loc[title_order,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.check_quality(weekly_data,'text').loc[text_order+[\"text_top3\"],:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlations of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(weekly_data,'title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(weekly_data.drop(\"title_recessie\",axis=1),'title'),\"Correlation heatmap of title sentiment scores at the Week level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(weekly_data,'text').loc[['text_gold']+text_order,[\"text_gold\"]+text_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.heatmap(analyze.correlate_results(weekly_data,'text'), linewidth=.1, annot=True,cmap='RdBu_r',vmin=-1).set_title(\n",
    "\"Correlation heatmap of text sentiment scores at the Week level\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlations of errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(weekly_data, 'title', errors=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(weekly_data,'title',errors=True),\"Heatmap of Week-level title error correlation coefficients\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.correlation_tests(weekly_data, 'text',errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corplot(analyze.correlate_results(weekly_data,'text',errors=True),\"Heatmap of Week-level text error correlation coefficients\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Method correlations with  baseline compared across granularities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.mean_correlations(data.loc[:,[\"title_\"+t for t in tool_order]+[\"title_gold\",\"title_top3\"]].dropna(),'title').loc[title_order+['title_top3','N'],:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze.mean_correlations(data.loc[:,[\"text_\"+t for t in tool_order]+[\"text_gold\",\"text_top3\"]].dropna(),'text').loc[text_order+['text_top3','N'],:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#min_max_norm = lambda x: (x-x.median())/(x.max()-x.min())\n",
    "weekly_data[[col for col in weekly_data.columns if \"text_\" in col and \"_err\" in col and not \"text_DANEW\" in col ]].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weekly_data[[col for col in weekly_data.columns if \"text_\" in col and not \"_err\" in col and not \"text_DANEW\" in col ]].plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Boukes bonus bonanza"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Predicting complexity of titles\")\n",
    "analyze.analyze_complexity(article_data, field='title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Predicting complexity of texts\")\n",
    "analyze.analyze_complexity(article_data, field='text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}