coding-01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 描述性统计量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Mean    Median       Min       Max  Std. Dev  Skewness   Kurtosis  \\\n",
      "RET   0.006294  0.009288 -0.291524  0.412291  0.054225  0.327891   9.738663   \n",
      "BM    0.562749  0.535383  0.120510  2.028478  0.265780  0.800912   1.456685   \n",
      "TBL   0.033742  0.029000  0.000100  0.163000  0.030727  1.109733   1.380064   \n",
      "LTY   0.050690  0.041900  0.016300  0.148200  0.027870  1.107032   0.651988   \n",
      "NTIS  0.016446  0.016609 -0.055954  0.177040  0.025873  1.619087   7.896769   \n",
      "INFL  0.002414  0.002402 -0.020548  0.058824  0.005285  1.088402  14.031294   \n",
      "SVAR  0.002838  0.001249  0.000072  0.070945  0.005697  5.843514  44.498046   \n",
      "DP    1.395922  1.197344 -1.148281  4.083117  1.401028  0.112531  -1.216368   \n",
      "DY    0.003975  0.004947 -0.093526  0.058424  0.011108 -2.042930  16.142617   \n",
      "DFR   0.045913  0.038200 -0.107100  0.204700  0.036221  0.889594   2.038943   \n",
      "DFY   0.011219  0.009000  0.003200  0.056400  0.006867  2.516416   9.133416   \n",
      "LTR   0.004777  0.003050 -0.112400  0.152300  0.024544  0.591783   4.478739   \n",
      "EP    2.036378  1.946204 -1.281813  4.976398  1.596745  0.042390  -1.117817   \n",
      "\n",
      "         JB p-value Q(1) p-value Q(10) p-value  \n",
      "RET    0.000000e+00            l             l  \n",
      "BM     1.213197e-47            l             l  \n",
      "TBL    3.041970e-69            l             l  \n",
      "LTY    2.826109e-54            l             l  \n",
      "NTIS   0.000000e+00            l             l  \n",
      "INFL   0.000000e+00            l             l  \n",
      "SVAR   0.000000e+00            l             l  \n",
      "DP     3.565630e-16            l             l  \n",
      "DY     0.000000e+00            l             l  \n",
      "DFR    5.492517e-74            l             l  \n",
      "DFY    0.000000e+00            l             l  \n",
      "LTR   4.381147e-215            l             l  \n",
      "EP     2.006748e-13            l             l  \n"
     ]
    }
   ],
   "source": [
    "# 生成描述性统计量的代码（表1）\n",
    "import pandas as pd\n",
    "from scipy.stats import jarque_bera\n",
    "from statsmodels.stats.diagnostic import acorr_ljungbox\n",
    "# 读取用户上传的更新后的数据文件\n",
    "file_path = '/Users/quanzaixuan/Desktop/Assignment/统计思想pre/data1.csv'\n",
    "df= pd.read_csv(file_path, parse_dates=True, index_col=0)\n",
    "\n",
    "dat = df.apply(pd.to_numeric, errors='coerce')\n",
    "# 初始化结果表格\n",
    "results = pd.DataFrame(columns=['Mean', 'Median', 'Min', 'Max', 'Std. Dev', 'Skewness', 'Kurtosis', 'JB p-value', 'Q(1) p-value', 'Q(10) p-value'])\n",
    "\n",
    "# 计算描述性统计量和检验\n",
    "for col in['RET','BM','TBL','LTY','NTIS','INFL','SVAR','DP','DY','DFR','DFY','LTR','EP']:\n",
    "    series = df[col].dropna()  # 去除缺失值\n",
    "    mean = series.mean()\n",
    "    median = series.median()\n",
    "    min_val = series.min()\n",
    "    max_val = series.max()\n",
    "    std_dev = series.std()\n",
    "    skewness = series.skew()\n",
    "    kurtosis = series.kurtosis()\n",
    "    \n",
    "    # Jarque-Bera 检验\n",
    "    jb_stat, jb_pvalue = jarque_bera(series)\n",
    "    \n",
    "    # Ljung-Box Q 检验\n",
    "    q1_stat, q1_pvalue = acorr_ljungbox(series, lags=[1], return_df=False)  # 滞后长度1\n",
    "    q10_stat, q10_pvalue = acorr_ljungbox(series, lags=[10], return_df=False)  # 滞后长度10\n",
    "    \n",
    "    # 填充结果\n",
    "    results.loc[col] = [mean, median, min_val, max_val, std_dev, skewness, kurtosis, jb_pvalue, q1_pvalue[0], q10_pvalue[0]]\n",
    "\n",
    "# 输出结果\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OLS预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BM</th>\n",
       "      <th>TBL</th>\n",
       "      <th>LTY</th>\n",
       "      <th>NTIS</th>\n",
       "      <th>INFL</th>\n",
       "      <th>SVAR</th>\n",
       "      <th>DP</th>\n",
       "      <th>DY</th>\n",
       "      <th>DFR</th>\n",
       "      <th>DFY</th>\n",
       "      <th>LTR</th>\n",
       "      <th>EP</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1947-01-01</th>\n",
       "      <td>0.016782</td>\n",
       "      <td>0.017252</td>\n",
       "      <td>0.013079</td>\n",
       "      <td>0.017595</td>\n",
       "      <td>0.017773</td>\n",
       "      <td>0.011864</td>\n",
       "      <td>0.028049</td>\n",
       "      <td>0.017595</td>\n",
       "      <td>0.015437</td>\n",
       "      <td>0.013252</td>\n",
       "      <td>0.020787</td>\n",
       "      <td>0.033603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-02-01</th>\n",
       "      <td>-0.019240</td>\n",
       "      <td>-0.018951</td>\n",
       "      <td>-0.023180</td>\n",
       "      <td>-0.018927</td>\n",
       "      <td>-0.018421</td>\n",
       "      <td>-0.025280</td>\n",
       "      <td>-0.002424</td>\n",
       "      <td>-0.018590</td>\n",
       "      <td>-0.022876</td>\n",
       "      <td>-0.022969</td>\n",
       "      <td>-0.017738</td>\n",
       "      <td>0.003815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-03-01</th>\n",
       "      <td>-0.020144</td>\n",
       "      <td>-0.021424</td>\n",
       "      <td>-0.025562</td>\n",
       "      <td>-0.021203</td>\n",
       "      <td>-0.033974</td>\n",
       "      <td>-0.026364</td>\n",
       "      <td>-0.003119</td>\n",
       "      <td>-0.021089</td>\n",
       "      <td>-0.025335</td>\n",
       "      <td>-0.025208</td>\n",
       "      <td>-0.020143</td>\n",
       "      <td>0.005795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-04-01</th>\n",
       "      <td>-0.042288</td>\n",
       "      <td>-0.044362</td>\n",
       "      <td>-0.048090</td>\n",
       "      <td>-0.044082</td>\n",
       "      <td>-0.043841</td>\n",
       "      <td>-0.049411</td>\n",
       "      <td>-0.019093</td>\n",
       "      <td>-0.042761</td>\n",
       "      <td>-0.043482</td>\n",
       "      <td>-0.047914</td>\n",
       "      <td>-0.038214</td>\n",
       "      <td>-0.012380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-05-01</th>\n",
       "      <td>-0.011619</td>\n",
       "      <td>-0.013848</td>\n",
       "      <td>-0.017374</td>\n",
       "      <td>-0.014852</td>\n",
       "      <td>-0.013358</td>\n",
       "      <td>-0.018566</td>\n",
       "      <td>0.007946</td>\n",
       "      <td>-0.012154</td>\n",
       "      <td>-0.018568</td>\n",
       "      <td>-0.017140</td>\n",
       "      <td>-0.013753</td>\n",
       "      <td>0.018284</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  BM       TBL       LTY      NTIS      INFL      SVAR  \\\n",
       "yyyymm                                                                   \n",
       "1947-01-01  0.016782  0.017252  0.013079  0.017595  0.017773  0.011864   \n",
       "1947-02-01 -0.019240 -0.018951 -0.023180 -0.018927 -0.018421 -0.025280   \n",
       "1947-03-01 -0.020144 -0.021424 -0.025562 -0.021203 -0.033974 -0.026364   \n",
       "1947-04-01 -0.042288 -0.044362 -0.048090 -0.044082 -0.043841 -0.049411   \n",
       "1947-05-01 -0.011619 -0.013848 -0.017374 -0.014852 -0.013358 -0.018566   \n",
       "\n",
       "                  DP        DY       DFR       DFY       LTR        EP  \n",
       "yyyymm                                                                  \n",
       "1947-01-01  0.028049  0.017595  0.015437  0.013252  0.020787  0.033603  \n",
       "1947-02-01 -0.002424 -0.018590 -0.022876 -0.022969 -0.017738  0.003815  \n",
       "1947-03-01 -0.003119 -0.021089 -0.025335 -0.025208 -0.020143  0.005795  \n",
       "1947-04-01 -0.019093 -0.042761 -0.043482 -0.047914 -0.038214 -0.012380  \n",
       "1947-05-01  0.007946 -0.012154 -0.018568 -0.017140 -0.013753  0.018284  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combined OLS with rolling window regression for single-variable prediction, store residuals and predictions\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "# Load data\n",
    "data_path = '/Users/quanzaixuan/Desktop/Assignment/统计思想pre/data1.csv'\n",
    "data = pd.read_csv(data_path)\n",
    "# Convert 'yyyymm' to datetime format and set as index\n",
    "data['yyyymm'] = pd.to_datetime(data['yyyymm'])\n",
    "data.set_index('yyyymm', inplace=True)\n",
    "# Define variables and split into train/test datasets\n",
    "variables = ['BM', 'TBL', 'LTY', 'NTIS', 'INFL', 'SVAR', 'DP', 'DY', 'DFR', 'DFY', 'LTR', 'EP']\n",
    "train_data = data.loc['1927-01-01':'1946-12-31']\n",
    "test_data = data.loc['1947-01-01':]\n",
    "# Initialize storage for rolling residuals and predictions\n",
    "rolling_residuals = {var: [] for var in variables}\n",
    "rolling_predictions = {var: [] for var in variables}\n",
    "# Perform rolling OLS regression\n",
    "for var in variables:\n",
    "    for i in range(len(test_data)):\n",
    "        # Define the rolling training data up to period k+i\n",
    "        rolling_train_data = data.iloc[:len(train_data) + i]\n",
    "        X_train = rolling_train_data[[var]]\n",
    "        y_train = rolling_train_data['RET']\n",
    "        # Define the next period (k+1) test data\n",
    "        X_test = test_data[[var]].iloc[i:i+1]\n",
    "        y_test = test_data['RET'].iloc[i:i+1]\n",
    "        # Fit the model\n",
    "        model = LinearRegression()\n",
    "        model.fit(X_train, y_train)\n",
    "        # Predict on test set\n",
    "        y_pred = model.predict(X_test)\n",
    "        # Store the residuals and predictions\n",
    "        residual = y_test.values[0] - y_pred[0]\n",
    "        rolling_residuals[var].append(residual)\n",
    "        rolling_predictions[var].append(y_pred[0])\n",
    "# Convert rolling residuals and predictions to DataFrames\n",
    "rolling_residuals_df = pd.DataFrame(rolling_residuals, index=test_data.index[:len(test_data)])\n",
    "rolling_predictions_df = pd.DataFrame(rolling_predictions, index=test_data.index[:len(test_data)])\n",
    "\n",
    "# Display rolling residuals and predictions\n",
    "rolling_residuals_df.head()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ols_BM_resid</th>\n",
       "      <th>ols_TBL_resid</th>\n",
       "      <th>ols_LTY_resid</th>\n",
       "      <th>ols_NTIS_resid</th>\n",
       "      <th>ols_INFL_resid</th>\n",
       "      <th>ols_SVAR_resid</th>\n",
       "      <th>ols_DP_resid</th>\n",
       "      <th>ols_DY_resid</th>\n",
       "      <th>ols_DFR_resid</th>\n",
       "      <th>ols_DFY_resid</th>\n",
       "      <th>ols_LTR_resid</th>\n",
       "      <th>ols_EP_resid</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1947-01-01</th>\n",
       "      <td>0.016782</td>\n",
       "      <td>0.017252</td>\n",
       "      <td>0.013079</td>\n",
       "      <td>0.017595</td>\n",
       "      <td>0.017773</td>\n",
       "      <td>0.011864</td>\n",
       "      <td>0.028049</td>\n",
       "      <td>0.017595</td>\n",
       "      <td>0.015437</td>\n",
       "      <td>0.013252</td>\n",
       "      <td>0.020787</td>\n",
       "      <td>0.033603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-02-01</th>\n",
       "      <td>-0.019240</td>\n",
       "      <td>-0.018951</td>\n",
       "      <td>-0.023180</td>\n",
       "      <td>-0.018927</td>\n",
       "      <td>-0.018421</td>\n",
       "      <td>-0.025280</td>\n",
       "      <td>-0.002424</td>\n",
       "      <td>-0.018590</td>\n",
       "      <td>-0.022876</td>\n",
       "      <td>-0.022969</td>\n",
       "      <td>-0.017738</td>\n",
       "      <td>0.003815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-03-01</th>\n",
       "      <td>-0.020144</td>\n",
       "      <td>-0.021424</td>\n",
       "      <td>-0.025562</td>\n",
       "      <td>-0.021203</td>\n",
       "      <td>-0.033974</td>\n",
       "      <td>-0.026364</td>\n",
       "      <td>-0.003119</td>\n",
       "      <td>-0.021089</td>\n",
       "      <td>-0.025335</td>\n",
       "      <td>-0.025208</td>\n",
       "      <td>-0.020143</td>\n",
       "      <td>0.005795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-04-01</th>\n",
       "      <td>-0.042288</td>\n",
       "      <td>-0.044362</td>\n",
       "      <td>-0.048090</td>\n",
       "      <td>-0.044082</td>\n",
       "      <td>-0.043841</td>\n",
       "      <td>-0.049411</td>\n",
       "      <td>-0.019093</td>\n",
       "      <td>-0.042761</td>\n",
       "      <td>-0.043482</td>\n",
       "      <td>-0.047914</td>\n",
       "      <td>-0.038214</td>\n",
       "      <td>-0.012380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-05-01</th>\n",
       "      <td>-0.011619</td>\n",
       "      <td>-0.013848</td>\n",
       "      <td>-0.017374</td>\n",
       "      <td>-0.014852</td>\n",
       "      <td>-0.013358</td>\n",
       "      <td>-0.018566</td>\n",
       "      <td>0.007946</td>\n",
       "      <td>-0.012154</td>\n",
       "      <td>-0.018568</td>\n",
       "      <td>-0.017140</td>\n",
       "      <td>-0.013753</td>\n",
       "      <td>0.018284</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ols_BM_resid  ols_TBL_resid  ols_LTY_resid  ols_NTIS_resid  \\\n",
       "yyyymm                                                                   \n",
       "1947-01-01      0.016782       0.017252       0.013079        0.017595   \n",
       "1947-02-01     -0.019240      -0.018951      -0.023180       -0.018927   \n",
       "1947-03-01     -0.020144      -0.021424      -0.025562       -0.021203   \n",
       "1947-04-01     -0.042288      -0.044362      -0.048090       -0.044082   \n",
       "1947-05-01     -0.011619      -0.013848      -0.017374       -0.014852   \n",
       "\n",
       "            ols_INFL_resid  ols_SVAR_resid  ols_DP_resid  ols_DY_resid  \\\n",
       "yyyymm                                                                   \n",
       "1947-01-01        0.017773        0.011864      0.028049      0.017595   \n",
       "1947-02-01       -0.018421       -0.025280     -0.002424     -0.018590   \n",
       "1947-03-01       -0.033974       -0.026364     -0.003119     -0.021089   \n",
       "1947-04-01       -0.043841       -0.049411     -0.019093     -0.042761   \n",
       "1947-05-01       -0.013358       -0.018566      0.007946     -0.012154   \n",
       "\n",
       "            ols_DFR_resid  ols_DFY_resid  ols_LTR_resid  ols_EP_resid  \n",
       "yyyymm                                                                 \n",
       "1947-01-01       0.015437       0.013252       0.020787      0.033603  \n",
       "1947-02-01      -0.022876      -0.022969      -0.017738      0.003815  \n",
       "1947-03-01      -0.025335      -0.025208      -0.020143      0.005795  \n",
       "1947-04-01      -0.043482      -0.047914      -0.038214     -0.012380  \n",
       "1947-05-01      -0.018568      -0.017140      -0.013753      0.018284  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Adjust column names for residuals and predictions\n",
    "\n",
    "# Rename columns for residuals and predictions\n",
    "rolling_residuals_df.columns = [f\"ols_{var}_resid\" for var in variables]\n",
    "rolling_predictions_df.columns = [f\"ols_{var}_pred\" for var in variables]\n",
    "\n",
    "# Display the updated dataframes\n",
    "rolling_residuals_df.head()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 选择最优带宽\n",
    "### 选择时间权重最优带宽"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "# Define the Laplace kernel function\n",
    "def laplace_kernel(u):\n",
    "    return np.exp(-np.abs(u)) / 2\n",
    "\n",
    "# Function to calculate time weights based on Laplace kernel\n",
    "def calculate_time_weights(t, s, h):\n",
    "    u = (t - s) / h\n",
    "    return laplace_kernel(u)\n",
    "\n",
    "# Function to perform weighted regression and return predictions\n",
    "def weighted_regression(X_train, y_train, X_test, time_weights):\n",
    "    model = LinearRegression()\n",
    "    # Apply weights to the regression\n",
    "    model.fit(X_train, y_train, sample_weight=time_weights)\n",
    "    return model.predict(X_test)\n",
    "\n",
    "# Cross-validation to select optimal time bandwidth\n",
    "def cross_validate_time_bandwidth(data, var, bandwidths, p=5):\n",
    "    # Split the dataset into train and test sets\n",
    "    train_data = data.loc['1927-01-01':'1946-12-31']\n",
    "    test_data = data.loc['1947-01-01':]\n",
    "\n",
    "    best_bandwidth = None\n",
    "    best_mse = float('inf')\n",
    "\n",
    "    # Loop over each candidate bandwidth\n",
    "    for h in bandwidths:\n",
    "        mse_list = []\n",
    "\n",
    "        # Perform leave-p-out cross-validation\n",
    "        for i in range(p, len(train_data)):\n",
    "            X_train = train_data[[var]].iloc[:i]\n",
    "            y_train = train_data['RET'].iloc[:i]\n",
    "            X_val = train_data[[var]].iloc[i:i+1]\n",
    "            y_val = train_data['RET'].iloc[i:i+1]\n",
    "\n",
    "            # Calculate time weights\n",
    "            t = np.arange(i)\n",
    "            s = np.array([i])\n",
    "            time_weights = calculate_time_weights(t, s, h)\n",
    "\n",
    "            # Perform weighted regression\n",
    "            y_pred = weighted_regression(X_train, y_train, X_val, time_weights)\n",
    "\n",
    "            # Calculate validation MSE\n",
    "            mse = mean_squared_error(y_val, y_pred)\n",
    "            mse_list.append(mse)\n",
    "\n",
    "        avg_mse = np.mean(mse_list)\n",
    "        if avg_mse < best_mse:\n",
    "            best_mse = avg_mse\n",
    "            best_bandwidth = h\n",
    "\n",
    "    return best_bandwidth, best_mse\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10.0, 0.009102506627779335)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example usage\n",
    "bandwidths = np.linspace(0.1, 10, 50)  # Candidate bandwidths for Laplace kernel\n",
    "best_h, best_mse = cross_validate_time_bandwidth(data, 'LTR', bandwidths)\n",
    "\n",
    "best_h, best_mse  # Output the best bandwidth and corresponding MSE\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'BM': 10.0,\n",
       " 'LTY': 10.0,\n",
       " 'NTIS': 10.0,\n",
       " 'INFL': 10.0,\n",
       " 'SVAR': 10.0,\n",
       " 'DP': 0.5040816326530613,\n",
       " 'DY': 10.0,\n",
       " 'DFR': 10.0,\n",
       " 'DFY': 10.0,\n",
       " 'LTR': 10.0,\n",
       " 'EP': 1.1102040816326533}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#为其他变量也选择相应的带宽\n",
    "best_bandwidths_time = {}\n",
    "variables = ['BM', 'LTY', 'NTIS', 'INFL', 'SVAR', 'DP', 'DY', 'DFR', 'DFY', 'LTR', 'EP']\n",
    "for var in variables:\n",
    "    best_h, best_mse = cross_validate_time_bandwidth(train_data, var, bandwidths)\n",
    "    best_bandwidths_time[var] = best_h\n",
    "\n",
    "best_bandwidths_time  # Output the best bandwidths for each variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 选择稳健权重最优带宽"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10.0, 0.008150433634071882)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the weight function based on OLS residuals (for robust regression)\n",
    "def calculate_residual_weights(residuals, h):\n",
    "    \"\"\"Calculate robust weights based on residuals and bandwidth h.\"\"\"\n",
    "    return np.exp(-np.abs(residuals / h)) / 2\n",
    "\n",
    "# Function to perform cross-validation for optimal residual weight bandwidth selection using train_data only\n",
    "def cross_validate_residual_bandwidth(train_data, var, bandwidths, p=5):\n",
    "    best_bandwidth = None\n",
    "    best_mse = float('inf')\n",
    "\n",
    "    # Loop over each candidate bandwidth\n",
    "    for h in bandwidths:\n",
    "        mse_list = []\n",
    "\n",
    "        # Perform leave-p-out cross-validation\n",
    "        for i in range(p, len(train_data)):\n",
    "            X_train = train_data[[var]].iloc[:i]\n",
    "            y_train = train_data['RET'].iloc[:i]\n",
    "            X_val = train_data[[var]].iloc[i:i+1]\n",
    "            y_val = train_data['RET'].iloc[i:i+1]\n",
    "\n",
    "            # First fit OLS to get residuals\n",
    "            model = LinearRegression()\n",
    "            model.fit(X_train, y_train)\n",
    "            y_train_pred = model.predict(X_train)\n",
    "            residuals = y_train - y_train_pred\n",
    "\n",
    "            # Calculate robust weights based on OLS residuals\n",
    "            residual_weights = calculate_residual_weights(residuals, h)\n",
    "\n",
    "            # Perform weighted regression with residual weights\n",
    "            model.fit(X_train, y_train, sample_weight=residual_weights)\n",
    "            y_pred = model.predict(X_val)\n",
    "\n",
    "            # Calculate validation MSE\n",
    "            mse = mean_squared_error(y_val, y_pred)\n",
    "            mse_list.append(mse)\n",
    "\n",
    "        avg_mse = np.mean(mse_list)\n",
    "        if avg_mse < best_mse:\n",
    "            best_mse = avg_mse\n",
    "            best_bandwidth = h\n",
    "\n",
    "    return best_bandwidth, best_mse\n",
    "\n",
    "# Example usage for 'BM' variable using train_data\n",
    "bandwidths = np.linspace(0.1, 10, 50)  # Candidate bandwidths for robust regression\n",
    "best_h_residual, best_mse_residual = cross_validate_residual_bandwidth(train_data, 'BM', bandwidths)\n",
    "\n",
    "best_h_residual, best_mse_residual  # Output the best residual weight bandwidth and corresponding MSE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'BM': 10.0,\n",
       " 'LTY': 0.5040816326530613,\n",
       " 'NTIS': 10.0,\n",
       " 'INFL': 0.1,\n",
       " 'SVAR': 10.0,\n",
       " 'DP': 0.1,\n",
       " 'DY': 10.0,\n",
       " 'DFR': 10.0,\n",
       " 'DFY': 10.0,\n",
       " 'LTR': 10.0,\n",
       " 'EP': 10.0}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#为其他变量也选择相应的带宽\n",
    "best_bandwidths_residual = {}\n",
    "variables = ['BM', 'LTY', 'NTIS', 'INFL', 'SVAR', 'DP', 'DY', 'DFR', 'DFY', 'LTR', 'EP']\n",
    "for var in variables:\n",
    "    best_h, best_mse = cross_validate_residual_bandwidth(train_data, var, bandwidths)\n",
    "    best_bandwidths_residual[var] = best_h\n",
    "\n",
    "best_bandwidths_residual  # Output the best bandwidths for each variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TRWLS 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the TRWLS regression process using both time weights and residual weights\n",
    "def trwls_predict(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual):\n",
    "    predictions = []\n",
    "\n",
    "    # Rolling prediction (same as OLS: use data until k to predict k+1)\n",
    "    for i in range(len(test_data)):\n",
    "        X_train = train_data[[var]].iloc[:len(train_data) + i]\n",
    "        y_train = train_data['RET'].iloc[:len(train_data) + i]\n",
    "        X_test = test_data[[var]].iloc[i:i+1]\n",
    "        \n",
    "        # Time weights (using Laplace kernel and best time bandwidth)\n",
    "        t = np.arange(len(X_train))\n",
    "        s = np.array([len(X_train)])\n",
    "        time_weights = calculate_time_weights(t, s, best_bandwidth_time)\n",
    "\n",
    "        # First OLS to get residuals\n",
    "        model_ols = LinearRegression()\n",
    "        model_ols.fit(X_train, y_train)\n",
    "        y_train_pred = model_ols.predict(X_train)\n",
    "        residuals = y_train - y_train_pred\n",
    "\n",
    "        # Residual weights (based on OLS residuals and best residual bandwidth)\n",
    "        residual_weights = calculate_residual_weights(residuals, best_bandwidth_residual)\n",
    "\n",
    "        # Final combined weights: time weights * residual weights\n",
    "        combined_weights = time_weights * residual_weights\n",
    "\n",
    "        # Perform weighted regression using TRWLS\n",
    "        model_trwls = LinearRegression()\n",
    "        model_trwls.fit(X_train, y_train, sample_weight=combined_weights)\n",
    "        y_pred = model_trwls.predict(X_test)\n",
    "\n",
    "        # Store prediction\n",
    "        predictions.append(y_pred[0])\n",
    "\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-0.003662638279096367,\n",
       " -0.004107049154393194,\n",
       " -0.008334106397125514,\n",
       " -0.010498405109265821,\n",
       " -0.010978539927018971]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example usage for 'BM' variable using stored best bandwidths for time and residuals\n",
    "var = 'BM'\n",
    "best_bandwidth_time = best_bandwidths_time[var]\n",
    "best_bandwidth_residual = best_bandwidths_residual[var]\n",
    "\n",
    "# Perform TRWLS predictions\n",
    "trwls_predictions = trwls_predict(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual)\n",
    "\n",
    "# Output the predictions\n",
    "trwls_predictions[:5]  # Display the first few predictions for reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BM_trwls_pred</th>\n",
       "      <th>LTY_trwls_pred</th>\n",
       "      <th>NTIS_trwls_pred</th>\n",
       "      <th>INFL_trwls_pred</th>\n",
       "      <th>SVAR_trwls_pred</th>\n",
       "      <th>DP_trwls_pred</th>\n",
       "      <th>DY_trwls_pred</th>\n",
       "      <th>DFR_trwls_pred</th>\n",
       "      <th>DFY_trwls_pred</th>\n",
       "      <th>LTR_trwls_pred</th>\n",
       "      <th>EP_trwls_pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1947-01-01</th>\n",
       "      <td>-0.003663</td>\n",
       "      <td>-0.001459</td>\n",
       "      <td>-0.000752</td>\n",
       "      <td>0.014951</td>\n",
       "      <td>0.006641</td>\n",
       "      <td>0.003314</td>\n",
       "      <td>-0.003018</td>\n",
       "      <td>-0.009540</td>\n",
       "      <td>-0.004678</td>\n",
       "      <td>-0.010276</td>\n",
       "      <td>0.088467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-02-01</th>\n",
       "      <td>-0.004107</td>\n",
       "      <td>-0.001459</td>\n",
       "      <td>0.002067</td>\n",
       "      <td>0.014951</td>\n",
       "      <td>0.012620</td>\n",
       "      <td>-0.042665</td>\n",
       "      <td>-0.003039</td>\n",
       "      <td>-0.003652</td>\n",
       "      <td>-0.004200</td>\n",
       "      <td>-0.004174</td>\n",
       "      <td>0.154493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-03-01</th>\n",
       "      <td>-0.008334</td>\n",
       "      <td>-0.001404</td>\n",
       "      <td>0.000266</td>\n",
       "      <td>-0.003756</td>\n",
       "      <td>0.004245</td>\n",
       "      <td>-0.057471</td>\n",
       "      <td>-0.003010</td>\n",
       "      <td>-0.003652</td>\n",
       "      <td>-0.002767</td>\n",
       "      <td>-0.004400</td>\n",
       "      <td>0.200784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-04-01</th>\n",
       "      <td>-0.010498</td>\n",
       "      <td>-0.001624</td>\n",
       "      <td>-0.000374</td>\n",
       "      <td>0.014951</td>\n",
       "      <td>0.005076</td>\n",
       "      <td>-0.113649</td>\n",
       "      <td>-0.005483</td>\n",
       "      <td>-0.016954</td>\n",
       "      <td>-0.001334</td>\n",
       "      <td>-0.017282</td>\n",
       "      <td>0.250604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-05-01</th>\n",
       "      <td>-0.010979</td>\n",
       "      <td>-0.001569</td>\n",
       "      <td>0.010331</td>\n",
       "      <td>0.014951</td>\n",
       "      <td>0.003175</td>\n",
       "      <td>-0.086827</td>\n",
       "      <td>-0.005448</td>\n",
       "      <td>-0.001472</td>\n",
       "      <td>-0.000856</td>\n",
       "      <td>-0.001462</td>\n",
       "      <td>0.252946</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            BM_trwls_pred  LTY_trwls_pred  NTIS_trwls_pred  INFL_trwls_pred  \\\n",
       "yyyymm                                                                        \n",
       "1947-01-01      -0.003663       -0.001459        -0.000752         0.014951   \n",
       "1947-02-01      -0.004107       -0.001459         0.002067         0.014951   \n",
       "1947-03-01      -0.008334       -0.001404         0.000266        -0.003756   \n",
       "1947-04-01      -0.010498       -0.001624        -0.000374         0.014951   \n",
       "1947-05-01      -0.010979       -0.001569         0.010331         0.014951   \n",
       "\n",
       "            SVAR_trwls_pred  DP_trwls_pred  DY_trwls_pred  DFR_trwls_pred  \\\n",
       "yyyymm                                                                      \n",
       "1947-01-01         0.006641       0.003314      -0.003018       -0.009540   \n",
       "1947-02-01         0.012620      -0.042665      -0.003039       -0.003652   \n",
       "1947-03-01         0.004245      -0.057471      -0.003010       -0.003652   \n",
       "1947-04-01         0.005076      -0.113649      -0.005483       -0.016954   \n",
       "1947-05-01         0.003175      -0.086827      -0.005448       -0.001472   \n",
       "\n",
       "            DFY_trwls_pred  LTR_trwls_pred  EP_trwls_pred  \n",
       "yyyymm                                                     \n",
       "1947-01-01       -0.004678       -0.010276       0.088467  \n",
       "1947-02-01       -0.004200       -0.004174       0.154493  \n",
       "1947-03-01       -0.002767       -0.004400       0.200784  \n",
       "1947-04-01       -0.001334       -0.017282       0.250604  \n",
       "1947-05-01       -0.000856       -0.001462       0.252946  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize a DataFrame to store TRWLS predictions for all variables\n",
    "trwls_pred_df = pd.DataFrame(index=test_data.index)\n",
    "\n",
    "# Loop through all variables and perform TRWLS prediction for each\n",
    "for var in variables:\n",
    "    best_bandwidth_time = best_bandwidths_time[var]\n",
    "    best_bandwidth_residual = best_bandwidths_residual[var]\n",
    "\n",
    "    # Perform TRWLS predictions for the variable\n",
    "    trwls_predictions = trwls_predict(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual)\n",
    "\n",
    "    # Store the predictions in the DataFrame with the appropriate column name\n",
    "    trwls_pred_df[f\"{var}_trwls_pred\"] = trwls_predictions\n",
    "\n",
    "# Display the TRWLS predictions DataFrame\n",
    "trwls_pred_df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估样本外预测表现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>R2_OLS</th>\n",
       "      <th>R2_TRWLS</th>\n",
       "      <th>CW_stat</th>\n",
       "      <th>CW_p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BM</th>\n",
       "      <td>-0.005668</td>\n",
       "      <td>-0.168634</td>\n",
       "      <td>-2.201577</td>\n",
       "      <td>2.795558e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LTY</th>\n",
       "      <td>-0.010613</td>\n",
       "      <td>-0.589546</td>\n",
       "      <td>-2.084951</td>\n",
       "      <td>3.736315e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NTIS</th>\n",
       "      <td>-0.004475</td>\n",
       "      <td>-1.273895</td>\n",
       "      <td>-1.579220</td>\n",
       "      <td>1.146470e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INFL</th>\n",
       "      <td>-0.009232</td>\n",
       "      <td>-0.012766</td>\n",
       "      <td>2.433455</td>\n",
       "      <td>1.515475e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVAR</th>\n",
       "      <td>0.063464</td>\n",
       "      <td>-0.119907</td>\n",
       "      <td>1.877644</td>\n",
       "      <td>6.076229e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DP</th>\n",
       "      <td>-0.291803</td>\n",
       "      <td>-6537.649059</td>\n",
       "      <td>-5.747164</td>\n",
       "      <td>1.252755e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DY</th>\n",
       "      <td>-0.003296</td>\n",
       "      <td>-0.059517</td>\n",
       "      <td>-0.890195</td>\n",
       "      <td>3.736059e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DFR</th>\n",
       "      <td>-0.019177</td>\n",
       "      <td>-7.742377</td>\n",
       "      <td>-4.482409</td>\n",
       "      <td>8.361135e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DFY</th>\n",
       "      <td>-0.004726</td>\n",
       "      <td>-0.248264</td>\n",
       "      <td>1.135744</td>\n",
       "      <td>2.563746e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LTR</th>\n",
       "      <td>-0.013774</td>\n",
       "      <td>-1.967906</td>\n",
       "      <td>-3.884624</td>\n",
       "      <td>1.102155e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EP</th>\n",
       "      <td>-0.115987</td>\n",
       "      <td>-2703.418009</td>\n",
       "      <td>4.232760</td>\n",
       "      <td>2.552248e-05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        R2_OLS     R2_TRWLS   CW_stat    CW_p_value\n",
       "BM   -0.005668    -0.168634 -2.201577  2.795558e-02\n",
       "LTY  -0.010613    -0.589546 -2.084951  3.736315e-02\n",
       "NTIS -0.004475    -1.273895 -1.579220  1.146470e-01\n",
       "INFL -0.009232    -0.012766  2.433455  1.515475e-02\n",
       "SVAR  0.063464    -0.119907  1.877644  6.076229e-02\n",
       "DP   -0.291803 -6537.649059 -5.747164  1.252755e-08\n",
       "DY   -0.003296    -0.059517 -0.890195  3.736059e-01\n",
       "DFR  -0.019177    -7.742377 -4.482409  8.361135e-06\n",
       "DFY  -0.004726    -0.248264  1.135744  2.563746e-01\n",
       "LTR  -0.013774    -1.967906 -3.884624  1.102155e-04\n",
       "EP   -0.115987 -2703.418009  4.232760  2.552248e-05"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Function to calculate R-squared for OLS and TRWLS predictions\n",
    "def calculate_r2(y_true, y_pred):\n",
    "    return r2_score(y_true, y_pred)\n",
    "\n",
    "# Function to perform Clark-West (CW) test\n",
    "def clark_west_test(y_true, y_pred_ols, y_pred_trwls):\n",
    "    # Calculate the prediction errors for both models\n",
    "    error_ols = y_true - y_pred_ols\n",
    "    error_trwls = y_true - y_pred_trwls\n",
    "\n",
    "    # Calculate the squared prediction errors\n",
    "    error_diff = (error_ols ** 2) - (error_trwls ** 2)\n",
    "    adjusted_error_diff = error_diff + ((y_pred_trwls - y_pred_ols) ** 2)\n",
    "\n",
    "    # Perform one-sample t-test\n",
    "    cw_stat, cw_p_value = stats.ttest_1samp(adjusted_error_diff, 0)\n",
    "\n",
    "    return cw_stat, cw_p_value\n",
    "\n",
    "# Initialize storage for results\n",
    "r2_ols_results = {}\n",
    "r2_trwls_results = {}\n",
    "cw_test_results = {}\n",
    "\n",
    "# Loop through all variables and compute R-squared and CW test\n",
    "for var in variables:\n",
    "    # Get the true values (RET) from the test data\n",
    "    y_true = test_data['RET'].values\n",
    "\n",
    "    # Get the OLS predictions\n",
    "    y_pred_ols = rolling_predictions_df[f\"ols_{var}_pred\"].values\n",
    "\n",
    "    # Get the TRWLS predictions\n",
    "    y_pred_trwls = trwls_pred_df[f\"{var}_trwls_pred\"].values\n",
    "\n",
    "    # Calculate R-squared for OLS and TRWLS\n",
    "    r2_ols = calculate_r2(y_true, y_pred_ols)\n",
    "    r2_trwls = calculate_r2(y_true, y_pred_trwls)\n",
    "    \n",
    "    # Store the R-squared results\n",
    "    r2_ols_results[var] = r2_ols\n",
    "    r2_trwls_results[var] = r2_trwls\n",
    "\n",
    "    # Perform CW test\n",
    "    cw_stat, cw_p_value = clark_west_test(y_true, y_pred_ols, y_pred_trwls)\n",
    "    cw_test_results[var] = {\"CW_stat\": cw_stat, \"CW_p_value\": cw_p_value}\n",
    "\n",
    "# Display the R-squared results and CW test results\n",
    "r2_ols_results, r2_trwls_results, cw_test_results\n",
    "\n",
    "# Convert the results to DataFrames for better display\n",
    "r2_ols_df = pd.DataFrame(r2_ols_results, index=[\"R2_OLS\"]).T\n",
    "r2_trwls_df = pd.DataFrame(r2_trwls_results, index=[\"R2_TRWLS\"]).T\n",
    "cw_test_df = pd.DataFrame(cw_test_results).T\n",
    "\n",
    "# Concatenate the DataFrames\n",
    "results_df = pd.concat([r2_ols_df, r2_trwls_df, cw_test_df], axis=1)\n",
    "results_df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(({'BM': -0.0026930234057651603,\n",
       "   'LTY': -0.017212442097554126,\n",
       "   'NTIS': 0.008210272473298308,\n",
       "   'INFL': -0.015530262752474489,\n",
       "   'SVAR': 0.004959181272883639,\n",
       "   'DP': -0.5830367871324305,\n",
       "   'DY': -0.0032036028596558186,\n",
       "   'DFR': -0.01348981798379656,\n",
       "   'DFY': -0.008346596381463067,\n",
       "   'LTR': 0.02699300754745304,\n",
       "   'EP': -0.22443744167049218},\n",
       "  {'BM': -0.1683527425245741,\n",
       "   'LTY': -0.5873198553248162,\n",
       "   'NTIS': -0.1592427395660696,\n",
       "   'INFL': -0.007675322262889583,\n",
       "   'SVAR': -0.021248159968980884,\n",
       "   'DP': -2090.626573290667,\n",
       "   'DY': -0.05207522344308191,\n",
       "   'DFR': -8.02148232084323,\n",
       "   'DFY': -0.2056985503595199,\n",
       "   'LTR': -1.2443937672583791,\n",
       "   'EP': -998.6779116821085},\n",
       "  {'BM': {'CW_stat': -1.2196758553451106, 'CW_p_value': 0.22323672458697325},\n",
       "   'LTY': {'CW_stat': -1.991270635252197, 'CW_p_value': 0.04706484461946905},\n",
       "   'NTIS': {'CW_stat': 1.0242854898214515, 'CW_p_value': 0.3062594368169186},\n",
       "   'INFL': {'CW_stat': 2.5238273366620416, 'CW_p_value': 0.011957097610795532},\n",
       "   'SVAR': {'CW_stat': 0.8023499222955193, 'CW_p_value': 0.42278078275296915},\n",
       "   'DP': {'CW_stat': -8.41372178941747, 'CW_p_value': 5.524101946571386e-16},\n",
       "   'DY': {'CW_stat': -0.3576272034479505, 'CW_p_value': 0.7207926384228736},\n",
       "   'DFR': {'CW_stat': -3.0147765457796973,\n",
       "    'CW_p_value': 0.0027194194882792315},\n",
       "   'DFY': {'CW_stat': 2.384381832887469, 'CW_p_value': 0.017527803299989257},\n",
       "   'LTR': {'CW_stat': -0.5165009956949033, 'CW_p_value': 0.6057622725492485},\n",
       "   'EP': {'CW_stat': 5.465764595009842, 'CW_p_value': 7.708601653769846e-08}}),\n",
       " ({'BM': -0.009213140392187125,\n",
       "   'LTY': -0.005516339797923875,\n",
       "   'NTIS': -0.01678812213441394,\n",
       "   'INFL': -0.004404661512751984,\n",
       "   'SVAR': 0.11549789230226659,\n",
       "   'DP': -0.029903230242028123,\n",
       "   'DY': -0.004234602884292782,\n",
       "   'DFR': -0.025183202194359433,\n",
       "   'DFY': -0.0023138107165223687,\n",
       "   'LTR': -0.051454268183648066,\n",
       "   'EP': -0.019002296607853353},\n",
       "  {'BM': -0.16988438695823915,\n",
       "   'LTY': -0.5929116586017262,\n",
       "   'NTIS': -2.282432050815269,\n",
       "   'INFL': -0.018227760362175882,\n",
       "   'SVAR': -0.2099579701643437,\n",
       "   'DP': -10559.149417881954,\n",
       "   'DY': -0.06714148645417617,\n",
       "   'DFR': -7.497786671751655,\n",
       "   'DFY': -0.28776797663167053,\n",
       "   'LTR': -2.6238115461791325,\n",
       "   'EP': -4245.205493291488},\n",
       "  {'BM': {'CW_stat': -1.9645759360804254, 'CW_p_value': 0.05010538935015961},\n",
       "   'LTY': {'CW_stat': -1.0434757178960568, 'CW_p_value': 0.2973129363473134},\n",
       "   'NTIS': {'CW_stat': -1.982423252230766, 'CW_p_value': 0.04806647507101516},\n",
       "   'INFL': {'CW_stat': 0.9382083430074623, 'CW_p_value': 0.3486628033086292},\n",
       "   'SVAR': {'CW_stat': 1.7571062437795117, 'CW_p_value': 0.07960969717126165},\n",
       "   'DP': {'CW_stat': -2.7045406386919972, 'CW_p_value': 0.007110790277109606},\n",
       "   'DY': {'CW_stat': -0.8684180050959832, 'CW_p_value': 0.38564883409006223},\n",
       "   'DFR': {'CW_stat': -3.3149120607960203, 'CW_p_value': 0.000994196870606296},\n",
       "   'DFY': {'CW_stat': -0.4170791035579231, 'CW_p_value': 0.676828209027766},\n",
       "   'LTR': {'CW_stat': -4.182390952386677, 'CW_p_value': 3.496300524674462e-05},\n",
       "   'EP': {'CW_stat': 2.0586635392718153, 'CW_p_value': 0.04012695554348618}}))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split the test data into two periods: 1947-1983 and 1984-2019\n",
    "test_data_1 = test_data.loc['1947-01-01':'1983-12-31']\n",
    "test_data_2 = test_data.loc['1984-01-01':'2019-12-31']\n",
    "\n",
    "# Function to calculate R-squared and CW test for a specific period\n",
    "def evaluate_model_performance(test_data_period, var, rolling_predictions_df, trwls_pred_df):\n",
    "    y_true = test_data_period['RET'].values\n",
    "    y_pred_ols = rolling_predictions_df[f\"ols_{var}_pred\"].loc[test_data_period.index].values\n",
    "    y_pred_trwls = trwls_pred_df[f\"{var}_trwls_pred\"].loc[test_data_period.index].values\n",
    "    \n",
    "    # Calculate R-squared for OLS and TRWLS\n",
    "    r2_ols = calculate_r2(y_true, y_pred_ols)\n",
    "    r2_trwls = calculate_r2(y_true, y_pred_trwls)\n",
    "    \n",
    "    # Perform CW test\n",
    "    cw_stat, cw_p_value = clark_west_test(y_true, y_pred_ols, y_pred_trwls)\n",
    "\n",
    "    return r2_ols, r2_trwls, cw_stat, cw_p_value\n",
    "\n",
    "# Initialize storage for results\n",
    "r2_ols_results_1 = {}\n",
    "r2_trwls_results_1 = {}\n",
    "cw_test_results_1 = {}\n",
    "\n",
    "r2_ols_results_2 = {}\n",
    "r2_trwls_results_2 = {}\n",
    "cw_test_results_2 = {}\n",
    "\n",
    "# Loop through all variables and evaluate performance for both periods\n",
    "for var in variables:\n",
    "    # Period 1: 1947-1983\n",
    "    r2_ols_1, r2_trwls_1, cw_stat_1, cw_p_value_1 = evaluate_model_performance(test_data_1, var, rolling_predictions_df, trwls_pred_df)\n",
    "    r2_ols_results_1[var] = r2_ols_1\n",
    "    r2_trwls_results_1[var] = r2_trwls_1\n",
    "    cw_test_results_1[var] = {\"CW_stat\": cw_stat_1, \"CW_p_value\": cw_p_value_1}\n",
    "    \n",
    "    # Period 2: 1984-2019\n",
    "    r2_ols_2, r2_trwls_2, cw_stat_2, cw_p_value_2 = evaluate_model_performance(test_data_2, var, rolling_predictions_df, trwls_pred_df)\n",
    "    r2_ols_results_2[var] = r2_ols_2\n",
    "    r2_trwls_results_2[var] = r2_trwls_2\n",
    "    cw_test_results_2[var] = {\"CW_stat\": cw_stat_2, \"CW_p_value\": cw_p_value_2}\n",
    "\n",
    "# Output the results for both periods\n",
    "(r2_ols_results_1, r2_trwls_results_1, cw_test_results_1), (r2_ols_results_2, r2_trwls_results_2, cw_test_results_2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>R2_OLS_1</th>\n",
       "      <th>R2_TRWLS_1</th>\n",
       "      <th>CW_stat</th>\n",
       "      <th>CW_p_value</th>\n",
       "      <th>R2_OLS_2</th>\n",
       "      <th>R2_TRWLS_2</th>\n",
       "      <th>CW_stat</th>\n",
       "      <th>CW_p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BM</th>\n",
       "      <td>-0.002693</td>\n",
       "      <td>-0.168353</td>\n",
       "      <td>-1.219676</td>\n",
       "      <td>2.232367e-01</td>\n",
       "      <td>-0.009213</td>\n",
       "      <td>-0.169884</td>\n",
       "      <td>-1.964576</td>\n",
       "      <td>0.050105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LTY</th>\n",
       "      <td>-0.017212</td>\n",
       "      <td>-0.587320</td>\n",
       "      <td>-1.991271</td>\n",
       "      <td>4.706484e-02</td>\n",
       "      <td>-0.005516</td>\n",
       "      <td>-0.592912</td>\n",
       "      <td>-1.043476</td>\n",
       "      <td>0.297313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NTIS</th>\n",
       "      <td>0.008210</td>\n",
       "      <td>-0.159243</td>\n",
       "      <td>1.024285</td>\n",
       "      <td>3.062594e-01</td>\n",
       "      <td>-0.016788</td>\n",
       "      <td>-2.282432</td>\n",
       "      <td>-1.982423</td>\n",
       "      <td>0.048066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INFL</th>\n",
       "      <td>-0.015530</td>\n",
       "      <td>-0.007675</td>\n",
       "      <td>2.523827</td>\n",
       "      <td>1.195710e-02</td>\n",
       "      <td>-0.004405</td>\n",
       "      <td>-0.018228</td>\n",
       "      <td>0.938208</td>\n",
       "      <td>0.348663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVAR</th>\n",
       "      <td>0.004959</td>\n",
       "      <td>-0.021248</td>\n",
       "      <td>0.802350</td>\n",
       "      <td>4.227808e-01</td>\n",
       "      <td>0.115498</td>\n",
       "      <td>-0.209958</td>\n",
       "      <td>1.757106</td>\n",
       "      <td>0.079610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DP</th>\n",
       "      <td>-0.583037</td>\n",
       "      <td>-2090.626573</td>\n",
       "      <td>-8.413722</td>\n",
       "      <td>5.524102e-16</td>\n",
       "      <td>-0.029903</td>\n",
       "      <td>-10559.149418</td>\n",
       "      <td>-2.704541</td>\n",
       "      <td>0.007111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DY</th>\n",
       "      <td>-0.003204</td>\n",
       "      <td>-0.052075</td>\n",
       "      <td>-0.357627</td>\n",
       "      <td>7.207926e-01</td>\n",
       "      <td>-0.004235</td>\n",
       "      <td>-0.067141</td>\n",
       "      <td>-0.868418</td>\n",
       "      <td>0.385649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DFR</th>\n",
       "      <td>-0.013490</td>\n",
       "      <td>-8.021482</td>\n",
       "      <td>-3.014777</td>\n",
       "      <td>2.719419e-03</td>\n",
       "      <td>-0.025183</td>\n",
       "      <td>-7.497787</td>\n",
       "      <td>-3.314912</td>\n",
       "      <td>0.000994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DFY</th>\n",
       "      <td>-0.008347</td>\n",
       "      <td>-0.205699</td>\n",
       "      <td>2.384382</td>\n",
       "      <td>1.752780e-02</td>\n",
       "      <td>-0.002314</td>\n",
       "      <td>-0.287768</td>\n",
       "      <td>-0.417079</td>\n",
       "      <td>0.676828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LTR</th>\n",
       "      <td>0.026993</td>\n",
       "      <td>-1.244394</td>\n",
       "      <td>-0.516501</td>\n",
       "      <td>6.057623e-01</td>\n",
       "      <td>-0.051454</td>\n",
       "      <td>-2.623812</td>\n",
       "      <td>-4.182391</td>\n",
       "      <td>0.000035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EP</th>\n",
       "      <td>-0.224437</td>\n",
       "      <td>-998.677912</td>\n",
       "      <td>5.465765</td>\n",
       "      <td>7.708602e-08</td>\n",
       "      <td>-0.019002</td>\n",
       "      <td>-4245.205493</td>\n",
       "      <td>2.058664</td>\n",
       "      <td>0.040127</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      R2_OLS_1   R2_TRWLS_1   CW_stat    CW_p_value  R2_OLS_2    R2_TRWLS_2  \\\n",
       "BM   -0.002693    -0.168353 -1.219676  2.232367e-01 -0.009213     -0.169884   \n",
       "LTY  -0.017212    -0.587320 -1.991271  4.706484e-02 -0.005516     -0.592912   \n",
       "NTIS  0.008210    -0.159243  1.024285  3.062594e-01 -0.016788     -2.282432   \n",
       "INFL -0.015530    -0.007675  2.523827  1.195710e-02 -0.004405     -0.018228   \n",
       "SVAR  0.004959    -0.021248  0.802350  4.227808e-01  0.115498     -0.209958   \n",
       "DP   -0.583037 -2090.626573 -8.413722  5.524102e-16 -0.029903 -10559.149418   \n",
       "DY   -0.003204    -0.052075 -0.357627  7.207926e-01 -0.004235     -0.067141   \n",
       "DFR  -0.013490    -8.021482 -3.014777  2.719419e-03 -0.025183     -7.497787   \n",
       "DFY  -0.008347    -0.205699  2.384382  1.752780e-02 -0.002314     -0.287768   \n",
       "LTR   0.026993    -1.244394 -0.516501  6.057623e-01 -0.051454     -2.623812   \n",
       "EP   -0.224437  -998.677912  5.465765  7.708602e-08 -0.019002  -4245.205493   \n",
       "\n",
       "       CW_stat  CW_p_value  \n",
       "BM   -1.964576    0.050105  \n",
       "LTY  -1.043476    0.297313  \n",
       "NTIS -1.982423    0.048066  \n",
       "INFL  0.938208    0.348663  \n",
       "SVAR  1.757106    0.079610  \n",
       "DP   -2.704541    0.007111  \n",
       "DY   -0.868418    0.385649  \n",
       "DFR  -3.314912    0.000994  \n",
       "DFY  -0.417079    0.676828  \n",
       "LTR  -4.182391    0.000035  \n",
       "EP    2.058664    0.040127  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert the results to DataFrames for better display\n",
    "r2_ols_df_1 = pd.DataFrame(r2_ols_results_1, index=[\"R2_OLS_1\"]).T\n",
    "r2_trwls_df_1 = pd.DataFrame(r2_trwls_results_1, index=[\"R2_TRWLS_1\"]).T\n",
    "cw_test_df_1 = pd.DataFrame(cw_test_results_1).T\n",
    "\n",
    "r2_ols_df_2 = pd.DataFrame(r2_ols_results_2, index=[\"R2_OLS_2\"]).T\n",
    "r2_trwls_df_2 = pd.DataFrame(r2_trwls_results_2, index=[\"R2_TRWLS_2\"]).T\n",
    "cw_test_df_2 = pd.DataFrame(cw_test_results_2).T\n",
    "\n",
    "# Concatenate the DataFrames for both periods\n",
    "result_df_Rsqua=pd.concat([r2_ols_df_1, r2_trwls_df_1, cw_test_df_1, r2_ols_df_2, r2_trwls_df_2, cw_test_df_2],axis=1)\n",
    "result_df_Rsqua"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 经济价值检验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variable</th>\n",
       "      <th>OLS_CG</th>\n",
       "      <th>OLS_CER</th>\n",
       "      <th>TRWLS_CG</th>\n",
       "      <th>TRWLS_CER</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM</td>\n",
       "      <td>0.061405</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>0.083048</td>\n",
       "      <td>0.000090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LTY</td>\n",
       "      <td>0.024234</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>-0.108371</td>\n",
       "      <td>-0.000132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NTIS</td>\n",
       "      <td>0.056033</td>\n",
       "      <td>0.000062</td>\n",
       "      <td>0.302917</td>\n",
       "      <td>0.000296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>INFL</td>\n",
       "      <td>0.031947</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.086027</td>\n",
       "      <td>0.000094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVAR</td>\n",
       "      <td>0.117974</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.459408</td>\n",
       "      <td>0.000407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DP</td>\n",
       "      <td>-0.045526</td>\n",
       "      <td>-0.000054</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.047612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DY</td>\n",
       "      <td>0.032932</td>\n",
       "      <td>0.000037</td>\n",
       "      <td>-0.017620</td>\n",
       "      <td>-0.000020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DFR</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>-0.214118</td>\n",
       "      <td>-0.000298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>DFY</td>\n",
       "      <td>0.037404</td>\n",
       "      <td>0.000042</td>\n",
       "      <td>0.106676</td>\n",
       "      <td>0.000114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LTR</td>\n",
       "      <td>0.080453</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.203491</td>\n",
       "      <td>0.000201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>EP</td>\n",
       "      <td>-0.013487</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>1615.645243</td>\n",
       "      <td>0.001021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Variable    OLS_CG   OLS_CER     TRWLS_CG  TRWLS_CER\n",
       "0        BM  0.061405  0.000068     0.083048   0.000090\n",
       "1       LTY  0.024234  0.000027    -0.108371  -0.000132\n",
       "2      NTIS  0.056033  0.000062     0.302917   0.000296\n",
       "3      INFL  0.031947  0.000036     0.086027   0.000094\n",
       "4      SVAR  0.117974  0.000127     0.459408   0.000407\n",
       "5        DP -0.045526 -0.000054    -1.000000  -0.047612\n",
       "6        DY  0.032932  0.000037    -0.017620  -0.000020\n",
       "7       DFR  0.039382  0.000044    -0.214118  -0.000298\n",
       "8       DFY  0.037404  0.000042     0.106676   0.000114\n",
       "9       LTR  0.080453  0.000088     0.203491   0.000201\n",
       "10       EP -0.013487 -0.000016  1615.645243   0.001021"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Function to calculate portfolio returns, cumulative gain (CG), and certainty equivalent return (CER)\n",
    "def portfolio_performance(y_pred, y_true, gamma=3):\n",
    "    # Portfolio return (assuming investment weight is proportional to predicted returns)\n",
    "    portfolio_returns = y_pred * y_true\n",
    "\n",
    "    # Cumulative Gain (CG)\n",
    "    cumulative_gain = np.prod(1 + portfolio_returns) - 1\n",
    "\n",
    "    # Expected return and variance\n",
    "    expected_return = np.mean(portfolio_returns)\n",
    "    return_variance = np.var(portfolio_returns)\n",
    "\n",
    "    # Certainty Equivalent Return (CER)\n",
    "    cer = expected_return - (gamma / 2) * return_variance\n",
    "\n",
    "    return cumulative_gain, cer\n",
    "\n",
    "# Initialize storage for performance results\n",
    "performance_results = {\n",
    "    \"Variable\": [],\n",
    "    \"OLS_CG\": [],\n",
    "    \"OLS_CER\": [],\n",
    "    \"TRWLS_CG\": [],\n",
    "    \"TRWLS_CER\": []\n",
    "}\n",
    "\n",
    "# Loop through each variable and calculate performance for both OLS and TRWLS\n",
    "for var in variables:\n",
    "    # Get true values (RET) and OLS/TRWLS predicted values\n",
    "    y_true = test_data['RET'].values\n",
    "    y_pred_ols = rolling_predictions_df[f\"ols_{var}_pred\"].values\n",
    "    y_pred_trwls = trwls_pred_df[f\"{var}_trwls_pred\"].values\n",
    "\n",
    "    # Calculate OLS portfolio performance\n",
    "    ols_cg, ols_cer = portfolio_performance(y_pred_ols, y_true, gamma=3)\n",
    "\n",
    "    # Calculate TRWLS portfolio performance\n",
    "    trwls_cg, trwls_cer = portfolio_performance(y_pred_trwls, y_true, gamma=3)\n",
    "\n",
    "    # Store results\n",
    "    performance_results[\"Variable\"].append(var)\n",
    "    performance_results[\"OLS_CG\"].append(ols_cg)\n",
    "    performance_results[\"OLS_CER\"].append(ols_cer)\n",
    "    performance_results[\"TRWLS_CG\"].append(trwls_cg)\n",
    "    performance_results[\"TRWLS_CER\"].append(trwls_cer)\n",
    "\n",
    "# Convert results to a DataFrame for better readability\n",
    "performance_df = pd.DataFrame(performance_results)\n",
    "\n",
    "# Display the performance results DataFrame\n",
    "performance_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variable</th>\n",
       "      <th>OLS_CG</th>\n",
       "      <th>OLS_CER</th>\n",
       "      <th>TRWLS_CG</th>\n",
       "      <th>TRWLS_CER</th>\n",
       "      <th>OLS_CG_1_x</th>\n",
       "      <th>OLS_CER_1_x</th>\n",
       "      <th>TRWLS_CG_1_x</th>\n",
       "      <th>TRWLS_CER_1_x</th>\n",
       "      <th>OLS_CG_1_y</th>\n",
       "      <th>OLS_CER_1_y</th>\n",
       "      <th>TRWLS_CG_1_y</th>\n",
       "      <th>TRWLS_CER_1_y</th>\n",
       "      <th>OLS_CG_2</th>\n",
       "      <th>OLS_CER_2</th>\n",
       "      <th>TRWLS_CG_2</th>\n",
       "      <th>TRWLS_CER_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM</td>\n",
       "      <td>0.061405</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>0.083048</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.018543</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>0.000168</td>\n",
       "      <td>-1.666353e-07</td>\n",
       "      <td>0.018543</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>0.000168</td>\n",
       "      <td>-1.666353e-07</td>\n",
       "      <td>0.042082</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.082866</td>\n",
       "      <td>0.000183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LTY</td>\n",
       "      <td>0.024234</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>-0.108371</td>\n",
       "      <td>-0.000132</td>\n",
       "      <td>0.005215</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>-0.039079</td>\n",
       "      <td>-9.120024e-05</td>\n",
       "      <td>0.005215</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>-0.039079</td>\n",
       "      <td>-9.120024e-05</td>\n",
       "      <td>0.018920</td>\n",
       "      <td>0.000043</td>\n",
       "      <td>-0.072109</td>\n",
       "      <td>-0.000175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NTIS</td>\n",
       "      <td>0.056033</td>\n",
       "      <td>0.000062</td>\n",
       "      <td>0.302917</td>\n",
       "      <td>0.000296</td>\n",
       "      <td>0.021355</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.086224</td>\n",
       "      <td>1.849871e-04</td>\n",
       "      <td>0.021355</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.086224</td>\n",
       "      <td>1.849871e-04</td>\n",
       "      <td>0.033953</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>0.199493</td>\n",
       "      <td>0.000411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>INFL</td>\n",
       "      <td>0.031947</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.086027</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>0.011978</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.039680</td>\n",
       "      <td>8.741975e-05</td>\n",
       "      <td>0.011978</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.039680</td>\n",
       "      <td>8.741975e-05</td>\n",
       "      <td>0.019733</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>0.044578</td>\n",
       "      <td>0.000101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVAR</td>\n",
       "      <td>0.117974</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.459408</td>\n",
       "      <td>0.000407</td>\n",
       "      <td>0.028913</td>\n",
       "      <td>0.000064</td>\n",
       "      <td>0.066903</td>\n",
       "      <td>1.451970e-04</td>\n",
       "      <td>0.028913</td>\n",
       "      <td>0.000064</td>\n",
       "      <td>0.066903</td>\n",
       "      <td>1.451970e-04</td>\n",
       "      <td>0.086558</td>\n",
       "      <td>0.000191</td>\n",
       "      <td>0.367892</td>\n",
       "      <td>0.000676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DP</td>\n",
       "      <td>-0.045526</td>\n",
       "      <td>-0.000054</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.047612</td>\n",
       "      <td>-0.048707</td>\n",
       "      <td>-0.000113</td>\n",
       "      <td>-0.990574</td>\n",
       "      <td>-1.611267e-02</td>\n",
       "      <td>-0.048707</td>\n",
       "      <td>-0.000113</td>\n",
       "      <td>-0.990574</td>\n",
       "      <td>-1.611267e-02</td>\n",
       "      <td>0.003345</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.079584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DY</td>\n",
       "      <td>0.032932</td>\n",
       "      <td>0.000037</td>\n",
       "      <td>-0.017620</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>0.013032</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>-0.007349</td>\n",
       "      <td>-1.663197e-05</td>\n",
       "      <td>0.013032</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>-0.007349</td>\n",
       "      <td>-1.663197e-05</td>\n",
       "      <td>0.019643</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>-0.010347</td>\n",
       "      <td>-0.000024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DFR</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>-0.214118</td>\n",
       "      <td>-0.000298</td>\n",
       "      <td>0.021974</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>0.043165</td>\n",
       "      <td>7.246619e-05</td>\n",
       "      <td>0.021974</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>0.043165</td>\n",
       "      <td>7.246619e-05</td>\n",
       "      <td>0.017033</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>-0.246637</td>\n",
       "      <td>-0.000678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>DFY</td>\n",
       "      <td>0.037404</td>\n",
       "      <td>0.000042</td>\n",
       "      <td>0.106676</td>\n",
       "      <td>0.000114</td>\n",
       "      <td>0.016094</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.074893</td>\n",
       "      <td>1.611369e-04</td>\n",
       "      <td>0.016094</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.074893</td>\n",
       "      <td>1.611369e-04</td>\n",
       "      <td>0.020972</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.029568</td>\n",
       "      <td>0.000065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LTR</td>\n",
       "      <td>0.080453</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.203491</td>\n",
       "      <td>0.000201</td>\n",
       "      <td>0.047349</td>\n",
       "      <td>0.000104</td>\n",
       "      <td>0.212242</td>\n",
       "      <td>4.272493e-04</td>\n",
       "      <td>0.047349</td>\n",
       "      <td>0.000104</td>\n",
       "      <td>0.212242</td>\n",
       "      <td>4.272493e-04</td>\n",
       "      <td>0.031608</td>\n",
       "      <td>0.000071</td>\n",
       "      <td>-0.007218</td>\n",
       "      <td>-0.000031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>EP</td>\n",
       "      <td>-0.013487</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>1615.645243</td>\n",
       "      <td>0.001021</td>\n",
       "      <td>-0.021959</td>\n",
       "      <td>-0.000050</td>\n",
       "      <td>4.784804</td>\n",
       "      <td>1.127168e-03</td>\n",
       "      <td>-0.021959</td>\n",
       "      <td>-0.000050</td>\n",
       "      <td>4.784804</td>\n",
       "      <td>1.127168e-03</td>\n",
       "      <td>0.008663</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>278.464145</td>\n",
       "      <td>0.001070</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Variable    OLS_CG   OLS_CER     TRWLS_CG  TRWLS_CER  OLS_CG_1_x  \\\n",
       "0        BM  0.061405  0.000068     0.083048   0.000090    0.018543   \n",
       "1       LTY  0.024234  0.000027    -0.108371  -0.000132    0.005215   \n",
       "2      NTIS  0.056033  0.000062     0.302917   0.000296    0.021355   \n",
       "3      INFL  0.031947  0.000036     0.086027   0.000094    0.011978   \n",
       "4      SVAR  0.117974  0.000127     0.459408   0.000407    0.028913   \n",
       "5        DP -0.045526 -0.000054    -1.000000  -0.047612   -0.048707   \n",
       "6        DY  0.032932  0.000037    -0.017620  -0.000020    0.013032   \n",
       "7       DFR  0.039382  0.000044    -0.214118  -0.000298    0.021974   \n",
       "8       DFY  0.037404  0.000042     0.106676   0.000114    0.016094   \n",
       "9       LTR  0.080453  0.000088     0.203491   0.000201    0.047349   \n",
       "10       EP -0.013487 -0.000016  1615.645243   0.001021   -0.021959   \n",
       "\n",
       "    OLS_CER_1_x  TRWLS_CG_1_x  TRWLS_CER_1_x  OLS_CG_1_y  OLS_CER_1_y  \\\n",
       "0      0.000041      0.000168  -1.666353e-07    0.018543     0.000041   \n",
       "1      0.000012     -0.039079  -9.120024e-05    0.005215     0.000012   \n",
       "2      0.000048      0.086224   1.849871e-04    0.021355     0.000048   \n",
       "3      0.000027      0.039680   8.741975e-05    0.011978     0.000027   \n",
       "4      0.000064      0.066903   1.451970e-04    0.028913     0.000064   \n",
       "5     -0.000113     -0.990574  -1.611267e-02   -0.048707    -0.000113   \n",
       "6      0.000029     -0.007349  -1.663197e-05    0.013032     0.000029   \n",
       "7      0.000049      0.043165   7.246619e-05    0.021974     0.000049   \n",
       "8      0.000036      0.074893   1.611369e-04    0.016094     0.000036   \n",
       "9      0.000104      0.212242   4.272493e-04    0.047349     0.000104   \n",
       "10    -0.000050      4.784804   1.127168e-03   -0.021959    -0.000050   \n",
       "\n",
       "    TRWLS_CG_1_y  TRWLS_CER_1_y  OLS_CG_2  OLS_CER_2  TRWLS_CG_2  TRWLS_CER_2  \n",
       "0       0.000168  -1.666353e-07  0.042082   0.000095    0.082866     0.000183  \n",
       "1      -0.039079  -9.120024e-05  0.018920   0.000043   -0.072109    -0.000175  \n",
       "2       0.086224   1.849871e-04  0.033953   0.000077    0.199493     0.000411  \n",
       "3       0.039680   8.741975e-05  0.019733   0.000045    0.044578     0.000101  \n",
       "4       0.066903   1.451970e-04  0.086558   0.000191    0.367892     0.000676  \n",
       "5      -0.990574  -1.611267e-02  0.003345   0.000008   -1.000000    -0.079584  \n",
       "6      -0.007349  -1.663197e-05  0.019643   0.000045   -0.010347    -0.000024  \n",
       "7       0.043165   7.246619e-05  0.017033   0.000039   -0.246637    -0.000678  \n",
       "8       0.074893   1.611369e-04  0.020972   0.000048    0.029568     0.000065  \n",
       "9       0.212242   4.272493e-04  0.031608   0.000071   -0.007218    -0.000031  \n",
       "10      4.784804   1.127168e-03  0.008663   0.000020  278.464145     0.001070  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Function to calculate portfolio performance (CG and CER) for a specific time period\n",
    "def evaluate_portfolio_performance_for_period(test_data_period, var, rolling_predictions_df, trwls_pred_df, gamma=3):\n",
    "    y_true = test_data_period['RET'].values\n",
    "    y_pred_ols = rolling_predictions_df[f\"ols_{var}_pred\"].loc[test_data_period.index].values\n",
    "    y_pred_trwls = trwls_pred_df[f\"{var}_trwls_pred\"].loc[test_data_period.index].values\n",
    "\n",
    "    # Calculate OLS portfolio performance\n",
    "    ols_cg, ols_cer = portfolio_performance(y_pred_ols, y_true, gamma)\n",
    "\n",
    "    # Calculate TRWLS portfolio performance\n",
    "    trwls_cg, trwls_cer = portfolio_performance(y_pred_trwls, y_true, gamma)\n",
    "\n",
    "    return ols_cg, ols_cer, trwls_cg, trwls_cer\n",
    "\n",
    "# Initialize storage for performance results for both periods\n",
    "performance_results_period_1 = {\n",
    "    \"Variable\": [],\n",
    "    \"OLS_CG_1\": [],\n",
    "    \"OLS_CER_1\": [],\n",
    "    \"TRWLS_CG_1\": [],\n",
    "    \"TRWLS_CER_1\": []\n",
    "}\n",
    "\n",
    "performance_results_period_2 = {\n",
    "    \"Variable\": [],\n",
    "    \"OLS_CG_2\": [],\n",
    "    \"OLS_CER_2\": [],\n",
    "    \"TRWLS_CG_2\": [],\n",
    "    \"TRWLS_CER_2\": []\n",
    "}\n",
    "\n",
    "# Loop through each variable and calculate performance for both periods\n",
    "for var in variables:\n",
    "    # Period 1: 1947-1983\n",
    "    ols_cg_1, ols_cer_1, trwls_cg_1, trwls_cer_1 = evaluate_portfolio_performance_for_period(test_data_1, var, rolling_predictions_df, trwls_pred_df, gamma=3)\n",
    "    performance_results_period_1[\"Variable\"].append(var)\n",
    "    performance_results_period_1[\"OLS_CG_1\"].append(ols_cg_1)\n",
    "    performance_results_period_1[\"OLS_CER_1\"].append(ols_cer_1)\n",
    "    performance_results_period_1[\"TRWLS_CG_1\"].append(trwls_cg_1)\n",
    "    performance_results_period_1[\"TRWLS_CER_1\"].append(trwls_cer_1)\n",
    "\n",
    "    # Period 2: 1984-2019\n",
    "    ols_cg_2, ols_cer_2, trwls_cg_2, trwls_cer_2 = evaluate_portfolio_performance_for_period(test_data_2, var, rolling_predictions_df, trwls_pred_df, gamma=3)\n",
    "    performance_results_period_2[\"Variable\"].append(var)\n",
    "    performance_results_period_2[\"OLS_CG_2\"].append(ols_cg_2)\n",
    "    performance_results_period_2[\"OLS_CER_2\"].append(ols_cer_2)\n",
    "    performance_results_period_2[\"TRWLS_CG_2\"].append(trwls_cg_2)\n",
    "    performance_results_period_2[\"TRWLS_CER_2\"].append(trwls_cer_2)\n",
    "\n",
    "# Convert results to DataFrames for better readability\n",
    "performance_df_period_1 = pd.DataFrame(performance_results_period_1)\n",
    "performance_df_period_2 = pd.DataFrame(performance_results_period_2)\n",
    "\n",
    "# Merge performance_df and performance_df_period_1,performance_df_perirod_2\n",
    "performance_df = pd.merge(performance_df, performance_df_period_1, on=\"Variable\")\n",
    "performance_df=pd.merge(performance_df, performance_df_period_2, on=\"Variable\")\n",
    "performance_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 考虑其他的预测模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BM_huber_pred</th>\n",
       "      <th>LTY_huber_pred</th>\n",
       "      <th>NTIS_huber_pred</th>\n",
       "      <th>INFL_huber_pred</th>\n",
       "      <th>SVAR_huber_pred</th>\n",
       "      <th>DP_huber_pred</th>\n",
       "      <th>DY_huber_pred</th>\n",
       "      <th>DFR_huber_pred</th>\n",
       "      <th>DFY_huber_pred</th>\n",
       "      <th>LTR_huber_pred</th>\n",
       "      <th>EP_huber_pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1947-01-01</th>\n",
       "      <td>0.006134</td>\n",
       "      <td>0.007745</td>\n",
       "      <td>0.006297</td>\n",
       "      <td>0.006385</td>\n",
       "      <td>0.011181</td>\n",
       "      <td>-0.003628</td>\n",
       "      <td>0.006672</td>\n",
       "      <td>0.007410</td>\n",
       "      <td>0.015677</td>\n",
       "      <td>0.003697</td>\n",
       "      <td>-0.001587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-02-01</th>\n",
       "      <td>0.005895</td>\n",
       "      <td>0.007745</td>\n",
       "      <td>0.006438</td>\n",
       "      <td>0.006385</td>\n",
       "      <td>0.013534</td>\n",
       "      <td>-0.008244</td>\n",
       "      <td>0.006677</td>\n",
       "      <td>0.008995</td>\n",
       "      <td>0.015581</td>\n",
       "      <td>0.005594</td>\n",
       "      <td>-0.005756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-03-01</th>\n",
       "      <td>0.003620</td>\n",
       "      <td>0.007766</td>\n",
       "      <td>0.006348</td>\n",
       "      <td>0.008367</td>\n",
       "      <td>0.010237</td>\n",
       "      <td>-0.009731</td>\n",
       "      <td>0.006671</td>\n",
       "      <td>0.008995</td>\n",
       "      <td>0.015295</td>\n",
       "      <td>0.005523</td>\n",
       "      <td>-0.008678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-04-01</th>\n",
       "      <td>0.002456</td>\n",
       "      <td>0.007683</td>\n",
       "      <td>0.006316</td>\n",
       "      <td>0.006385</td>\n",
       "      <td>0.010565</td>\n",
       "      <td>-0.015371</td>\n",
       "      <td>0.007177</td>\n",
       "      <td>0.005413</td>\n",
       "      <td>0.015008</td>\n",
       "      <td>0.001519</td>\n",
       "      <td>-0.011824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-05-01</th>\n",
       "      <td>0.002198</td>\n",
       "      <td>0.007703</td>\n",
       "      <td>0.006854</td>\n",
       "      <td>0.006385</td>\n",
       "      <td>0.009816</td>\n",
       "      <td>-0.012678</td>\n",
       "      <td>0.007170</td>\n",
       "      <td>0.009582</td>\n",
       "      <td>0.014913</td>\n",
       "      <td>0.006436</td>\n",
       "      <td>-0.011972</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            BM_huber_pred  LTY_huber_pred  NTIS_huber_pred  INFL_huber_pred  \\\n",
       "yyyymm                                                                        \n",
       "1947-01-01       0.006134        0.007745         0.006297         0.006385   \n",
       "1947-02-01       0.005895        0.007745         0.006438         0.006385   \n",
       "1947-03-01       0.003620        0.007766         0.006348         0.008367   \n",
       "1947-04-01       0.002456        0.007683         0.006316         0.006385   \n",
       "1947-05-01       0.002198        0.007703         0.006854         0.006385   \n",
       "\n",
       "            SVAR_huber_pred  DP_huber_pred  DY_huber_pred  DFR_huber_pred  \\\n",
       "yyyymm                                                                      \n",
       "1947-01-01         0.011181      -0.003628       0.006672        0.007410   \n",
       "1947-02-01         0.013534      -0.008244       0.006677        0.008995   \n",
       "1947-03-01         0.010237      -0.009731       0.006671        0.008995   \n",
       "1947-04-01         0.010565      -0.015371       0.007177        0.005413   \n",
       "1947-05-01         0.009816      -0.012678       0.007170        0.009582   \n",
       "\n",
       "            DFY_huber_pred  LTR_huber_pred  EP_huber_pred  \n",
       "yyyymm                                                     \n",
       "1947-01-01        0.015677        0.003697      -0.001587  \n",
       "1947-02-01        0.015581        0.005594      -0.005756  \n",
       "1947-03-01        0.015295        0.005523      -0.008678  \n",
       "1947-04-01        0.015008        0.001519      -0.011824  \n",
       "1947-05-01        0.014913        0.006436      -0.011972  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import HuberRegressor\n",
    "import pandas as pd\n",
    "\n",
    "# Function to perform Huber regression with forward step prediction\n",
    "def huber_predict_forward(train_data, test_data, var):\n",
    "    n_test = len(test_data)\n",
    "    \n",
    "    # Prepare the data\n",
    "    X_test = test_data[[var]].values\n",
    "\n",
    "    huber_predictions = []\n",
    "\n",
    "    # Perform forward step (one-step ahead) prediction\n",
    "    for i in range(n_test):\n",
    "        # Use data up to k for training and predict k+1\n",
    "        X_train_step = train_data[[var]].iloc[:len(train_data) + i].values\n",
    "        y_train_step = train_data['RET'].iloc[:len(train_data) + i].values\n",
    "        \n",
    "        # Initialize the Huber Regressor\n",
    "        model = HuberRegressor()\n",
    "\n",
    "        # Fit the model on the rolling training data\n",
    "        model.fit(X_train_step, y_train_step)\n",
    "\n",
    "        # Predict the next step\n",
    "        pred = model.predict(X_test[i].reshape(1, -1))\n",
    "        huber_predictions.append(pred[0])\n",
    "\n",
    "    return huber_predictions\n",
    "\n",
    "# Initialize a DataFrame to store Huber predictions for all variables\n",
    "huber_pred_df = pd.DataFrame(index=test_data.index)\n",
    "\n",
    "# Loop through all variables and perform Huber prediction with forward step for each\n",
    "for var in variables:\n",
    "    huber_predictions = huber_predict_forward(train_data, test_data, var)\n",
    "    huber_pred_df[f\"{var}_huber_pred\"] = huber_predictions\n",
    "\n",
    "# Display the Huber predictions DataFrame\n",
    "huber_pred_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BM_vls_pred</th>\n",
       "      <th>LTY_vls_pred</th>\n",
       "      <th>NTIS_vls_pred</th>\n",
       "      <th>INFL_vls_pred</th>\n",
       "      <th>SVAR_vls_pred</th>\n",
       "      <th>DP_vls_pred</th>\n",
       "      <th>DY_vls_pred</th>\n",
       "      <th>DFR_vls_pred</th>\n",
       "      <th>DFY_vls_pred</th>\n",
       "      <th>LTR_vls_pred</th>\n",
       "      <th>EP_vls_pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1947-01-01</th>\n",
       "      <td>0.004171</td>\n",
       "      <td>0.007874</td>\n",
       "      <td>0.003358</td>\n",
       "      <td>0.003180</td>\n",
       "      <td>0.009089</td>\n",
       "      <td>-0.007096</td>\n",
       "      <td>0.003358</td>\n",
       "      <td>0.005516</td>\n",
       "      <td>0.007701</td>\n",
       "      <td>0.000166</td>\n",
       "      <td>-0.012650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-02-01</th>\n",
       "      <td>0.004004</td>\n",
       "      <td>0.007874</td>\n",
       "      <td>0.003688</td>\n",
       "      <td>0.003180</td>\n",
       "      <td>0.010053</td>\n",
       "      <td>-0.012871</td>\n",
       "      <td>0.003347</td>\n",
       "      <td>0.007642</td>\n",
       "      <td>0.007668</td>\n",
       "      <td>0.002482</td>\n",
       "      <td>-0.019200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-03-01</th>\n",
       "      <td>0.002423</td>\n",
       "      <td>0.007938</td>\n",
       "      <td>0.003477</td>\n",
       "      <td>0.016245</td>\n",
       "      <td>0.008703</td>\n",
       "      <td>-0.014730</td>\n",
       "      <td>0.003362</td>\n",
       "      <td>0.007642</td>\n",
       "      <td>0.007571</td>\n",
       "      <td>0.002396</td>\n",
       "      <td>-0.023791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-04-01</th>\n",
       "      <td>0.001613</td>\n",
       "      <td>0.007682</td>\n",
       "      <td>0.003403</td>\n",
       "      <td>0.003180</td>\n",
       "      <td>0.008837</td>\n",
       "      <td>-0.021786</td>\n",
       "      <td>0.002099</td>\n",
       "      <td>0.002839</td>\n",
       "      <td>0.007473</td>\n",
       "      <td>-0.002494</td>\n",
       "      <td>-0.028733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947-05-01</th>\n",
       "      <td>0.001433</td>\n",
       "      <td>0.007746</td>\n",
       "      <td>0.004654</td>\n",
       "      <td>0.003180</td>\n",
       "      <td>0.008530</td>\n",
       "      <td>-0.018417</td>\n",
       "      <td>0.002117</td>\n",
       "      <td>0.008430</td>\n",
       "      <td>0.007441</td>\n",
       "      <td>0.003511</td>\n",
       "      <td>-0.028965</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            BM_vls_pred  LTY_vls_pred  NTIS_vls_pred  INFL_vls_pred  \\\n",
       "yyyymm                                                                \n",
       "1947-01-01     0.004171      0.007874       0.003358       0.003180   \n",
       "1947-02-01     0.004004      0.007874       0.003688       0.003180   \n",
       "1947-03-01     0.002423      0.007938       0.003477       0.016245   \n",
       "1947-04-01     0.001613      0.007682       0.003403       0.003180   \n",
       "1947-05-01     0.001433      0.007746       0.004654       0.003180   \n",
       "\n",
       "            SVAR_vls_pred  DP_vls_pred  DY_vls_pred  DFR_vls_pred  \\\n",
       "yyyymm                                                              \n",
       "1947-01-01       0.009089    -0.007096     0.003358      0.005516   \n",
       "1947-02-01       0.010053    -0.012871     0.003347      0.007642   \n",
       "1947-03-01       0.008703    -0.014730     0.003362      0.007642   \n",
       "1947-04-01       0.008837    -0.021786     0.002099      0.002839   \n",
       "1947-05-01       0.008530    -0.018417     0.002117      0.008430   \n",
       "\n",
       "            DFY_vls_pred  LTR_vls_pred  EP_vls_pred  \n",
       "yyyymm                                               \n",
       "1947-01-01      0.007701      0.000166    -0.012650  \n",
       "1947-02-01      0.007668      0.002482    -0.019200  \n",
       "1947-03-01      0.007571      0.002396    -0.023791  \n",
       "1947-04-01      0.007473     -0.002494    -0.028733  \n",
       "1947-05-01      0.007441      0.003511    -0.028965  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Function to perform Volatility-Weighted Least Squares (VLS) with forward step prediction\n",
    "def vls_predict_forward(train_data, test_data, var):\n",
    "    n_test = len(test_data)\n",
    "\n",
    "    # Prepare the data\n",
    "    X_test = test_data[[var]].values\n",
    "\n",
    "    vls_predictions = []\n",
    "\n",
    "    # Perform forward step (one-step ahead) prediction\n",
    "    for i in range(n_test):\n",
    "        # Use data up to k for training and predict k+1\n",
    "        X_train_step = train_data[[var]].iloc[:len(train_data) + i].values\n",
    "        y_train_step = train_data['RET'].iloc[:len(train_data) + i].values\n",
    "\n",
    "        # Calculate the volatility (standard deviation) for weighting\n",
    "        vol_train = X_train_step.std() if len(X_train_step) > 1 else 1  # Handle very small sample sizes\n",
    "        weights = 1 / (vol_train + 1e-8)  # Avoid division by zero with small epsilon\n",
    "\n",
    "        # Initialize the Linear Regression with weighted least squares\n",
    "        model = LinearRegression()\n",
    "\n",
    "        # Fit the model using volatility as weights\n",
    "        model.fit(X_train_step, y_train_step, sample_weight=weights)\n",
    "\n",
    "        # Predict the next step\n",
    "        pred = model.predict(X_test[i].reshape(1, -1))\n",
    "        vls_predictions.append(pred[0])\n",
    "\n",
    "    return vls_predictions\n",
    "\n",
    "# Initialize a DataFrame to store VLS predictions for all variables\n",
    "vls_pred_df = pd.DataFrame(index=test_data.index)\n",
    "\n",
    "# Loop through all variables and perform VLS prediction with forward step for each\n",
    "for var in variables:\n",
    "    vls_predictions = vls_predict_forward(train_data, test_data, var)\n",
    "    vls_pred_df[f\"{var}_vls_pred\"] = vls_predictions\n",
    "\n",
    "# Display the VLS predictions DataFrame\n",
    "vls_pred_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variable</th>\n",
       "      <th>Huber_R2</th>\n",
       "      <th>Huber_CER</th>\n",
       "      <th>VLS_R2</th>\n",
       "      <th>VLS_CER</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM</td>\n",
       "      <td>-0.055432</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>-0.014987</td>\n",
       "      <td>0.000073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LTY</td>\n",
       "      <td>-0.039677</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>-0.452447</td>\n",
       "      <td>-0.000086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NTIS</td>\n",
       "      <td>0.000264</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>-0.004307</td>\n",
       "      <td>0.000060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>INFL</td>\n",
       "      <td>-0.001398</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>-0.014547</td>\n",
       "      <td>0.000029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVAR</td>\n",
       "      <td>0.076672</td>\n",
       "      <td>0.000229</td>\n",
       "      <td>0.055326</td>\n",
       "      <td>0.000127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DP</td>\n",
       "      <td>-69.374660</td>\n",
       "      <td>-0.002215</td>\n",
       "      <td>-108.999819</td>\n",
       "      <td>-0.002886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DY</td>\n",
       "      <td>-0.000402</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>-0.007404</td>\n",
       "      <td>0.000024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DFR</td>\n",
       "      <td>-0.388251</td>\n",
       "      <td>-0.000003</td>\n",
       "      <td>-0.793993</td>\n",
       "      <td>-0.000036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>DFY</td>\n",
       "      <td>-0.029802</td>\n",
       "      <td>0.000079</td>\n",
       "      <td>-0.003009</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LTR</td>\n",
       "      <td>-0.138909</td>\n",
       "      <td>0.000114</td>\n",
       "      <td>-0.228695</td>\n",
       "      <td>0.000108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>EP</td>\n",
       "      <td>-11.197001</td>\n",
       "      <td>-0.000804</td>\n",
       "      <td>-29.166714</td>\n",
       "      <td>-0.001384</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Variable   Huber_R2  Huber_CER      VLS_R2   VLS_CER\n",
       "0        BM  -0.055432   0.000105   -0.014987  0.000073\n",
       "1       LTY  -0.039677   0.000008   -0.452447 -0.000086\n",
       "2      NTIS   0.000264   0.000059   -0.004307  0.000060\n",
       "3      INFL  -0.001398   0.000046   -0.014547  0.000029\n",
       "4      SVAR   0.076672   0.000229    0.055326  0.000127\n",
       "5        DP -69.374660  -0.002215 -108.999819 -0.002886\n",
       "6        DY  -0.000402   0.000046   -0.007404  0.000024\n",
       "7       DFR  -0.388251  -0.000003   -0.793993 -0.000036\n",
       "8       DFY  -0.029802   0.000079   -0.003009  0.000044\n",
       "9       LTR  -0.138909   0.000114   -0.228695  0.000108\n",
       "10       EP -11.197001  -0.000804  -29.166714 -0.001384"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Function to calculate R-squared and CER for Huber and VLS predictions\n",
    "def evaluate_huber_vls_performance(y_true, y_pred_huber, y_pred_vls, gamma=3):\n",
    "    # R-squared for Huber\n",
    "    r2_huber = r2_score(y_true, y_pred_huber)\n",
    "    # R-squared for VLS\n",
    "    r2_vls = r2_score(y_true, y_pred_vls)\n",
    "\n",
    "    # Portfolio performance for Huber and VLS (CER calculation)\n",
    "    cer_huber = portfolio_performance(y_pred_huber, y_true, gamma)[1]  # We are interested in CER (index 1)\n",
    "    cer_vls = portfolio_performance(y_pred_vls, y_true, gamma)[1]\n",
    "\n",
    "    return r2_huber, cer_huber, r2_vls, cer_vls\n",
    "\n",
    "# Initialize a DataFrame to store performance results for Huber and VLS\n",
    "performance_results_huber_vls = {\n",
    "    \"Variable\": [],\n",
    "    \"Huber_R2\": [],\n",
    "    \"Huber_CER\": [],\n",
    "    \"VLS_R2\": [],\n",
    "    \"VLS_CER\": []\n",
    "}\n",
    "\n",
    "# Loop through all variables and calculate R-squared and CER for Huber and VLS\n",
    "for var in variables:\n",
    "    y_true = test_data['RET'].values\n",
    "    y_pred_huber = huber_pred_df[f\"{var}_huber_pred\"].values\n",
    "    y_pred_vls = vls_pred_df[f\"{var}_vls_pred\"].values\n",
    "\n",
    "    # Calculate performance\n",
    "    r2_huber, cer_huber, r2_vls, cer_vls = evaluate_huber_vls_performance(y_true, y_pred_huber, y_pred_vls)\n",
    "\n",
    "    # Store results\n",
    "    performance_results_huber_vls[\"Variable\"].append(var)\n",
    "    performance_results_huber_vls[\"Huber_R2\"].append(r2_huber)\n",
    "    performance_results_huber_vls[\"Huber_CER\"].append(cer_huber)\n",
    "    performance_results_huber_vls[\"VLS_R2\"].append(r2_vls)\n",
    "    performance_results_huber_vls[\"VLS_CER\"].append(cer_vls)\n",
    "\n",
    "# Convert results to a DataFrame for better readability\n",
    "performance_df_huber_vls = pd.DataFrame(performance_results_huber_vls)\n",
    "\n",
    "# Display the performance results for Huber and VLS\n",
    "performance_df_huber_vls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 稳健性检验\n",
    "### 子集回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BM_ols_pred</th>\n",
       "      <th>TBL_ols_pred</th>\n",
       "      <th>LTY_ols_pred</th>\n",
       "      <th>NTIS_ols_pred</th>\n",
       "      <th>INFL_ols_pred</th>\n",
       "      <th>SVAR_ols_pred</th>\n",
       "      <th>DP_ols_pred</th>\n",
       "      <th>DY_ols_pred</th>\n",
       "      <th>DFR_ols_pred</th>\n",
       "      <th>DFY_ols_pred</th>\n",
       "      <th>LTR_ols_pred</th>\n",
       "      <th>EP_ols_pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1960-01-01</th>\n",
       "      <td>0.011514</td>\n",
       "      <td>0.004920</td>\n",
       "      <td>-0.000544</td>\n",
       "      <td>0.007467</td>\n",
       "      <td>0.003105</td>\n",
       "      <td>0.010397</td>\n",
       "      <td>-0.018210</td>\n",
       "      <td>0.005010</td>\n",
       "      <td>0.003438</td>\n",
       "      <td>0.008190</td>\n",
       "      <td>0.009229</td>\n",
       "      <td>-0.005791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-02-01</th>\n",
       "      <td>0.011686</td>\n",
       "      <td>0.005051</td>\n",
       "      <td>-0.000058</td>\n",
       "      <td>0.006994</td>\n",
       "      <td>0.007231</td>\n",
       "      <td>0.010088</td>\n",
       "      <td>-0.016342</td>\n",
       "      <td>0.005034</td>\n",
       "      <td>0.007071</td>\n",
       "      <td>0.008002</td>\n",
       "      <td>0.012639</td>\n",
       "      <td>-0.004800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-03-01</th>\n",
       "      <td>0.010060</td>\n",
       "      <td>0.005269</td>\n",
       "      <td>0.000672</td>\n",
       "      <td>0.006611</td>\n",
       "      <td>0.005164</td>\n",
       "      <td>0.010331</td>\n",
       "      <td>-0.017664</td>\n",
       "      <td>0.005051</td>\n",
       "      <td>0.010423</td>\n",
       "      <td>0.008077</td>\n",
       "      <td>0.015530</td>\n",
       "      <td>-0.005113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-04-01</th>\n",
       "      <td>0.009666</td>\n",
       "      <td>0.005296</td>\n",
       "      <td>0.000064</td>\n",
       "      <td>0.006615</td>\n",
       "      <td>0.007224</td>\n",
       "      <td>0.010765</td>\n",
       "      <td>-0.017816</td>\n",
       "      <td>0.005998</td>\n",
       "      <td>-0.005887</td>\n",
       "      <td>0.008115</td>\n",
       "      <td>-0.001222</td>\n",
       "      <td>-0.005003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-05-01</th>\n",
       "      <td>0.010286</td>\n",
       "      <td>0.005276</td>\n",
       "      <td>0.000429</td>\n",
       "      <td>0.006901</td>\n",
       "      <td>0.005164</td>\n",
       "      <td>0.011062</td>\n",
       "      <td>-0.016428</td>\n",
       "      <td>0.005996</td>\n",
       "      <td>0.005674</td>\n",
       "      <td>0.007852</td>\n",
       "      <td>0.010712</td>\n",
       "      <td>-0.004263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-08-01</th>\n",
       "      <td>-0.003056</td>\n",
       "      <td>0.003182</td>\n",
       "      <td>-0.019428</td>\n",
       "      <td>0.010063</td>\n",
       "      <td>0.010962</td>\n",
       "      <td>0.010951</td>\n",
       "      <td>-0.045467</td>\n",
       "      <td>0.005523</td>\n",
       "      <td>-0.017971</td>\n",
       "      <td>0.006726</td>\n",
       "      <td>0.003782</td>\n",
       "      <td>-0.021325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-09-01</th>\n",
       "      <td>-0.003355</td>\n",
       "      <td>0.002933</td>\n",
       "      <td>-0.020239</td>\n",
       "      <td>0.010596</td>\n",
       "      <td>0.011727</td>\n",
       "      <td>0.009977</td>\n",
       "      <td>-0.047519</td>\n",
       "      <td>0.005528</td>\n",
       "      <td>-0.021709</td>\n",
       "      <td>0.006801</td>\n",
       "      <td>0.000557</td>\n",
       "      <td>-0.022215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-10-01</th>\n",
       "      <td>-0.005614</td>\n",
       "      <td>0.002449</td>\n",
       "      <td>-0.024575</td>\n",
       "      <td>0.010397</td>\n",
       "      <td>0.010034</td>\n",
       "      <td>0.008160</td>\n",
       "      <td>-0.049705</td>\n",
       "      <td>0.005631</td>\n",
       "      <td>-0.050557</td>\n",
       "      <td>0.006163</td>\n",
       "      <td>-0.026090</td>\n",
       "      <td>-0.023068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-11-01</th>\n",
       "      <td>-0.005359</td>\n",
       "      <td>0.002419</td>\n",
       "      <td>-0.023562</td>\n",
       "      <td>0.010257</td>\n",
       "      <td>0.010800</td>\n",
       "      <td>0.009541</td>\n",
       "      <td>-0.046517</td>\n",
       "      <td>0.005636</td>\n",
       "      <td>-0.009449</td>\n",
       "      <td>0.006313</td>\n",
       "      <td>0.016605</td>\n",
       "      <td>-0.021731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-12-01</th>\n",
       "      <td>-0.004747</td>\n",
       "      <td>0.002335</td>\n",
       "      <td>-0.023683</td>\n",
       "      <td>0.010322</td>\n",
       "      <td>0.011546</td>\n",
       "      <td>0.011000</td>\n",
       "      <td>-0.047756</td>\n",
       "      <td>0.005639</td>\n",
       "      <td>-0.018425</td>\n",
       "      <td>0.005975</td>\n",
       "      <td>0.007191</td>\n",
       "      <td>-0.022199</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            BM_ols_pred  TBL_ols_pred  LTY_ols_pred  NTIS_ols_pred  \\\n",
       "yyyymm                                                               \n",
       "1960-01-01     0.011514      0.004920     -0.000544       0.007467   \n",
       "1960-02-01     0.011686      0.005051     -0.000058       0.006994   \n",
       "1960-03-01     0.010060      0.005269      0.000672       0.006611   \n",
       "1960-04-01     0.009666      0.005296      0.000064       0.006615   \n",
       "1960-05-01     0.010286      0.005276      0.000429       0.006901   \n",
       "...                 ...           ...           ...            ...   \n",
       "1979-08-01    -0.003056      0.003182     -0.019428       0.010063   \n",
       "1979-09-01    -0.003355      0.002933     -0.020239       0.010596   \n",
       "1979-10-01    -0.005614      0.002449     -0.024575       0.010397   \n",
       "1979-11-01    -0.005359      0.002419     -0.023562       0.010257   \n",
       "1979-12-01    -0.004747      0.002335     -0.023683       0.010322   \n",
       "\n",
       "            INFL_ols_pred  SVAR_ols_pred  DP_ols_pred  DY_ols_pred  \\\n",
       "yyyymm                                                               \n",
       "1960-01-01       0.003105       0.010397    -0.018210     0.005010   \n",
       "1960-02-01       0.007231       0.010088    -0.016342     0.005034   \n",
       "1960-03-01       0.005164       0.010331    -0.017664     0.005051   \n",
       "1960-04-01       0.007224       0.010765    -0.017816     0.005998   \n",
       "1960-05-01       0.005164       0.011062    -0.016428     0.005996   \n",
       "...                   ...            ...          ...          ...   \n",
       "1979-08-01       0.010962       0.010951    -0.045467     0.005523   \n",
       "1979-09-01       0.011727       0.009977    -0.047519     0.005528   \n",
       "1979-10-01       0.010034       0.008160    -0.049705     0.005631   \n",
       "1979-11-01       0.010800       0.009541    -0.046517     0.005636   \n",
       "1979-12-01       0.011546       0.011000    -0.047756     0.005639   \n",
       "\n",
       "            DFR_ols_pred  DFY_ols_pred  LTR_ols_pred  EP_ols_pred  \n",
       "yyyymm                                                             \n",
       "1960-01-01      0.003438      0.008190      0.009229    -0.005791  \n",
       "1960-02-01      0.007071      0.008002      0.012639    -0.004800  \n",
       "1960-03-01      0.010423      0.008077      0.015530    -0.005113  \n",
       "1960-04-01     -0.005887      0.008115     -0.001222    -0.005003  \n",
       "1960-05-01      0.005674      0.007852      0.010712    -0.004263  \n",
       "...                  ...           ...           ...          ...  \n",
       "1979-08-01     -0.017971      0.006726      0.003782    -0.021325  \n",
       "1979-09-01     -0.021709      0.006801      0.000557    -0.022215  \n",
       "1979-10-01     -0.050557      0.006163     -0.026090    -0.023068  \n",
       "1979-11-01     -0.009449      0.006313      0.016605    -0.021731  \n",
       "1979-12-01     -0.018425      0.005975      0.007191    -0.022199  \n",
       "\n",
       "[240 rows x 12 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "# Split the dataset into two subsets for subset regression\n",
    "train_data_1 = data.loc[:'1959-12-31']\n",
    "test_data_1 = data.loc['1960-01-01':'1979-12-31']\n",
    "\n",
    "train_data_2 = data.loc['1980-01-01':'1999-12-31']\n",
    "test_data_2 = data.loc['2000-01-01':]\n",
    "\n",
    "# Function for OLS prediction with forward step (one-step ahead)\n",
    "def ols_predict(train_data, test_data, var):\n",
    "    ols_predictions = []\n",
    "    for i in range(len(test_data)):\n",
    "        X_train = train_data[[var]].iloc[:len(train_data) + i]\n",
    "        y_train = train_data['RET'].iloc[:len(train_data) + i]\n",
    "        X_test = test_data[[var]].iloc[i:i+1]\n",
    "        \n",
    "        # OLS regression\n",
    "        if len(X_train) > 1:  # Ensure there's enough data to train\n",
    "            model = LinearRegression()\n",
    "            model.fit(X_train, y_train)\n",
    "            # Predict the next step\n",
    "            pred = model.predict(X_test)\n",
    "            ols_predictions.append(pred[0])\n",
    "        else:\n",
    "            ols_predictions.append(None)  # Append None if not enough data\n",
    "    return ols_predictions\n",
    "\n",
    "# Initialize DataFrames for storing OLS predictions\n",
    "ols_pred_df_1 = pd.DataFrame(index=test_data_1.index)\n",
    "ols_pred_df_2 = pd.DataFrame(index=test_data_2.index)\n",
    "\n",
    "# Loop through variables and perform OLS on subsets\n",
    "variables_without_tbl = ['BM', 'TBL', 'LTY', 'NTIS', 'INFL', 'SVAR', 'DP', 'DY', 'DFR', 'DFY', 'LTR', 'EP']\n",
    "for var in variables_without_tbl:\n",
    "    # OLS predictions for both subsets\n",
    "    ols_pred_df_1[f\"{var}_ols_pred\"] = ols_predict(train_data_1, test_data_1, var)\n",
    "    ols_pred_df_2[f\"{var}_ols_pred\"] = ols_predict(train_data_2, test_data_2, var)\n",
    "\n",
    "# Display the OLS predictions DataFrame\n",
    "ols_pred_df_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BM_trwls_pred</th>\n",
       "      <th>LTY_trwls_pred</th>\n",
       "      <th>NTIS_trwls_pred</th>\n",
       "      <th>INFL_trwls_pred</th>\n",
       "      <th>SVAR_trwls_pred</th>\n",
       "      <th>DP_trwls_pred</th>\n",
       "      <th>DY_trwls_pred</th>\n",
       "      <th>DFR_trwls_pred</th>\n",
       "      <th>DFY_trwls_pred</th>\n",
       "      <th>LTR_trwls_pred</th>\n",
       "      <th>EP_trwls_pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yyyymm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1960-01-01</th>\n",
       "      <td>0.010515</td>\n",
       "      <td>0.007556</td>\n",
       "      <td>0.020180</td>\n",
       "      <td>0.010803</td>\n",
       "      <td>0.007514</td>\n",
       "      <td>-0.147045</td>\n",
       "      <td>-0.011260</td>\n",
       "      <td>0.007697</td>\n",
       "      <td>0.010576</td>\n",
       "      <td>0.005003</td>\n",
       "      <td>-0.054987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-02-01</th>\n",
       "      <td>0.010518</td>\n",
       "      <td>0.008427</td>\n",
       "      <td>0.015484</td>\n",
       "      <td>0.011296</td>\n",
       "      <td>0.001902</td>\n",
       "      <td>-0.053379</td>\n",
       "      <td>-0.010702</td>\n",
       "      <td>0.004771</td>\n",
       "      <td>0.011532</td>\n",
       "      <td>0.001384</td>\n",
       "      <td>0.013974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-03-01</th>\n",
       "      <td>0.010491</td>\n",
       "      <td>0.009733</td>\n",
       "      <td>0.011680</td>\n",
       "      <td>0.011049</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>-0.119643</td>\n",
       "      <td>-0.010298</td>\n",
       "      <td>0.002069</td>\n",
       "      <td>0.011150</td>\n",
       "      <td>-0.001685</td>\n",
       "      <td>-0.007809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-04-01</th>\n",
       "      <td>0.010485</td>\n",
       "      <td>0.008644</td>\n",
       "      <td>0.011720</td>\n",
       "      <td>0.011295</td>\n",
       "      <td>0.014183</td>\n",
       "      <td>-0.127269</td>\n",
       "      <td>0.011785</td>\n",
       "      <td>0.015210</td>\n",
       "      <td>0.010959</td>\n",
       "      <td>0.016098</td>\n",
       "      <td>-0.000154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1960-05-01</th>\n",
       "      <td>0.010495</td>\n",
       "      <td>0.009298</td>\n",
       "      <td>0.014562</td>\n",
       "      <td>0.011049</td>\n",
       "      <td>0.019572</td>\n",
       "      <td>-0.057675</td>\n",
       "      <td>0.011723</td>\n",
       "      <td>0.005896</td>\n",
       "      <td>0.012297</td>\n",
       "      <td>0.003430</td>\n",
       "      <td>0.051392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-08-01</th>\n",
       "      <td>0.010277</td>\n",
       "      <td>-0.026271</td>\n",
       "      <td>0.045962</td>\n",
       "      <td>0.011742</td>\n",
       "      <td>0.017555</td>\n",
       "      <td>-1.513774</td>\n",
       "      <td>0.000700</td>\n",
       "      <td>0.024945</td>\n",
       "      <td>0.018033</td>\n",
       "      <td>0.010787</td>\n",
       "      <td>-1.136851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-09-01</th>\n",
       "      <td>0.010272</td>\n",
       "      <td>-0.027722</td>\n",
       "      <td>0.051255</td>\n",
       "      <td>0.011834</td>\n",
       "      <td>-0.000112</td>\n",
       "      <td>-1.616677</td>\n",
       "      <td>0.000814</td>\n",
       "      <td>0.027956</td>\n",
       "      <td>0.017651</td>\n",
       "      <td>0.014210</td>\n",
       "      <td>-1.198836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-10-01</th>\n",
       "      <td>0.010235</td>\n",
       "      <td>-0.035489</td>\n",
       "      <td>0.049286</td>\n",
       "      <td>0.011631</td>\n",
       "      <td>-0.033090</td>\n",
       "      <td>-1.726300</td>\n",
       "      <td>0.003230</td>\n",
       "      <td>0.051198</td>\n",
       "      <td>0.020902</td>\n",
       "      <td>0.042497</td>\n",
       "      <td>-1.258216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-11-01</th>\n",
       "      <td>0.010239</td>\n",
       "      <td>-0.033675</td>\n",
       "      <td>0.047894</td>\n",
       "      <td>0.011723</td>\n",
       "      <td>-0.008023</td>\n",
       "      <td>-1.566427</td>\n",
       "      <td>0.003342</td>\n",
       "      <td>0.018080</td>\n",
       "      <td>0.020137</td>\n",
       "      <td>-0.002826</td>\n",
       "      <td>-1.165170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-12-01</th>\n",
       "      <td>0.010249</td>\n",
       "      <td>-0.033892</td>\n",
       "      <td>0.048540</td>\n",
       "      <td>0.011812</td>\n",
       "      <td>0.018451</td>\n",
       "      <td>-1.628548</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.025311</td>\n",
       "      <td>0.021858</td>\n",
       "      <td>0.007167</td>\n",
       "      <td>-1.197746</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            BM_trwls_pred  LTY_trwls_pred  NTIS_trwls_pred  INFL_trwls_pred  \\\n",
       "yyyymm                                                                        \n",
       "1960-01-01       0.010515        0.007556         0.020180         0.010803   \n",
       "1960-02-01       0.010518        0.008427         0.015484         0.011296   \n",
       "1960-03-01       0.010491        0.009733         0.011680         0.011049   \n",
       "1960-04-01       0.010485        0.008644         0.011720         0.011295   \n",
       "1960-05-01       0.010495        0.009298         0.014562         0.011049   \n",
       "...                   ...             ...              ...              ...   \n",
       "1979-08-01       0.010277       -0.026271         0.045962         0.011742   \n",
       "1979-09-01       0.010272       -0.027722         0.051255         0.011834   \n",
       "1979-10-01       0.010235       -0.035489         0.049286         0.011631   \n",
       "1979-11-01       0.010239       -0.033675         0.047894         0.011723   \n",
       "1979-12-01       0.010249       -0.033892         0.048540         0.011812   \n",
       "\n",
       "            SVAR_trwls_pred  DP_trwls_pred  DY_trwls_pred  DFR_trwls_pred  \\\n",
       "yyyymm                                                                      \n",
       "1960-01-01         0.007514      -0.147045      -0.011260        0.007697   \n",
       "1960-02-01         0.001902      -0.053379      -0.010702        0.004771   \n",
       "1960-03-01         0.006305      -0.119643      -0.010298        0.002069   \n",
       "1960-04-01         0.014183      -0.127269       0.011785        0.015210   \n",
       "1960-05-01         0.019572      -0.057675       0.011723        0.005896   \n",
       "...                     ...            ...            ...             ...   \n",
       "1979-08-01         0.017555      -1.513774       0.000700        0.024945   \n",
       "1979-09-01        -0.000112      -1.616677       0.000814        0.027956   \n",
       "1979-10-01        -0.033090      -1.726300       0.003230        0.051198   \n",
       "1979-11-01        -0.008023      -1.566427       0.003342        0.018080   \n",
       "1979-12-01         0.018451      -1.628548       0.003408        0.025311   \n",
       "\n",
       "            DFY_trwls_pred  LTR_trwls_pred  EP_trwls_pred  \n",
       "yyyymm                                                     \n",
       "1960-01-01        0.010576        0.005003      -0.054987  \n",
       "1960-02-01        0.011532        0.001384       0.013974  \n",
       "1960-03-01        0.011150       -0.001685      -0.007809  \n",
       "1960-04-01        0.010959        0.016098      -0.000154  \n",
       "1960-05-01        0.012297        0.003430       0.051392  \n",
       "...                    ...             ...            ...  \n",
       "1979-08-01        0.018033        0.010787      -1.136851  \n",
       "1979-09-01        0.017651        0.014210      -1.198836  \n",
       "1979-10-01        0.020902        0.042497      -1.258216  \n",
       "1979-11-01        0.020137       -0.002826      -1.165170  \n",
       "1979-12-01        0.021858        0.007167      -1.197746  \n",
       "\n",
       "[240 rows x 11 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Function for TRWLS prediction with forward step (one-step ahead)\n",
    "def trwls_predict(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual):\n",
    "    trwls_predictions = []\n",
    "    for i in range(len(test_data)):\n",
    "        X_train = train_data[[var]].iloc[:len(train_data) + i]\n",
    "        y_train = train_data['RET'].iloc[:len(train_data) + i]\n",
    "        X_test = test_data[[var]].iloc[i:i+1]\n",
    "        \n",
    "        # Calculate time and residual weights\n",
    "        time_weights = calculate_time_weights(np.arange(len(X_train)), np.array([len(X_train)]), best_bandwidth_time)\n",
    "        residual_weights = calculate_residual_weights(y_train - LinearRegression().fit(X_train, y_train).predict(X_train), best_bandwidth_residual)\n",
    "        combined_weights = time_weights * residual_weights\n",
    "        \n",
    "        # TRWLS regression\n",
    "        model = LinearRegression()\n",
    "        model.fit(X_train, y_train, sample_weight=combined_weights)\n",
    "        \n",
    "        # Predict the next step\n",
    "        pred = model.predict(X_test)\n",
    "        trwls_predictions.append(pred[0])\n",
    "    return trwls_predictions\n",
    "\n",
    "# Initialize DataFrames for storing TRWLS predictions\n",
    "trwls_pred_df_1 = pd.DataFrame(index=test_data_1.index)\n",
    "trwls_pred_df_2 = pd.DataFrame(index=test_data_2.index)\n",
    "\n",
    "# Loop through variables and perform TRWLS on subsets (using predefined best_bandwidths)\n",
    "for var in variables:\n",
    "    # TRWLS predictions for both subsets\n",
    "    trwls_pred_df_1[f\"{var}_trwls_pred\"] = trwls_predict(train_data_1, test_data_1, var, best_bandwidths_time[var], best_bandwidths_residual[var])\n",
    "    trwls_pred_df_2[f\"{var}_trwls_pred\"] = trwls_predict(train_data_2, test_data_2, var, best_bandwidths_time[var], best_bandwidths_residual[var])\n",
    "\n",
    "# Display the TRWLS predictions DataFrame\n",
    "trwls_pred_df_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variable</th>\n",
       "      <th>OLS_R2_Subset1</th>\n",
       "      <th>OLS_CER_Subset1</th>\n",
       "      <th>OLS_R2_Subset2</th>\n",
       "      <th>OLS_CER_Subset2</th>\n",
       "      <th>TRWLS_R2_Subset1</th>\n",
       "      <th>TRWLS_CER_Subset1</th>\n",
       "      <th>TRWLS_R2_Subset2</th>\n",
       "      <th>TRWLS_CER_Subset2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM</td>\n",
       "      <td>-0.000424</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>-0.069881</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>-0.030096</td>\n",
       "      <td>0.000034</td>\n",
       "      <td>-1.098837</td>\n",
       "      <td>-0.000167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LTY</td>\n",
       "      <td>-0.074634</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.224905</td>\n",
       "      <td>0.000119</td>\n",
       "      <td>-0.091601</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>-0.296251</td>\n",
       "      <td>0.000132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NTIS</td>\n",
       "      <td>-0.000289</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.100467</td>\n",
       "      <td>0.000055</td>\n",
       "      <td>-0.374284</td>\n",
       "      <td>0.000164</td>\n",
       "      <td>-8.642627</td>\n",
       "      <td>0.000236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>INFL</td>\n",
       "      <td>-0.029287</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>-0.148701</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>-0.040622</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.479505</td>\n",
       "      <td>0.000048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVAR</td>\n",
       "      <td>-0.010967</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.138800</td>\n",
       "      <td>0.000304</td>\n",
       "      <td>-0.394491</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>0.010357</td>\n",
       "      <td>0.000367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DP</td>\n",
       "      <td>-0.642570</td>\n",
       "      <td>-0.000046</td>\n",
       "      <td>-0.012828</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>-433.246292</td>\n",
       "      <td>-0.001665</td>\n",
       "      <td>-250.014122</td>\n",
       "      <td>-0.002870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DY</td>\n",
       "      <td>-0.002598</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>-0.139608</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>-0.002792</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>-2.831770</td>\n",
       "      <td>-0.000129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DFR</td>\n",
       "      <td>0.022719</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>-0.542554</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>-0.212481</td>\n",
       "      <td>-0.000035</td>\n",
       "      <td>-0.115315</td>\n",
       "      <td>0.000092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>DFY</td>\n",
       "      <td>-0.023397</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>-0.040000</td>\n",
       "      <td>0.000043</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>-0.457034</td>\n",
       "      <td>0.000058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LTR</td>\n",
       "      <td>0.068469</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>-0.295485</td>\n",
       "      <td>-0.000107</td>\n",
       "      <td>-0.169243</td>\n",
       "      <td>-0.000070</td>\n",
       "      <td>-0.121301</td>\n",
       "      <td>0.000102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>EP</td>\n",
       "      <td>-0.114499</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>-0.009425</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>-197.427684</td>\n",
       "      <td>-0.000557</td>\n",
       "      <td>-9.755500</td>\n",
       "      <td>-0.000204</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Variable  OLS_R2_Subset1  OLS_CER_Subset1  OLS_R2_Subset2  OLS_CER_Subset2  \\\n",
       "0        BM       -0.000424         0.000039       -0.069881         0.000061   \n",
       "1       LTY       -0.074634        -0.000006       -0.224905         0.000119   \n",
       "2      NTIS       -0.000289         0.000035       -0.100467         0.000055   \n",
       "3      INFL       -0.029287         0.000012       -0.148701         0.000047   \n",
       "4      SVAR       -0.010967         0.000048        0.138800         0.000304   \n",
       "5        DP       -0.642570        -0.000046       -0.012828         0.000008   \n",
       "6        DY       -0.002598         0.000020       -0.139608         0.000026   \n",
       "7       DFR        0.022719         0.000084       -0.542554         0.000012   \n",
       "8       DFY       -0.023397         0.000016       -0.040000         0.000043   \n",
       "9       LTR        0.068469         0.000111       -0.295485        -0.000107   \n",
       "10       EP       -0.114499        -0.000011       -0.009425         0.000031   \n",
       "\n",
       "    TRWLS_R2_Subset1  TRWLS_CER_Subset1  TRWLS_R2_Subset2  TRWLS_CER_Subset2  \n",
       "0          -0.030096           0.000034         -1.098837          -0.000167  \n",
       "1          -0.091601           0.000018         -0.296251           0.000132  \n",
       "2          -0.374284           0.000164         -8.642627           0.000236  \n",
       "3          -0.040622           0.000035         -0.479505           0.000048  \n",
       "4          -0.394491           0.000269          0.010357           0.000367  \n",
       "5        -433.246292          -0.001665       -250.014122          -0.002870  \n",
       "6          -0.002792           0.000052         -2.831770          -0.000129  \n",
       "7          -0.212481          -0.000035         -0.115315           0.000092  \n",
       "8          -0.027888           0.000088         -0.457034           0.000058  \n",
       "9          -0.169243          -0.000070         -0.121301           0.000102  \n",
       "10       -197.427684          -0.000557         -9.755500          -0.000204  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "import numpy as np\n",
    "\n",
    "# Function to calculate portfolio returns, cumulative gain (CG), and CER\n",
    "def portfolio_performance(y_pred, y_true, gamma=3):\n",
    "    portfolio_returns = y_pred * y_true\n",
    "    expected_return = np.mean(portfolio_returns)\n",
    "    return_variance = np.var(portfolio_returns)\n",
    "    cer = expected_return - (gamma / 2) * return_variance\n",
    "    return cer\n",
    "\n",
    "# Function to calculate R-squared and CER\n",
    "def evaluate_model_performance(y_true, y_pred, gamma=3):\n",
    "    r_squared = r2_score(y_true, y_pred)\n",
    "    cer = portfolio_performance(y_pred, y_true, gamma)\n",
    "    return r_squared, cer\n",
    "\n",
    "# Initialize a DataFrame to store performance results for both OLS and TRWLS\n",
    "performance_results = {\n",
    "    \"Variable\": [],\n",
    "    \"OLS_R2_Subset1\": [],\n",
    "    \"OLS_CER_Subset1\": [],\n",
    "    \"OLS_R2_Subset2\": [],\n",
    "    \"OLS_CER_Subset2\": [],\n",
    "    \"TRWLS_R2_Subset1\": [],\n",
    "    \"TRWLS_CER_Subset1\": [],\n",
    "    \"TRWLS_R2_Subset2\": [],\n",
    "    \"TRWLS_CER_Subset2\": []\n",
    "}\n",
    "\n",
    "# Loop through each variable and calculate R-squared and CER for OLS and TRWLS on both subsets\n",
    "for var in variables:\n",
    "    y_true_1 = test_data_1['RET'].values\n",
    "    y_true_2 = test_data_2['RET'].values\n",
    "    \n",
    "    # OLS predictions\n",
    "    y_pred_ols_1 = ols_pred_df_1[f\"{var}_ols_pred\"].values\n",
    "    y_pred_ols_2 = ols_pred_df_2[f\"{var}_ols_pred\"].values\n",
    "    \n",
    "    # TRWLS predictions\n",
    "    y_pred_trwls_1 = trwls_pred_df_1[f\"{var}_trwls_pred\"].values\n",
    "    y_pred_trwls_2 = trwls_pred_df_2[f\"{var}_trwls_pred\"].values\n",
    "\n",
    "    # Evaluate OLS performance on both subsets\n",
    "    ols_r2_1, ols_cer_1 = evaluate_model_performance(y_true_1, y_pred_ols_1)\n",
    "    ols_r2_2, ols_cer_2 = evaluate_model_performance(y_true_2, y_pred_ols_2)\n",
    "    \n",
    "    # Evaluate TRWLS performance on both subsets\n",
    "    trwls_r2_1, trwls_cer_1 = evaluate_model_performance(y_true_1, y_pred_trwls_1)\n",
    "    trwls_r2_2, trwls_cer_2 = evaluate_model_performance(y_true_2, y_pred_trwls_2)\n",
    "\n",
    "    # Store results\n",
    "    performance_results[\"Variable\"].append(var)\n",
    "    performance_results[\"OLS_R2_Subset1\"].append(ols_r2_1)\n",
    "    performance_results[\"OLS_CER_Subset1\"].append(ols_cer_1)\n",
    "    performance_results[\"OLS_R2_Subset2\"].append(ols_r2_2)\n",
    "    performance_results[\"OLS_CER_Subset2\"].append(ols_cer_2)\n",
    "    performance_results[\"TRWLS_R2_Subset1\"].append(trwls_r2_1)\n",
    "    performance_results[\"TRWLS_CER_Subset1\"].append(trwls_cer_1)\n",
    "    performance_results[\"TRWLS_R2_Subset2\"].append(trwls_r2_2)\n",
    "    performance_results[\"TRWLS_CER_Subset2\"].append(trwls_cer_2)\n",
    "\n",
    "# Convert results to DataFrame for better readability\n",
    "performance_df_subset = pd.DataFrame(performance_results)\n",
    "\n",
    "# Display the performance results\n",
    "performance_df_subset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 考虑不同的核函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions for Epanechnikov kernel saved to 'TRWLS_Predictions_with_Epanechnikov_Kernel.csv'\n",
      "Predictions for Gaussian kernel saved to 'TRWLS_Predictions_with_Gaussian_Kernel.csv'\n",
      "Predictions for Quantic kernel saved to 'TRWLS_Predictions_with_Quantic_Kernel.csv'\n",
      "Predictions for Linear kernel saved to 'TRWLS_Predictions_with_Linear_Kernel.csv'\n",
      "Predictions for Cosine kernel saved to 'TRWLS_Predictions_with_Cosine_Kernel.csv'\n"
     ]
    }
   ],
   "source": [
    "# Function for TRWLS prediction with forward step and custom kernel, with weight normalization handling\n",
    "def trwls_predict_with_kernel(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual, kernel_function):\n",
    "    trwls_predictions = []\n",
    "    for i in range(len(test_data)):\n",
    "        X_train = train_data[[var]].iloc[:len(train_data) + i]\n",
    "        y_train = train_data['RET'].iloc[:len(train_data) + i]\n",
    "        X_test = test_data[[var]].iloc[i:i+1]\n",
    "\n",
    "        # Calculate time weights using the custom kernel\n",
    "        time_weights = calculate_time_weights_with_kernel(np.arange(len(X_train)), np.array([len(X_train)]), best_bandwidth_time, kernel_function)\n",
    "        residual_weights = calculate_residual_weights(y_train - LinearRegression().fit(X_train, y_train).predict(X_train), best_bandwidth_residual)\n",
    "        \n",
    "        # Combine weights\n",
    "        combined_weights = time_weights * residual_weights\n",
    "\n",
    "        # Handle the case where weights sum to zero or are too small\n",
    "        if np.sum(combined_weights) == 0:\n",
    "            combined_weights = np.ones_like(combined_weights)  # Use equal weights if sum is zero\n",
    "        \n",
    "        # Normalize the weights to avoid scaling issues\n",
    "        combined_weights = combined_weights / np.sum(combined_weights)\n",
    "\n",
    "        # TRWLS regression\n",
    "        model = LinearRegression()\n",
    "        model.fit(X_train, y_train, sample_weight=combined_weights)\n",
    "        \n",
    "        # Predict the next step\n",
    "        pred = model.predict(X_test)\n",
    "        trwls_predictions.append(pred[0])\n",
    "    \n",
    "    return trwls_predictions\n",
    "\n",
    "# Perform TRWLS prediction using various kernels and handle weight issues\n",
    "for var in variables:\n",
    "    for kernel_name, kernel_function in kernels.items():\n",
    "        # Perform TRWLS with each kernel\n",
    "        predictions = trwls_predict_with_kernel(train_data, test_data, var, best_bandwidths_time[var], best_bandwidths_residual[var], kernel_function)\n",
    "        \n",
    "        # Store the predictions in the corresponding DataFrame\n",
    "        kernel_pred_dfs[kernel_name][f\"{var}_{kernel_name}_pred\"] = predictions\n",
    "\n",
    "# Save predictions for each kernel to CSV\n",
    "for kernel_name, df in kernel_pred_dfs.items():\n",
    "    df.to_csv(f\"TRWLS_Predictions_with_{kernel_name.capitalize()}_Kernel.csv\")\n",
    "    print(f\"Predictions for {kernel_name.capitalize()} kernel saved to 'TRWLS_Predictions_with_{kernel_name.capitalize()}_Kernel.csv'\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               Kernel           R2       CER\n",
      "0     BM_epanechnikov    -0.369801 -0.000085\n",
      "1    LTY_epanechnikov -1932.287297 -0.018365\n",
      "2   NTIS_epanechnikov    -7.226866  0.000693\n",
      "3   INFL_epanechnikov    -0.016462  0.000016\n",
      "4   SVAR_epanechnikov    -0.083172  0.000247\n",
      "5     DP_epanechnikov  -108.999819 -0.002886\n",
      "6     DY_epanechnikov    -1.246515 -0.000231\n",
      "7    DFR_epanechnikov   -15.061932 -0.000513\n",
      "8    DFY_epanechnikov  -163.793072  0.001481\n",
      "9    LTR_epanechnikov    -3.326298  0.000148\n",
      "10    EP_epanechnikov    -0.725749  0.000289\n",
      "11        BM_gaussian    -1.380964  0.000221\n",
      "12       LTY_gaussian   -69.646618 -0.001977\n",
      "13      NTIS_gaussian    -2.557178  0.000413\n",
      "14      INFL_gaussian    -0.009975  0.000091\n",
      "15      SVAR_gaussian    -0.120641  0.000401\n",
      "16        DP_gaussian -6485.736126 -0.047308\n",
      "17        DY_gaussian    -0.092927 -0.000039\n",
      "18       DFR_gaussian    -6.855048 -0.000291\n",
      "19       DFY_gaussian    -1.503048  0.000271\n",
      "20       LTR_gaussian    -1.519428  0.000166\n",
      "21        EP_gaussian -5256.891461 -0.004871\n",
      "22         BM_quantic    -1.669073 -0.000319\n",
      "23        LTY_quantic -3548.717414 -0.029522\n",
      "24       NTIS_quantic   -13.151196  0.000936\n",
      "25       INFL_quantic     0.002795  0.000062\n",
      "26       SVAR_quantic    -0.053719  0.000349\n",
      "27         DP_quantic  -108.999819 -0.002886\n",
      "28         DY_quantic    -5.906145 -0.000464\n",
      "29        DFR_quantic   -24.012234 -0.000648\n",
      "30        DFY_quantic  -458.052423  0.000974\n",
      "31        LTR_quantic    -5.848030  0.000225\n",
      "32         EP_quantic    -0.725749  0.000289\n",
      "33          BM_linear    -0.401846 -0.000131\n",
      "34         LTY_linear -2160.578372 -0.019995\n",
      "35        NTIS_linear    -8.637521  0.000762\n",
      "36        INFL_linear     0.001598  0.000045\n",
      "37        SVAR_linear    -0.051924  0.000291\n",
      "38          DP_linear  -108.999819 -0.002886\n",
      "39          DY_linear    -1.745459 -0.000256\n",
      "40         DFR_linear   -17.703335 -0.000549\n",
      "41         DFY_linear  -221.911741  0.001477\n",
      "42         LTR_linear    -4.072227  0.000185\n",
      "43          EP_linear    -0.725749  0.000289\n",
      "44          BM_cosine    -0.385930 -0.000107\n",
      "45         LTY_cosine -1979.812370 -0.018707\n",
      "46        NTIS_cosine    -7.494473  0.000706\n",
      "47        INFL_cosine    -0.014596  0.000018\n",
      "48        SVAR_cosine    -0.075808  0.000254\n",
      "49          DP_cosine  -108.999819 -0.002886\n",
      "50          DY_cosine    -1.439266 -0.000244\n",
      "51         DFR_cosine   -16.182707 -0.000532\n",
      "52         DFY_cosine  -189.969215  0.001485\n",
      "53         LTR_cosine    -3.623588  0.000159\n",
      "54          EP_cosine    -0.725749  0.000289\n"
     ]
    }
   ],
   "source": [
    "# Function to calculate portfolio returns, cumulative gain (CG), and CER\n",
    "def portfolio_performance(y_pred, y_true, gamma=3):\n",
    "    portfolio_returns = y_pred * y_true\n",
    "    expected_return = np.mean(portfolio_returns)\n",
    "    return_variance = np.var(portfolio_returns)\n",
    "    cer = expected_return - (gamma / 2) * return_variance\n",
    "    return cer\n",
    "\n",
    "# Function to calculate R-squared and CER\n",
    "def evaluate_model_performance(y_true, y_pred, gamma=3):\n",
    "    r_squared = r2_score(y_true, y_pred)\n",
    "    cer = portfolio_performance(y_pred, y_true, gamma)\n",
    "    return r_squared, cer\n",
    "\n",
    "# Initialize a DataFrame to store performance results\n",
    "performance_results = {\n",
    "    \"Kernel\": [],\n",
    "    \"R2\": [],\n",
    "    \"CER\": []\n",
    "}\n",
    "\n",
    "# Loop through each kernel and calculate R-squared and CER\n",
    "y_true = test_data['RET'].values\n",
    "\n",
    "for kernel_name, df in kernel_pred_dfs.items():\n",
    "    for var in variables:\n",
    "        y_pred = df[f\"{var}_{kernel_name}_pred\"].values\n",
    "        \n",
    "        # Calculate R² and CER\n",
    "        r_squared, cer = evaluate_model_performance(y_true, y_pred)\n",
    "        \n",
    "        # Store results\n",
    "        performance_results[\"Kernel\"].append(f\"{var}_{kernel_name}\")\n",
    "        performance_results[\"R2\"].append(r_squared)\n",
    "        performance_results[\"CER\"].append(cer)\n",
    "\n",
    "# Convert results to a DataFrame for better readability\n",
    "performance_df = pd.DataFrame(performance_results)\n",
    "\n",
    "# Display the performance results\n",
    "print(performance_df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 考虑不同的验证集长度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         best_bandwidth_time  best_bandwidth_residual       mse\n",
      "BM   6                   5.0                      0.5  0.010512\n",
      "     9                   5.0                      0.5  0.009995\n",
      "     15                  5.0                      0.5  0.012338\n",
      "     18                  5.0                      0.5  0.018737\n",
      "TBL  6                   5.0                      0.5  0.011335\n",
      "     9                   5.0                      0.5  0.010947\n",
      "     15                  5.0                      5.0  0.010157\n",
      "     18                  5.0                      1.5  0.008779\n",
      "LTY  6                   5.0                      0.5  0.010500\n",
      "     9                   5.0                      0.5  0.011369\n",
      "     15                  5.0                      0.5  0.010852\n",
      "     18                  5.0                      0.5  0.017086\n",
      "NTIS 6                   5.0                      0.5  0.010065\n",
      "     9                   5.0                      0.5  0.010327\n",
      "     15                  5.0                      0.5  0.008724\n",
      "     18                  5.0                      0.5  0.021216\n",
      "INFL 6                   5.0                      0.5  0.011419\n",
      "     9                   5.0                      0.5  0.011800\n",
      "     15                  5.0                      0.5  0.010292\n",
      "     18                  5.0                      0.5  0.009802\n",
      "SVAR 6                   5.0                      0.5  0.011724\n",
      "     9                   2.0                      0.5  0.013776\n",
      "     15                  5.0                      0.5  0.020638\n",
      "     18                  5.0                      0.5  0.017760\n",
      "DP   6                   3.0                      5.0  0.005261\n",
      "     9                   5.0                      0.5  0.006438\n",
      "     15                  0.5                      0.5  0.010890\n",
      "     18                  5.0                      0.5  0.022149\n",
      "DY   6                   5.0                      0.5  0.011410\n",
      "     9                   5.0                      0.5  0.011304\n",
      "     15                  5.0                      0.5  0.010559\n",
      "     18                  5.0                      0.5  0.012228\n",
      "DFR  6                   5.0                      0.5  0.010654\n",
      "     9                   5.0                      0.5  0.011434\n",
      "     15                  5.0                      5.0  0.010560\n",
      "     18                  5.0                      0.5  0.011681\n",
      "DFY  6                   5.0                      0.5  0.011346\n",
      "     9                   5.0                      0.5  0.010507\n",
      "     15                  4.0                      0.5  0.009696\n",
      "     18                  5.0                      0.5  0.014353\n",
      "LTR  6                   5.0                      0.5  0.010647\n",
      "     9                   5.0                      0.5  0.011215\n",
      "     15                  5.0                      5.0  0.010068\n",
      "     18                  5.0                      5.0  0.011113\n",
      "EP   6                   3.5                      0.5  0.009227\n",
      "     9                   5.0                      0.5  0.012896\n",
      "     15                  5.0                      0.5  0.010617\n",
      "     18                  5.0                      0.5  0.017748\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "# Function to calculate time weights based on kernel function (e.g., Laplace)\n",
    "def calculate_time_weights_with_kernel(t, s, h, kernel_function):\n",
    "    u = (t - s) / h\n",
    "    return kernel_function(u)\n",
    "\n",
    "# Function to perform cross-validation for a specific bandwidth combination\n",
    "def cross_validate_bandwidth_single(train_data, var, h_time, h_residual, kernel_function, val_window, n_splits):\n",
    "    tscv = TimeSeriesSplit(n_splits=n_splits)\n",
    "    mse_list = []\n",
    "\n",
    "    for train_index, val_index in tscv.split(train_data):\n",
    "        X_train, y_train = train_data[[var]].iloc[train_index], train_data['RET'].iloc[train_index]\n",
    "        X_val, y_val = train_data[[var]].iloc[val_index], train_data['RET'].iloc[val_index]\n",
    "\n",
    "        # Fit OLS to get residuals for weighting\n",
    "        model_ols = LinearRegression()\n",
    "        model_ols.fit(X_train, y_train)\n",
    "        residuals = y_train - model_ols.predict(X_train)\n",
    "\n",
    "        # Calculate time weights and residual weights\n",
    "        time_weights = calculate_time_weights_with_kernel(np.arange(len(X_train)), np.array([len(X_train)]), h_time, kernel_function)\n",
    "        residual_weights = calculate_residual_weights(residuals, h_residual)\n",
    "        combined_weights = time_weights * residual_weights\n",
    "\n",
    "        # Ensure no division by zero in weights\n",
    "        combined_weights = np.clip(combined_weights, a_min=1e-8, a_max=None)\n",
    "\n",
    "        # Fit TRWLS model with weights\n",
    "        model_trwls = LinearRegression()\n",
    "        model_trwls.fit(X_train, y_train, sample_weight=combined_weights)\n",
    "\n",
    "        # Predict on validation set\n",
    "        y_pred_val = model_trwls.predict(X_val)\n",
    "\n",
    "        # Calculate validation MSE\n",
    "        mse = mean_squared_error(y_val, y_pred_val)\n",
    "        mse_list.append(mse)\n",
    "\n",
    "    return np.mean(mse_list)\n",
    "\n",
    "# Function to parallelize cross-validation for different bandwidths\n",
    "def cross_validate_bandwidth(train_data, var, bandwidths_time, bandwidths_residual, kernel_function, val_window, n_splits=5):\n",
    "    results = Parallel(n_jobs=-1)(delayed(cross_validate_bandwidth_single)(train_data, var, h_time, h_residual, kernel_function, val_window, n_splits)\n",
    "                                  for h_time in bandwidths_time\n",
    "                                  for h_residual in bandwidths_residual)\n",
    "\n",
    "    min_mse_idx = np.argmin(results)\n",
    "    best_h_time = bandwidths_time[min_mse_idx // len(bandwidths_residual)]\n",
    "    best_h_residual = bandwidths_residual[min_mse_idx % len(bandwidths_residual)]\n",
    "    return best_h_time, best_h_residual, np.min(results)\n",
    "\n",
    "# Define kernel function (you can use Laplace or other kernels as needed)\n",
    "def laplace_kernel(u):\n",
    "    return np.exp(-np.abs(u)) / 2\n",
    "\n",
    "# Define reduced bandwidth ranges for time and residuals (reduce size to speed up)\n",
    "bandwidths_time = np.linspace(0.5, 5, 10)  # Reduced range for time weights\n",
    "bandwidths_residual = np.linspace(0.5, 5, 10)  # Reduced range for residual weights\n",
    "\n",
    "# Validation window sizes (6 months, 9 months, 15 months, 18 months)\n",
    "validation_windows = [6, 9, 15, 18]\n",
    "\n",
    "# Initialize dictionary to store the best bandwidths for each validation window\n",
    "best_bandwidths = {}\n",
    "\n",
    "# Loop through each variable and validation window, and perform cross-validation to find the best bandwidths\n",
    "for var in variables_without_tbl:\n",
    "    best_bandwidths[var] = {}\n",
    "    for val_window in validation_windows:\n",
    "        n_splits = len(train_data) // val_window  # Reduce number of splits\n",
    "        best_bandwidth_time, best_bandwidth_residual, best_mse = cross_validate_bandwidth(train_data, var, bandwidths_time, bandwidths_residual, laplace_kernel, val_window, n_splits)\n",
    "        best_bandwidths[var][val_window] = {\"best_bandwidth_time\": best_bandwidth_time, \"best_bandwidth_residual\": best_bandwidth_residual, \"mse\": best_mse}\n",
    "\n",
    "# Display best bandwidths for each variable and validation window\n",
    "best_bandwidths_df = pd.DataFrame.from_dict({(i,j): best_bandwidths[i][j] \n",
    "                                             for i in best_bandwidths.keys() \n",
    "                                             for j in best_bandwidths[i].keys()},\n",
    "                                            orient='index')\n",
    "\n",
    "print(best_bandwidths_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions for 6-month window saved to 'TRWLS_Predictions_6_Months.csv'\n",
      "Predictions for 9-month window saved to 'TRWLS_Predictions_9_Months.csv'\n",
      "Predictions for 15-month window saved to 'TRWLS_Predictions_15_Months.csv'\n",
      "Predictions for 18-month window saved to 'TRWLS_Predictions_18_Months.csv'\n"
     ]
    }
   ],
   "source": [
    "# Function for TRWLS prediction with forward step and optimal bandwidths\n",
    "def trwls_predict_with_optimal_bandwidth(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual):\n",
    "    trwls_predictions = []\n",
    "    for i in range(len(test_data)):\n",
    "        X_train = train_data[[var]].iloc[:len(train_data) + i]\n",
    "        y_train = train_data['RET'].iloc[:len(train_data) + i]\n",
    "        X_test = test_data[[var]].iloc[i:i+1]\n",
    "\n",
    "        # Calculate time weights using the optimal bandwidth\n",
    "        time_weights = calculate_time_weights_with_kernel(np.arange(len(X_train)), np.array([len(X_train)]), best_bandwidth_time, laplace_kernel)\n",
    "        residual_weights = calculate_residual_weights(y_train - LinearRegression().fit(X_train, y_train).predict(X_train), best_bandwidth_residual)\n",
    "        combined_weights = time_weights * residual_weights\n",
    "\n",
    "        # Ensure weights are valid\n",
    "        combined_weights = np.clip(combined_weights, a_min=1e-8, a_max=None)\n",
    "        combined_weights = combined_weights / np.sum(combined_weights)  # Normalize the weights\n",
    "\n",
    "        # TRWLS regression\n",
    "        model = LinearRegression()\n",
    "        model.fit(X_train, y_train, sample_weight=combined_weights)\n",
    "\n",
    "        # Predict the next step\n",
    "        pred = model.predict(X_test)\n",
    "        trwls_predictions.append(pred[0])\n",
    "    \n",
    "    return trwls_predictions\n",
    "\n",
    "# Initialize a DataFrame to store TRWLS predictions for all validation windows\n",
    "trwls_pred_dfs = {\n",
    "    6: pd.DataFrame(index=test_data.index),\n",
    "    9: pd.DataFrame(index=test_data.index),\n",
    "    15: pd.DataFrame(index=test_data.index),\n",
    "    18: pd.DataFrame(index=test_data.index)\n",
    "}\n",
    "\n",
    "# Loop through each variable and validation window to perform TRWLS with optimal bandwidths\n",
    "for var in variables:\n",
    "    for val_window in [6, 9, 15, 18]:\n",
    "        best_bandwidth_time = best_bandwidths[var][val_window][\"best_bandwidth_time\"]\n",
    "        best_bandwidth_residual = best_bandwidths[var][val_window][\"best_bandwidth_residual\"]\n",
    "        \n",
    "        # Perform TRWLS using the optimal bandwidths\n",
    "        predictions = trwls_predict_with_optimal_bandwidth(train_data, test_data, var, best_bandwidth_time, best_bandwidth_residual)\n",
    "        \n",
    "        # Store the predictions in the corresponding DataFrame for each window\n",
    "        trwls_pred_dfs[val_window][f\"{var}_trwls_pred_{val_window}\"] = predictions\n",
    "\n",
    "# Save the TRWLS predictions for each validation window to CSV files\n",
    "for val_window, df in trwls_pred_dfs.items():\n",
    "    df.to_csv(f\"TRWLS_Predictions_{val_window}_Months.csv\")\n",
    "    print(f\"Predictions for {val_window}-month window saved to 'TRWLS_Predictions_{val_window}_Months.csv'\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance evaluation saved to 'TRWLS_Prediction_Performance.csv'\n",
      "   Variable Validation_Window           R2       CER\n",
      "0        BM          6 Months    -0.395126  0.000106\n",
      "1       LTY          6 Months  -327.744998 -0.005199\n",
      "2      NTIS          6 Months    -3.117175  0.000474\n",
      "3      INFL          6 Months    -0.006219  0.000029\n",
      "4      SVAR          6 Months    -0.089722  0.000395\n",
      "5        DP          6 Months -4164.811708 -0.033358\n",
      "6        DY          6 Months    -0.378869 -0.000099\n",
      "7       DFR          6 Months   -11.606559 -0.000400\n",
      "8       DFY          6 Months    -3.446507  0.000388\n",
      "9       LTR          6 Months    -2.703108  0.000200\n",
      "10       EP          6 Months   -10.170415  0.000737\n",
      "11       BM          9 Months    -0.395126  0.000106\n",
      "12      LTY          9 Months  -327.744998 -0.005199\n",
      "13     NTIS          9 Months    -3.117175  0.000474\n",
      "14     INFL          9 Months    -0.006219  0.000029\n",
      "15     SVAR          9 Months    -0.322978  0.000534\n",
      "16       DP          9 Months -2614.826757 -0.023417\n",
      "17       DY          9 Months    -0.378869 -0.000099\n",
      "18      DFR          9 Months   -11.606559 -0.000400\n",
      "19      DFY          9 Months    -3.446507  0.000388\n",
      "20      LTR          9 Months    -2.703108  0.000200\n",
      "21       EP          9 Months   -25.746230 -0.001302\n",
      "22       BM         15 Months    -0.395126  0.000106\n",
      "23      LTY         15 Months  -327.744998 -0.005199\n",
      "24     NTIS         15 Months    -3.117175  0.000474\n",
      "25     INFL         15 Months    -0.006219  0.000029\n",
      "26     SVAR         15 Months    -0.089722  0.000395\n",
      "27       DP         15 Months -6268.827549 -0.046034\n",
      "28       DY         15 Months    -0.378869 -0.000099\n",
      "29      DFR         15 Months   -12.459192 -0.000423\n",
      "30      DFY         15 Months   -11.445209  0.000665\n",
      "31      LTR         15 Months    -2.865072  0.000197\n",
      "32       EP         15 Months   -25.746230 -0.001302\n",
      "33       BM         18 Months    -0.395126  0.000106\n",
      "34      LTY         18 Months  -327.744998 -0.005199\n",
      "35     NTIS         18 Months    -3.117175  0.000474\n",
      "36     INFL         18 Months    -0.006219  0.000029\n",
      "37     SVAR         18 Months    -0.089722  0.000395\n",
      "38       DP         18 Months -2614.826757 -0.023417\n",
      "39       DY         18 Months    -0.378869 -0.000099\n",
      "40      DFR         18 Months   -11.606559 -0.000400\n",
      "41      DFY         18 Months    -3.446507  0.000388\n",
      "42      LTR         18 Months    -2.865072  0.000197\n",
      "43       EP         18 Months   -25.746230 -0.001302\n"
     ]
    }
   ],
   "source": [
    "# Function to calculate portfolio returns, cumulative gain (CG), and CER\n",
    "def portfolio_performance(y_pred, y_true, gamma=3):\n",
    "    portfolio_returns = y_pred * y_true\n",
    "    expected_return = np.mean(portfolio_returns)\n",
    "    return_variance = np.var(portfolio_returns)\n",
    "    cer = expected_return - (gamma / 2) * return_variance\n",
    "    return cer\n",
    "\n",
    "# Function to calculate R-squared and CER\n",
    "def evaluate_model_performance(y_true, y_pred, gamma=3):\n",
    "    r_squared = r2_score(y_true, y_pred)\n",
    "    cer = portfolio_performance(y_pred, y_true, gamma)\n",
    "    return r_squared, cer\n",
    "\n",
    "# Initialize a DataFrame to store performance results for each validation window\n",
    "performance_results = {\n",
    "    \"Variable\": [],\n",
    "    \"Validation_Window\": [],\n",
    "    \"R2\": [],\n",
    "    \"CER\": []\n",
    "}\n",
    "\n",
    "# Loop through each variable and validation window to calculate R² and CER\n",
    "y_true = test_data['RET'].values\n",
    "\n",
    "for val_window, df in trwls_pred_dfs.items():\n",
    "    for var in variables:\n",
    "        y_pred = df[f\"{var}_trwls_pred_{val_window}\"].values\n",
    "        \n",
    "        # Calculate R² and CER\n",
    "        r_squared, cer = evaluate_model_performance(y_true, y_pred)\n",
    "        \n",
    "        # Store results\n",
    "        performance_results[\"Variable\"].append(var)\n",
    "        performance_results[\"Validation_Window\"].append(f\"{val_window} Months\")\n",
    "        performance_results[\"R2\"].append(r_squared)\n",
    "        performance_results[\"CER\"].append(cer)\n",
    "\n",
    "# Convert results to a DataFrame for better readability\n",
    "performance_df = pd.DataFrame(performance_results)\n",
    "\n",
    "# Save the performance results to a CSV file\n",
    "performance_df.to_csv(\"TRWLS_Prediction_Performance.csv\", index=False)\n",
    "print(\"Performance evaluation saved to 'TRWLS_Prediction_Performance.csv'\")\n",
    "\n",
    "# Display the performance results\n",
    "print(performance_df)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}