04_generate_all_simulation_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate all simulation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    x *= 2\n",
    "    return np.exp(x) / (np.exp(x) + np.exp(-x))\n",
    "\n",
    "def permute(X, noisy_features):\n",
    "    tmp = X.copy()\n",
    "    for j in range(X.shape[1]):\n",
    "        if noisy_features[j] == 1:\n",
    "            tmp[:, j] = np.random.permutation(tmp[:, j])\n",
    "    return tmp\n",
    "\n",
    "def f(X, noisy_features):\n",
    "    probs = sigmoid(np.mean(X[:, noisy_features == 0], 1))\n",
    "    return np.array([np.random.choice([0, 1], 1, p=[1 - prob, prob]) for prob in probs]).flatten()\n",
    "\n",
    "for ind in range(40):\n",
    "    \n",
    "    X_train = np.array(pd.read_csv('02_enhancer/X_train.csv', header=0, index_col=0))\n",
    "    X_test = np.array(pd.read_csv('02_enhancer/X_test.csv', header=0, index_col=0))\n",
    "    y_train = np.array(pd.read_csv('02_enhancer/y_train.csv', header=0, index_col=0))\n",
    "    y_test = np.array(pd.read_csv('02_enhancer/y_test.csv', header=0, index_col=0))\n",
    "    n, m = X_train.shape\n",
    "    names = np.arange(m)\n",
    "\n",
    "    n_features = X_train.shape[1]\n",
    "    n, m = X_train.shape\n",
    "    names = np.arange(m)\n",
    "    noisy_features = np.ones((n_features, ), dtype=int)\n",
    "    noisy_features[np.random.choice(range(n_features), 5, replace=False)] = 0\n",
    "    scaler = MinMaxScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    X_train = permute(X_train, noisy_features)\n",
    "    X_test = permute(X_test, noisy_features)\n",
    "    y_train = f(X_train, noisy_features)\n",
    "    y_test = f(X_test, noisy_features)\n",
    "\n",
    "    np.savetxt('04_aggregate/classification1/permuted{}_X_train.csv'.format(ind), X_train, delimiter=',')\n",
    "    np.savetxt('04_aggregate/classification1/permuted{}_y_train.csv'.format(ind), y_train, delimiter=',')\n",
    "    np.savetxt('04_aggregate/classification1/permuted{}_X_test.csv'.format(ind), X_test, delimiter=',')\n",
    "    np.savetxt('04_aggregate/classification1/permuted{}_y_test.csv'.format(ind), y_test, delimiter=',')\n",
    "    np.savetxt('04_aggregate/classification1/permuted{}_noisy_features.csv'.format(ind), noisy_features, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(X, noisy_features):\n",
    "    y = np.mean(X[:, noisy_features == 0], 1) \n",
    "    y += np.random.normal(0, np.std(y) ** 2 * 100, y.shape)\n",
    "    return y\n",
    "\n",
    "for ind in range(40):\n",
    "    # ### load data\n",
    "    X_train = np.array(pd.read_csv('02_enhancer/X_train.csv', header=0, index_col=0))\n",
    "    X_test = np.array(pd.read_csv('02_enhancer/X_test.csv', header=0, index_col=0))\n",
    "    y_train = np.array(pd.read_csv('02_enhancer/y_train.csv', header=0, index_col=0))\n",
    "    y_test = np.array(pd.read_csv('02_enhancer/y_test.csv', header=0, index_col=0))\n",
    "    n, m = X_train.shape\n",
    "    names = np.arange(m)\n",
    "\n",
    "    n_features = X_train.shape[1]\n",
    "    n, m = X_train.shape\n",
    "    names = np.arange(m)\n",
    "    scaler = MinMaxScaler()\n",
    "    noisy_features = np.ones((n_features, ), dtype=int)\n",
    "    noisy_features[np.random.choice(range(n_features), 5, replace=False)] = 0\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    X_train = permute(X_train, noisy_features)\n",
    "    X_test = permute(X_test, noisy_features)\n",
    "    y_train = f(X_train, noisy_features)\n",
    "    y_test = f(X_test, noisy_features)\n",
    "    \n",
    "    np.savetxt('04_aggregate/regression1/permuted{}_X_train.csv'.format(ind), X_train, delimiter=',')\n",
    "    np.savetxt('04_aggregate/regression1/permuted{}_y_train.csv'.format(ind), y_train, delimiter=',')\n",
    "    np.savetxt('04_aggregate/regression1/permuted{}_X_test.csv'.format(ind), X_test, delimiter=',')\n",
    "    np.savetxt('04_aggregate/regression1/permuted{}_y_test.csv'.format(ind), y_test, delimiter=',')\n",
    "    np.savetxt('04_aggregate/regression1/permuted{}_noisy_features.csv'.format(ind), noisy_features, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    x *= 1\n",
    "    return np.exp(x) / (np.exp(x) + np.exp(-x))\n",
    "\n",
    "def f(X, noisy_features):\n",
    "    probs = sigmoid(np.mean(X[:, noisy_features == 0], 1))\n",
    "    return np.array([np.random.choice([0, 1], 1, p=[1 - prob, prob]) for prob in probs]).flatten()\n",
    "\n",
    "n_samples = 1000\n",
    "n_features = 50\n",
    "\n",
    "for ind in range(40):\n",
    "    \n",
    "    X_train = np.zeros((n_samples, n_features))\n",
    "    X_test = np.zeros((n_samples, n_features))\n",
    "    for i in range(n_features):\n",
    "        X_train[:, i] = np.random.choice(list(range(i+2)), n_samples)\n",
    "        X_test[:, i] = np.random.choice(list(range(i+2)), n_samples)\n",
    "    \n",
    "    noisy_features = np.ones((n_features, ), dtype=int)\n",
    "    noisy_features[np.random.choice(range(n_features//5), 5, replace=False)] = 0\n",
    "    scaler = MinMaxScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    y_train = f(X_train, noisy_features)\n",
    "    y_test = f(X_test, noisy_features)\n",
    "\n",
    "    np.savetxt('04_aggregate/classification2/permuted{}_X_train.csv'.format(ind), X_train, delimiter=',', fmt='%10.5f')\n",
    "    np.savetxt('04_aggregate/classification2/permuted{}_y_train.csv'.format(ind), y_train, delimiter=',', fmt='%d')\n",
    "    np.savetxt('04_aggregate/classification2/permuted{}_X_test.csv'.format(ind), X_test, delimiter=',', fmt='%10.5f')\n",
    "    np.savetxt('04_aggregate/classification2/permuted{}_y_test.csv'.format(ind), y_test, delimiter=',', fmt='%d')\n",
    "    np.savetxt('04_aggregate/classification2/permuted{}_noisy_features.csv'.format(ind), noisy_features, delimiter=',', fmt='%d')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(X, noisy_features):\n",
    "    y = np.mean(X[:, noisy_features == 0], 1) \n",
    "    y += np.random.normal(0, np.std(y) ** 2 * 100, y.shape)\n",
    "    return y\n",
    "\n",
    "n_samples = 1000\n",
    "n_features = 50\n",
    "\n",
    "for ind in range(40):\n",
    "    # ### load data\n",
    "    X_train = np.zeros((n_samples, n_features))\n",
    "    X_test = np.zeros((n_samples, n_features))\n",
    "    for i in range(n_features):\n",
    "        X_train[:, i] = np.random.choice(list(range(i+2)), n_samples)\n",
    "        X_test[:, i] = np.random.choice(list(range(i+2)), n_samples)\n",
    "\n",
    "    scaler = MinMaxScaler()\n",
    "    noisy_features = np.ones((n_features, ), dtype=int)\n",
    "    noisy_features[np.random.choice(range(n_features//5), 5, replace=False)] = 0\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    X_train = permute(X_train, noisy_features)\n",
    "    X_test = permute(X_test, noisy_features)\n",
    "    y_train = f(X_train, noisy_features)\n",
    "    y_test = f(X_test, noisy_features)\n",
    "    \n",
    "    np.savetxt('04_aggregate/regression2/permuted{}_X_train.csv'.format(ind), X_train, delimiter=',', fmt='%10.5f')\n",
    "    np.savetxt('04_aggregate/regression2/permuted{}_y_train.csv'.format(ind), y_train, delimiter=',', fmt='%10.5f')\n",
    "    np.savetxt('04_aggregate/regression2/permuted{}_X_test.csv'.format(ind), X_test, delimiter=',', fmt='%10.5f')\n",
    "    np.savetxt('04_aggregate/regression2/permuted{}_y_test.csv'.format(ind), y_test, delimiter=',', fmt='%10.5f')\n",
    "    np.savetxt('04_aggregate/regression2/permuted{}_noisy_features.csv'.format(ind), noisy_features, delimiter=',', fmt='%d')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}