Imbalanced_ssfd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-17 17:56:18.139824: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-05-17 17:56:18.160929: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-05-17 17:56:18.464972: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import errno\n",
    "import argparse\n",
    "import sys\n",
    "import pickle\n",
    "\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "from data_utils import load_MNIST_data, load_EMNIST_data, generate_EMNIST_writer_based_data, generate_partial_data\n",
    "from skd import FedMD\n",
    "from Neural_Networks import train_models, cnn_2layer_fc_model, cnn_3layer_fc_model\n",
    "\n",
    "from tensorflow.keras.models import clone_model\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "import tensorflow as tf\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_file = os.path.abspath(\"conf/EMNIST_imbalance_conf.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def poisoning(targets):\n",
    "\n",
    "    for idx in range(len(targets)):\n",
    "        if targets[idx] == 10:\n",
    "            targets[idx] = 14\n",
    "        elif targets[idx] == 11:\n",
    "            targets[idx] = 12\n",
    "        elif targets[idx] == 12:\n",
    "            targets[idx] = 11\n",
    "        elif targets[idx] == 13:\n",
    "            targets[idx] = 15\n",
    "        elif targets[idx] == 14:\n",
    "            targets[idx] = 10\n",
    "        elif targets[idx] == 15:\n",
    "            targets[idx] = 13\n",
    "\n",
    "    return targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "CANDIDATE_MODELS = {\"2_layer_CNN\": cnn_2layer_fc_model, \n",
    "                    \"3_layer_CNN\": cnn_3layer_fc_model} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(conf_file, \"r\") as f:\n",
    "    conf_dict = eval(f.read())\n",
    "    \n",
    "    #n_classes = conf_dict[\"n_classes\"]\n",
    "    model_config = conf_dict[\"models\"]\n",
    "    pre_train_params = conf_dict[\"pre_train_params\"]\n",
    "    model_saved_dir = conf_dict[\"model_saved_dir\"]\n",
    "    model_saved_names = conf_dict[\"model_saved_names\"]\n",
    "    is_early_stopping = conf_dict[\"early_stopping\"]\n",
    "    public_classes = conf_dict[\"public_classes\"]\n",
    "    private_classes = conf_dict[\"private_classes\"]\n",
    "    n_classes = len(public_classes) + len(private_classes)\n",
    "    \n",
    "    emnist_data_dir = conf_dict[\"EMNIST_dir\"]    \n",
    "    N_parties = conf_dict[\"N_parties\"]\n",
    "    # N_samples_per_class = conf_dict[\"N_samples_per_class\"]\n",
    "    N_samples_per_class = 5  # ssfd\n",
    "    \n",
    "    N_rounds = conf_dict[\"N_rounds\"]\n",
    "    N_alignment = conf_dict[\"N_alignment\"]\n",
    "    N_private_training_round = conf_dict[\"N_private_training_round\"]\n",
    "    private_training_batchsize = conf_dict[\"private_training_batchsize\"]\n",
    "    N_logits_matching_round = conf_dict[\"N_logits_matching_round\"]\n",
    "    logits_matching_batchsize = conf_dict[\"logits_matching_batchsize\"]\n",
    "    \n",
    "    \n",
    "    result_save_dir = conf_dict[\"result_save_dir\"]\n",
    "\n",
    "\n",
    "del conf_dict, conf_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MNIST dataset ... \n",
      "X_train shape : (60000, 28, 28)\n",
      "X_test shape : (10000, 28, 28)\n",
      "y_train shape : (60000,)\n",
      "y_test shape : (10000,)\n",
      "EMNIST-letter dataset ... \n",
      "X_train shape : (124800, 28, 28)\n",
      "X_test shape : (20800, 28, 28)\n",
      "y_train shape : (124800,)\n",
      "y_test shape : (20800,)\n"
     ]
    }
   ],
   "source": [
    "X_train_MNIST, y_train_MNIST, X_test_MNIST, y_test_MNIST \\\n",
    "= load_MNIST_data(standarized = True, verbose = True)\n",
    "\n",
    "public_dataset = {\"X\": X_train_MNIST, \"y\": y_train_MNIST}\n",
    "\n",
    "\n",
    "X_train_EMNIST, y_train_EMNIST, X_test_EMNIST, y_test_EMNIST, \\\n",
    "writer_ids_train_EMNIST, writer_ids_test_EMNIST \\\n",
    "= load_EMNIST_data(emnist_data_dir,\n",
    "                    standarized = True, verbose = True)\n",
    "\n",
    "y_train_EMNIST += len(public_classes)\n",
    "y_test_EMNIST += len(public_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(124800, 28, 28)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_EMNIST.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  1,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,\n",
       "         15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,\n",
       "         28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,\n",
       "         41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,  52,  53,\n",
       "         54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,  65,  66,\n",
       "         67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,  79,\n",
       "         80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  92,  93,\n",
       "         94,  95,  96, 104, 121]),\n",
       " array([  3,   2,   3,   7,   3,  10,  14,  13,  11,  44,  34,  56,  67,\n",
       "        135, 159, 192, 205, 213, 236, 203, 174, 121, 109,  61,  49,  23,\n",
       "         17,  15,   6,   9,  12,  10,  15,  19,   9,  19,  14,  21,  24,\n",
       "         21,  11,  14,  19,  22,  30,  27,  19,  18,  30,  31,  38,  29,\n",
       "         28,  33,  34,  45,  34,  34,  41,  35,  38,  43,  33,  37,  38,\n",
       "         36,  29,  42,  38,  36,  35,  34,  26,  17,  25,  24,  23,  13,\n",
       "         13,  11,   9,  19,   7,   9,   2,   4,   2,   2,   1,   1,   1,\n",
       "          2,   2,   1,   1,   1]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(np.unique(writer_ids_train_EMNIST, return_counts=True)[1], return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot take a larger sample than population when 'replace=False'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[39m#generate private data\u001b[39;00m\n\u001b[1;32m      2\u001b[0m private_data, total_private_data\\\n\u001b[0;32m----> 3\u001b[0m \u001b[39m=\u001b[39mgenerate_EMNIST_writer_based_data(X_train_EMNIST, y_train_EMNIST,\n\u001b[1;32m      4\u001b[0m                                     writer_ids_train_EMNIST,\n\u001b[1;32m      5\u001b[0m                                     N_parties \u001b[39m=\u001b[39;49m N_parties, \n\u001b[1;32m      6\u001b[0m                                     classes_in_use \u001b[39m=\u001b[39;49m private_classes, \n\u001b[1;32m      7\u001b[0m                                     N_priv_data_min \u001b[39m=\u001b[39;49m N_samples_per_class \u001b[39m*\u001b[39;49m \u001b[39mlen\u001b[39;49m(private_classes)\n\u001b[1;32m      8\u001b[0m                                     )\n\u001b[1;32m     10\u001b[0m X_tmp, y_tmp \u001b[39m=\u001b[39m generate_partial_data(X \u001b[39m=\u001b[39m X_test_EMNIST, y\u001b[39m=\u001b[39m y_test_EMNIST, \n\u001b[1;32m     11\u001b[0m                                         class_in_use \u001b[39m=\u001b[39m private_classes, verbose \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m     12\u001b[0m private_test_data \u001b[39m=\u001b[39m {\u001b[39m\"\u001b[39m\u001b[39mX\u001b[39m\u001b[39m\"\u001b[39m: X_tmp, \u001b[39m\"\u001b[39m\u001b[39my\u001b[39m\u001b[39m\"\u001b[39m: y_tmp}\n",
      "File \u001b[0;32m~/Documents/other/FedMD_clean/data_utils.py:289\u001b[0m, in \u001b[0;36mgenerate_EMNIST_writer_based_data\u001b[0;34m(X, y, writer_info, N_priv_data_min, N_parties, classes_in_use)\u001b[0m\n\u001b[1;32m    284\u001b[0m         data_by_writer[wt_id] \u001b[39m=\u001b[39m {\u001b[39m\"\u001b[39m\u001b[39mX\u001b[39m\u001b[39m\"\u001b[39m: X[idx], \u001b[39m\"\u001b[39m\u001b[39my\u001b[39m\u001b[39m\"\u001b[39m: y[idx], \n\u001b[1;32m    285\u001b[0m                                  \u001b[39m\"\u001b[39m\u001b[39midx\u001b[39m\u001b[39m\"\u001b[39m: idx, \u001b[39m\"\u001b[39m\u001b[39mwriter_id\u001b[39m\u001b[39m\"\u001b[39m: wt_id}\n\u001b[1;32m    287\u001b[0m \u001b[39m# each participant in the collaborative group is assigned data \u001b[39;00m\n\u001b[1;32m    288\u001b[0m \u001b[39m# from a single writer.\u001b[39;00m\n\u001b[0;32m--> 289\u001b[0m ids_to_use \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39;49mrandom\u001b[39m.\u001b[39;49mchoice(writer_ids, size \u001b[39m=\u001b[39;49m N_parties, replace \u001b[39m=\u001b[39;49m \u001b[39mFalse\u001b[39;49;00m)\n\u001b[1;32m    290\u001b[0m combined_idx \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39marray([], dtype \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mint64)\n\u001b[1;32m    291\u001b[0m private_data \u001b[39m=\u001b[39m []\n",
      "File \u001b[0;32mmtrand.pyx:965\u001b[0m, in \u001b[0;36mnumpy.random.mtrand.RandomState.choice\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot take a larger sample than population when 'replace=False'"
     ]
    }
   ],
   "source": [
    "#generate private data\n",
    "private_data, total_private_data\\\n",
    "=generate_EMNIST_writer_based_data(X_train_EMNIST, y_train_EMNIST,\n",
    "                                    writer_ids_train_EMNIST,\n",
    "                                    N_parties = N_parties, \n",
    "                                    classes_in_use = private_classes, \n",
    "                                    N_priv_data_min = N_samples_per_class * len(private_classes)\n",
    "                                    )\n",
    "\n",
    "X_tmp, y_tmp = generate_partial_data(X = X_test_EMNIST, y= y_test_EMNIST, \n",
    "                                        class_in_use = private_classes, verbose = True)\n",
    "private_test_data = {\"X\": X_tmp, \"y\": y_tmp}\n",
    "del X_tmp, y_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 28, 28)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "private_data[0][\"X\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4800, 28, 28)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "private_test_data[\"X\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15 15 12 14 11 14 10 10 10 15 10 10 11 15 12 10 15 10 10 10]\n",
      "[10 15 10 12 12 14 10 13 10 11 13 14 13 15 11 12 15 11 14 15 14 11 15]\n",
      "[14 15 10 10 13 15 15 13 10 15 13 11 12 13 10 12 12 14 10 11]\n",
      "[10 10 10 12 10 15 14 14 13 10 13 10 15 14 10 13 15 15 12 12 10 11 15 11\n",
      " 11]\n"
     ]
    }
   ],
   "source": [
    "# Poisoning\n",
    "\n",
    "poison_devices = [0,1,2,3]\n",
    "\n",
    "for device in poison_devices:\n",
    "    private_data[device]['y'] = poisoning(private_data[device]['y'])\n",
    "    print(private_data[device]['y'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:54:48.337458: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.352534: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.352609: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.353466: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.353532: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.353573: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.652743: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.652831: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.652880: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2023-05-12 08:54:48.652924: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1635] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 9778 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3080 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 0 : CNN_128_256\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape (Reshape)           (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d (Conv2D)             (None, 28, 28, 128)       1280      \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 28, 28, 128)      512       \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " activation (Activation)     (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d (AverageP  (None, 28, 28, 128)      0         \n",
      " ooling2D)                                                       \n",
      "                                                                 \n",
      " conv2d_1 (Conv2D)           (None, 13, 13, 256)       295168    \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 13, 13, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_1 (Activation)   (None, 13, 13, 256)       0         \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 13, 13, 256)       0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 43264)             0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 16)                692224    \n",
      "                                                                 \n",
      " activation_2 (Activation)   (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 990,208\n",
      "Trainable params: 989,440\n",
      "Non-trainable params: 768\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "model 1 : CNN_128_384\n",
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_2 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_1 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_2 (Conv2D)           (None, 28, 28, 128)       1280      \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 28, 28, 128)      512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_3 (Activation)   (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_1 (Averag  (None, 28, 28, 128)      0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_3 (Conv2D)           (None, 13, 13, 384)       442752    \n",
      "                                                                 \n",
      " batch_normalization_3 (Batc  (None, 13, 13, 384)      1536      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_4 (Activation)   (None, 13, 13, 384)       0         \n",
      "                                                                 \n",
      " dropout_3 (Dropout)         (None, 13, 13, 384)       0         \n",
      "                                                                 \n",
      " flatten_1 (Flatten)         (None, 64896)             0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 16)                1038336   \n",
      "                                                                 \n",
      " activation_5 (Activation)   (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,484,416\n",
      "Trainable params: 1,483,392\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "model 2 : CNN_128_512\n",
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_3 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_2 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_4 (Conv2D)           (None, 28, 28, 128)       1280      \n",
      "                                                                 \n",
      " batch_normalization_4 (Batc  (None, 28, 28, 128)      512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_6 (Activation)   (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " dropout_4 (Dropout)         (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_2 (Averag  (None, 28, 28, 128)      0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_5 (Conv2D)           (None, 13, 13, 512)       590336    \n",
      "                                                                 \n",
      " batch_normalization_5 (Batc  (None, 13, 13, 512)      2048      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_7 (Activation)   (None, 13, 13, 512)       0         \n",
      "                                                                 \n",
      " dropout_5 (Dropout)         (None, 13, 13, 512)       0         \n",
      "                                                                 \n",
      " flatten_2 (Flatten)         (None, 86528)             0         \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 16)                1384448   \n",
      "                                                                 \n",
      " activation_8 (Activation)   (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,978,624\n",
      "Trainable params: 1,977,344\n",
      "Non-trainable params: 1,280\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "model 3 : CNN_256_256\n",
      "Model: \"model_3\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_4 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_3 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_6 (Conv2D)           (None, 28, 28, 256)       2560      \n",
      "                                                                 \n",
      " batch_normalization_6 (Batc  (None, 28, 28, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_9 (Activation)   (None, 28, 28, 256)       0         \n",
      "                                                                 \n",
      " dropout_6 (Dropout)         (None, 28, 28, 256)       0         \n",
      "                                                                 \n",
      " average_pooling2d_3 (Averag  (None, 28, 28, 256)      0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_7 (Conv2D)           (None, 13, 13, 256)       590080    \n",
      "                                                                 \n",
      " batch_normalization_7 (Batc  (None, 13, 13, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_10 (Activation)  (None, 13, 13, 256)       0         \n",
      "                                                                 \n",
      " dropout_7 (Dropout)         (None, 13, 13, 256)       0         \n",
      "                                                                 \n",
      " flatten_3 (Flatten)         (None, 43264)             0         \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 16)                692224    \n",
      "                                                                 \n",
      " activation_11 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,286,912\n",
      "Trainable params: 1,285,888\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "model 4 : CNN_256_512\n",
      "Model: \"model_4\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_5 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_4 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_8 (Conv2D)           (None, 28, 28, 256)       2560      \n",
      "                                                                 \n",
      " batch_normalization_8 (Batc  (None, 28, 28, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_12 (Activation)  (None, 28, 28, 256)       0         \n",
      "                                                                 \n",
      " dropout_8 (Dropout)         (None, 28, 28, 256)       0         \n",
      "                                                                 \n",
      " average_pooling2d_4 (Averag  (None, 28, 28, 256)      0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_9 (Conv2D)           (None, 13, 13, 512)       1180160   \n",
      "                                                                 \n",
      " batch_normalization_9 (Batc  (None, 13, 13, 512)      2048      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " activation_13 (Activation)  (None, 13, 13, 512)       0         \n",
      "                                                                 \n",
      " dropout_9 (Dropout)         (None, 13, 13, 512)       0         \n",
      "                                                                 \n",
      " flatten_4 (Flatten)         (None, 86528)             0         \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 16)                1384448   \n",
      "                                                                 \n",
      " activation_14 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 2,570,240\n",
      "Trainable params: 2,568,704\n",
      "Non-trainable params: 1,536\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "model 5 : CNN_64_128_256\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_6 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_5 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_10 (Conv2D)          (None, 28, 28, 64)        640       \n",
      "                                                                 \n",
      " batch_normalization_10 (Bat  (None, 28, 28, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_15 (Activation)  (None, 28, 28, 64)        0         \n",
      "                                                                 \n",
      " dropout_10 (Dropout)        (None, 28, 28, 64)        0         \n",
      "                                                                 \n",
      " average_pooling2d_5 (Averag  (None, 28, 28, 64)       0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_11 (Conv2D)          (None, 14, 14, 128)       32896     \n",
      "                                                                 \n",
      " batch_normalization_11 (Bat  (None, 14, 14, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_16 (Activation)  (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " dropout_11 (Dropout)        (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_6 (Averag  (None, 7, 7, 128)        0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_12 (Conv2D)          (None, 3, 3, 256)         295168    \n",
      "                                                                 \n",
      " batch_normalization_12 (Bat  (None, 3, 3, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_17 (Activation)  (None, 3, 3, 256)         0         \n",
      "                                                                 \n",
      " dropout_12 (Dropout)        (None, 3, 3, 256)         0         \n",
      "                                                                 \n",
      " flatten_5 (Flatten)         (None, 2304)              0         \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 16)                36864     \n",
      "                                                                 \n",
      " activation_18 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 367,360\n",
      "Trainable params: 366,464\n",
      "Non-trainable params: 896\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 6 : CNN_64_128_192\n",
      "Model: \"model_6\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_7 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_6 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_13 (Conv2D)          (None, 28, 28, 64)        640       \n",
      "                                                                 \n",
      " batch_normalization_13 (Bat  (None, 28, 28, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_19 (Activation)  (None, 28, 28, 64)        0         \n",
      "                                                                 \n",
      " dropout_13 (Dropout)        (None, 28, 28, 64)        0         \n",
      "                                                                 \n",
      " average_pooling2d_7 (Averag  (None, 28, 28, 64)       0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_14 (Conv2D)          (None, 14, 14, 128)       32896     \n",
      "                                                                 \n",
      " batch_normalization_14 (Bat  (None, 14, 14, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_20 (Activation)  (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " dropout_14 (Dropout)        (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_8 (Averag  (None, 7, 7, 128)        0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_15 (Conv2D)          (None, 3, 3, 192)         221376    \n",
      "                                                                 \n",
      " batch_normalization_15 (Bat  (None, 3, 3, 192)        768       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_21 (Activation)  (None, 3, 3, 192)         0         \n",
      "                                                                 \n",
      " dropout_15 (Dropout)        (None, 3, 3, 192)         0         \n",
      "                                                                 \n",
      " flatten_6 (Flatten)         (None, 1728)              0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 16)                27648     \n",
      "                                                                 \n",
      " activation_22 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 284,096\n",
      "Trainable params: 283,328\n",
      "Non-trainable params: 768\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 7 : CNN_128_192_256\n",
      "Model: \"model_7\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_8 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_7 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_16 (Conv2D)          (None, 28, 28, 128)       1280      \n",
      "                                                                 \n",
      " batch_normalization_16 (Bat  (None, 28, 28, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_23 (Activation)  (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " dropout_16 (Dropout)        (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_9 (Averag  (None, 28, 28, 128)      0         \n",
      " ePooling2D)                                                     \n",
      "                                                                 \n",
      " conv2d_17 (Conv2D)          (None, 14, 14, 192)       98496     \n",
      "                                                                 \n",
      " batch_normalization_17 (Bat  (None, 14, 14, 192)      768       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_24 (Activation)  (None, 14, 14, 192)       0         \n",
      "                                                                 \n",
      " dropout_17 (Dropout)        (None, 14, 14, 192)       0         \n",
      "                                                                 \n",
      " average_pooling2d_10 (Avera  (None, 7, 7, 192)        0         \n",
      " gePooling2D)                                                    \n",
      "                                                                 \n",
      " conv2d_18 (Conv2D)          (None, 3, 3, 256)         442624    \n",
      "                                                                 \n",
      " batch_normalization_18 (Bat  (None, 3, 3, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_25 (Activation)  (None, 3, 3, 256)         0         \n",
      "                                                                 \n",
      " dropout_18 (Dropout)        (None, 3, 3, 256)         0         \n",
      "                                                                 \n",
      " flatten_7 (Flatten)         (None, 2304)              0         \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 16)                36864     \n",
      "                                                                 \n",
      " activation_26 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 581,568\n",
      "Trainable params: 580,416\n",
      "Non-trainable params: 1,152\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 8 : CNN_128_128_128\n",
      "Model: \"model_8\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_9 (InputLayer)        [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_8 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_19 (Conv2D)          (None, 28, 28, 128)       1280      \n",
      "                                                                 \n",
      " batch_normalization_19 (Bat  (None, 28, 28, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_27 (Activation)  (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " dropout_19 (Dropout)        (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_11 (Avera  (None, 28, 28, 128)      0         \n",
      " gePooling2D)                                                    \n",
      "                                                                 \n",
      " conv2d_20 (Conv2D)          (None, 14, 14, 128)       65664     \n",
      "                                                                 \n",
      " batch_normalization_20 (Bat  (None, 14, 14, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_28 (Activation)  (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " dropout_20 (Dropout)        (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_12 (Avera  (None, 7, 7, 128)        0         \n",
      " gePooling2D)                                                    \n",
      "                                                                 \n",
      " conv2d_21 (Conv2D)          (None, 3, 3, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_21 (Bat  (None, 3, 3, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_29 (Activation)  (None, 3, 3, 128)         0         \n",
      "                                                                 \n",
      " dropout_21 (Dropout)        (None, 3, 3, 128)         0         \n",
      "                                                                 \n",
      " flatten_8 (Flatten)         (None, 1152)              0         \n",
      "                                                                 \n",
      " dense_8 (Dense)             (None, 16)                18432     \n",
      "                                                                 \n",
      " activation_30 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 234,496\n",
      "Trainable params: 233,728\n",
      "Non-trainable params: 768\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 9 : CNN_128_128_192\n",
      "Model: \"model_9\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_10 (InputLayer)       [(None, 28, 28)]          0         \n",
      "                                                                 \n",
      " reshape_9 (Reshape)         (None, 28, 28, 1)         0         \n",
      "                                                                 \n",
      " conv2d_22 (Conv2D)          (None, 28, 28, 128)       1280      \n",
      "                                                                 \n",
      " batch_normalization_22 (Bat  (None, 28, 28, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_31 (Activation)  (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " dropout_22 (Dropout)        (None, 28, 28, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_13 (Avera  (None, 28, 28, 128)      0         \n",
      " gePooling2D)                                                    \n",
      "                                                                 \n",
      " conv2d_23 (Conv2D)          (None, 14, 14, 128)       65664     \n",
      "                                                                 \n",
      " batch_normalization_23 (Bat  (None, 14, 14, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_32 (Activation)  (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " dropout_23 (Dropout)        (None, 14, 14, 128)       0         \n",
      "                                                                 \n",
      " average_pooling2d_14 (Avera  (None, 7, 7, 128)        0         \n",
      " gePooling2D)                                                    \n",
      "                                                                 \n",
      " conv2d_24 (Conv2D)          (None, 3, 3, 192)         221376    \n",
      "                                                                 \n",
      " batch_normalization_24 (Bat  (None, 3, 3, 192)        768       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_33 (Activation)  (None, 3, 3, 192)         0         \n",
      "                                                                 \n",
      " dropout_24 (Dropout)        (None, 3, 3, 192)         0         \n",
      "                                                                 \n",
      " flatten_9 (Flatten)         (None, 1728)              0         \n",
      "                                                                 \n",
      " dense_9 (Dense)             (None, 16)                27648     \n",
      "                                                                 \n",
      " activation_34 (Activation)  (None, 16)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 317,760\n",
      "Trainable params: 316,864\n",
      "Non-trainable params: 896\n",
      "_________________________________________________________________\n",
      "None\n",
      "Training model  0\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:54:49.495682: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel/dropout/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "2023-05-12 08:54:50.304039: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:424] Loaded cuDNN version 8801\n",
      "2023-05-12 08:54:51.232236: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:637] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2023-05-12 08:54:51.234311: I tensorflow/compiler/xla/service/service.cc:169] XLA service 0x7f9a743dc400 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2023-05-12 08:54:51.234325: I tensorflow/compiler/xla/service/service.cc:177]   StreamExecutor device (0): NVIDIA GeForce RTX 3080 Ti, Compute Capability 8.6\n",
      "2023-05-12 08:54:51.236373: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2023-05-12 08:54:51.294725: I ./tensorflow/compiler/jit/device_compiler.h:180] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 6s 8ms/step - loss: 0.3859 - accuracy: 0.9366 - val_loss: 0.2752 - val_accuracy: 0.9271\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.1656 - accuracy: 0.9740 - val_loss: 0.1575 - val_accuracy: 0.9771\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.1380 - accuracy: 0.9786 - val_loss: 0.1306 - val_accuracy: 0.9808\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.1051 - accuracy: 0.9840 - val_loss: 0.1093 - val_accuracy: 0.9836\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0878 - accuracy: 0.9869 - val_loss: 0.0972 - val_accuracy: 0.9825\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0781 - accuracy: 0.9882 - val_loss: 0.1143 - val_accuracy: 0.9790\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 4s 7ms/step - loss: 0.0766 - accuracy: 0.9880 - val_loss: 0.0964 - val_accuracy: 0.9833\n",
      "Training model  1\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:55:16.939775: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_1/dropout_2/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 5s 9ms/step - loss: 0.5108 - accuracy: 0.9371 - val_loss: 0.2008 - val_accuracy: 0.9606\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 4s 9ms/step - loss: 0.2307 - accuracy: 0.9724 - val_loss: 0.1637 - val_accuracy: 0.9816\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 4s 9ms/step - loss: 0.1439 - accuracy: 0.9821 - val_loss: 0.1275 - val_accuracy: 0.9850\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 4s 9ms/step - loss: 0.1126 - accuracy: 0.9850 - val_loss: 0.1129 - val_accuracy: 0.9843\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 4s 9ms/step - loss: 0.0894 - accuracy: 0.9880 - val_loss: 0.1066 - val_accuracy: 0.9831\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 4s 9ms/step - loss: 0.0833 - accuracy: 0.9876 - val_loss: 0.1060 - val_accuracy: 0.9827\n",
      "Training model  2\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:55:42.731226: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_2/dropout_4/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 6s 11ms/step - loss: 0.7093 - accuracy: 0.9314 - val_loss: 0.4051 - val_accuracy: 0.9257\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.3143 - accuracy: 0.9714 - val_loss: 0.2162 - val_accuracy: 0.9789\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.2344 - accuracy: 0.9773 - val_loss: 0.1880 - val_accuracy: 0.9817\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.1462 - accuracy: 0.9846 - val_loss: 0.1690 - val_accuracy: 0.9751\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.1117 - accuracy: 0.9873 - val_loss: 0.1283 - val_accuracy: 0.9819\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.0947 - accuracy: 0.9875 - val_loss: 0.1124 - val_accuracy: 0.9831\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.0833 - accuracy: 0.9882 - val_loss: 0.1093 - val_accuracy: 0.9826\n",
      "Epoch 8/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.0803 - accuracy: 0.9886 - val_loss: 0.0937 - val_accuracy: 0.9860\n",
      "Epoch 9/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.0790 - accuracy: 0.9886 - val_loss: 0.0969 - val_accuracy: 0.9844\n",
      "Epoch 10/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.0811 - accuracy: 0.9889 - val_loss: 0.0881 - val_accuracy: 0.9862\n",
      "Epoch 11/20\n",
      "469/469 [==============================] - 5s 10ms/step - loss: 0.0811 - accuracy: 0.9881 - val_loss: 0.1029 - val_accuracy: 0.9817\n",
      "Training model  3\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:56:37.149517: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_3/dropout_6/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 7s 14ms/step - loss: 0.3810 - accuracy: 0.9381 - val_loss: 0.1701 - val_accuracy: 0.9636\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.1772 - accuracy: 0.9736 - val_loss: 0.1738 - val_accuracy: 0.9750\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.1390 - accuracy: 0.9798 - val_loss: 0.1475 - val_accuracy: 0.9799\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.1116 - accuracy: 0.9835 - val_loss: 0.1316 - val_accuracy: 0.9779\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.0954 - accuracy: 0.9858 - val_loss: 0.0934 - val_accuracy: 0.9854\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.0822 - accuracy: 0.9880 - val_loss: 0.0910 - val_accuracy: 0.9862\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.0836 - accuracy: 0.9869 - val_loss: 0.0899 - val_accuracy: 0.9859\n",
      "Epoch 8/20\n",
      "469/469 [==============================] - 6s 13ms/step - loss: 0.0773 - accuracy: 0.9888 - val_loss: 0.1139 - val_accuracy: 0.9788\n",
      "Training model  4\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:57:26.871037: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_4/dropout_8/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 9s 18ms/step - loss: 0.7638 - accuracy: 0.9263 - val_loss: 0.5159 - val_accuracy: 0.8890\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.3560 - accuracy: 0.9665 - val_loss: 0.2890 - val_accuracy: 0.9751\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.2568 - accuracy: 0.9769 - val_loss: 0.2149 - val_accuracy: 0.9827\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.2025 - accuracy: 0.9808 - val_loss: 0.1913 - val_accuracy: 0.9829\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.1515 - accuracy: 0.9851 - val_loss: 0.1566 - val_accuracy: 0.9814\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.1254 - accuracy: 0.9867 - val_loss: 0.1171 - val_accuracy: 0.9864\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.1045 - accuracy: 0.9878 - val_loss: 0.1153 - val_accuracy: 0.9846\n",
      "Epoch 8/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.0969 - accuracy: 0.9878 - val_loss: 0.1113 - val_accuracy: 0.9845\n",
      "Epoch 9/20\n",
      "469/469 [==============================] - 8s 17ms/step - loss: 0.0971 - accuracy: 0.9868 - val_loss: 0.1024 - val_accuracy: 0.9853\n",
      "Training model  5\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:58:39.995728: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_5/dropout_10/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 3s 5ms/step - loss: 0.1593 - accuracy: 0.9596 - val_loss: 0.1555 - val_accuracy: 0.9841\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0734 - accuracy: 0.9840 - val_loss: 0.0585 - val_accuracy: 0.9877\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0577 - accuracy: 0.9880 - val_loss: 0.0502 - val_accuracy: 0.9900\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0529 - accuracy: 0.9889 - val_loss: 0.0621 - val_accuracy: 0.9860\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0495 - accuracy: 0.9897 - val_loss: 0.0546 - val_accuracy: 0.9893\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0458 - accuracy: 0.9908 - val_loss: 0.0530 - val_accuracy: 0.9891\n",
      "Training model  6\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:58:53.196306: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_6/dropout_13/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 3s 4ms/step - loss: 0.1613 - accuracy: 0.9598 - val_loss: 0.1760 - val_accuracy: 0.9783\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0714 - accuracy: 0.9854 - val_loss: 0.0608 - val_accuracy: 0.9868\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0595 - accuracy: 0.9878 - val_loss: 0.0562 - val_accuracy: 0.9882\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0527 - accuracy: 0.9893 - val_loss: 0.0464 - val_accuracy: 0.9905\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0479 - accuracy: 0.9902 - val_loss: 0.0520 - val_accuracy: 0.9882\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0455 - accuracy: 0.9911 - val_loss: 0.0491 - val_accuracy: 0.9895\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0443 - accuracy: 0.9912 - val_loss: 0.0411 - val_accuracy: 0.9911\n",
      "Training model  7\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:59:08.095388: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_7/dropout_16/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 5s 8ms/step - loss: 0.1450 - accuracy: 0.9639 - val_loss: 0.1784 - val_accuracy: 0.9789\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0700 - accuracy: 0.9846 - val_loss: 0.0670 - val_accuracy: 0.9844\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0577 - accuracy: 0.9880 - val_loss: 0.0555 - val_accuracy: 0.9882\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0513 - accuracy: 0.9893 - val_loss: 0.0497 - val_accuracy: 0.9890\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0468 - accuracy: 0.9902 - val_loss: 0.0666 - val_accuracy: 0.9832\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0439 - accuracy: 0.9917 - val_loss: 0.0421 - val_accuracy: 0.9907\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0425 - accuracy: 0.9916 - val_loss: 0.0468 - val_accuracy: 0.9907\n",
      "Epoch 8/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0394 - accuracy: 0.9928 - val_loss: 0.0462 - val_accuracy: 0.9900\n",
      "Epoch 9/20\n",
      "469/469 [==============================] - 3s 7ms/step - loss: 0.0392 - accuracy: 0.9922 - val_loss: 0.0427 - val_accuracy: 0.9912\n",
      "Training model  8\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:59:39.019224: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_8/dropout_19/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 4s 7ms/step - loss: 0.1901 - accuracy: 0.9506 - val_loss: 0.2800 - val_accuracy: 0.9728\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0853 - accuracy: 0.9811 - val_loss: 0.0544 - val_accuracy: 0.9887\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0675 - accuracy: 0.9857 - val_loss: 0.0532 - val_accuracy: 0.9890\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0603 - accuracy: 0.9873 - val_loss: 0.0544 - val_accuracy: 0.9877\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0557 - accuracy: 0.9879 - val_loss: 0.0552 - val_accuracy: 0.9878\n",
      "Training model  9\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 08:59:54.947135: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_9/dropout_22/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469/469 [==============================] - 4s 6ms/step - loss: 0.1707 - accuracy: 0.9563 - val_loss: 0.1921 - val_accuracy: 0.9811\n",
      "Epoch 2/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0777 - accuracy: 0.9826 - val_loss: 0.0542 - val_accuracy: 0.9893\n",
      "Epoch 3/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0642 - accuracy: 0.9861 - val_loss: 0.0523 - val_accuracy: 0.9884\n",
      "Epoch 4/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0580 - accuracy: 0.9878 - val_loss: 0.0552 - val_accuracy: 0.9875\n",
      "Epoch 5/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0536 - accuracy: 0.9887 - val_loss: 0.0456 - val_accuracy: 0.9907\n",
      "Epoch 6/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0504 - accuracy: 0.9897 - val_loss: 0.0443 - val_accuracy: 0.9908\n",
      "Epoch 7/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0488 - accuracy: 0.9900 - val_loss: 0.0486 - val_accuracy: 0.9896\n",
      "Epoch 8/20\n",
      "469/469 [==============================] - 3s 6ms/step - loss: 0.0472 - accuracy: 0.9903 - val_loss: 0.0440 - val_accuracy: 0.9914\n",
      "pre-train accuracy: \n",
      "[0.983299970626831, 0.982699990272522, 0.9817000031471252, 0.9787999987602234, 0.9853000044822693, 0.9890999794006348, 0.991100013256073, 0.9911999702453613, 0.9878000020980835, 0.9914000034332275]\n"
     ]
    }
   ],
   "source": [
    "parties = []\n",
    "if model_saved_dir is None:\n",
    "    for i, item in enumerate(model_config):\n",
    "        model_name = item[\"model_type\"]\n",
    "        model_params = item[\"params\"]\n",
    "        tmp = CANDIDATE_MODELS[model_name](n_classes=n_classes, \n",
    "                                            input_shape=(28,28),\n",
    "                                            **model_params)\n",
    "        print(\"model {0} : {1}\".format(i, model_saved_names[i]))\n",
    "        print(tmp.summary())\n",
    "        parties.append(tmp)\n",
    "        \n",
    "        del model_name, model_params, tmp\n",
    "    #END FOR LOOP\n",
    "    pre_train_result = train_models(parties, \n",
    "                                    X_train_MNIST, y_train_MNIST, \n",
    "                                    X_test_MNIST, y_test_MNIST,\n",
    "                                    save_dir = model_saved_dir, save_names = model_saved_names,\n",
    "                                    early_stopping = is_early_stopping,\n",
    "                                    **pre_train_params\n",
    "                                    )\n",
    "else:\n",
    "    dpath = os.path.abspath(model_saved_dir)\n",
    "    model_names = os.listdir(dpath)\n",
    "    for name in model_names:\n",
    "        tmp = None\n",
    "        tmp = load_model(os.path.join(dpath ,name))\n",
    "        parties.append(tmp)\n",
    "\n",
    "del  X_train_MNIST, y_train_MNIST, X_test_MNIST, y_test_MNIST, \\\n",
    "X_train_EMNIST, y_train_EMNIST, X_test_EMNIST, y_test_EMNIST, writer_ids_train_EMNIST, writer_ids_test_EMNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  0\n",
      "(20, 28, 28) (20,)\n",
      "[15 15 12 14 11 14 10 10 10 15 10 10 11 15 12 10 15 10 10 10]\n",
      "model  1\n",
      "(23, 28, 28) (23,)\n",
      "[10 15 10 12 12 14 10 13 10 11 13 14 13 15 11 12 15 11 14 15 14 11 15]\n",
      "model  2\n",
      "(20, 28, 28) (20,)\n",
      "[14 15 10 10 13 15 15 13 10 15 13 11 12 13 10 12 12 14 10 11]\n",
      "model  3\n",
      "(25, 28, 28) (25,)\n",
      "[10 10 10 12 10 15 14 14 13 10 13 10 15 14 10 13 15 15 12 12 10 11 15 11\n",
      " 11]\n",
      "model  4\n",
      "(19, 28, 28) (19,)\n",
      "[13 15 13 15 14 11 10 12 11 13 11 15 13 13 10 11 13 12 10]\n",
      "model  5\n",
      "(22, 28, 28) (22,)\n",
      "[10 15 10 13 10 14 13 13 13 13 14 13 14 13 14 15 14 12 11 12 13 11]\n",
      "model  6\n",
      "(18, 28, 28) (18,)\n",
      "[13 15 12 14 15 11 12 13 12 11 10 12 10 10 10 11 13 14]\n",
      "model  7\n",
      "(19, 28, 28) (19,)\n",
      "[10 14 11 13 10 11 14 15 14 11 15 12 12 14 15 14 10 12 11]\n",
      "model  8\n",
      "(20, 28, 28) (20,)\n",
      "[15 12 10 10 14 11 14 11 11 10 14 12 15 10 13 12 13 15 13 10]\n",
      "model  9\n",
      "(22, 28, 28) (22,)\n",
      "[14 11 10 10 14 11 11 10 14 15 10 13 13 13 12 10 15 11 13 13 15 14]\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(\"model \", i)\n",
    "\n",
    "    device = i\n",
    "\n",
    "    X_train = private_data[device]['X']\n",
    "    y_train = private_data[device]['y']\n",
    "\n",
    "    print(X_train.shape, y_train.shape)\n",
    "    print(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  0\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:31.750127: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel/dropout/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 40ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  1\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:34.637714: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_1/dropout_2/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 64ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  2\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:38.720423: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_2/dropout_4/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 31ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  3\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:42.319412: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_3/dropout_6/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 70ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  4\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:46.857938: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_4/dropout_8/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fa14f5f35e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fa14f5f35e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 197ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  5\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:53.011672: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_5/dropout_10/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fa14c5533a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fa14c5533a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 35ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  6\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:56.465898: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_6/dropout_13/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 35ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  7\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:04:59.985642: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_7/dropout_16/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 70ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  8\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:05:04.031892: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_8/dropout_19/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 35ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model  9\n",
      "Semi-supervised training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:05:07.854400: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_9/dropout_22/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semi-supervised training done\n",
      "1/1 [==============================] - 0s 36ms/step\n"
     ]
    }
   ],
   "source": [
    "# Label propagation\n",
    "\n",
    "for i in range(10):\n",
    "    print(\"model \", i)\n",
    "\n",
    "    device = i\n",
    "\n",
    "    X_train = private_data[device]['X']\n",
    "    y_train = private_data[device]['y']\n",
    "\n",
    "    X_lab, X_unlab, y_lab, y_unlab = train_test_split(X_train, y_train, test_size=.5, random_state=1)\n",
    "\n",
    "    model_A_twin = None\n",
    "    model_A_twin = clone_model(parties[i]) # load private model\n",
    "    model_A_twin.set_weights(parties[i].get_weights())\n",
    "    model_A_twin.compile(optimizer=tf.keras.optimizers.Adam(lr = 1e-3), \n",
    "                            loss = \"sparse_categorical_crossentropy\",\n",
    "                            metrics = [\"accuracy\"])\n",
    "\n",
    "    print(\"Semi-supervised training ... \")        \n",
    "\n",
    "    # train private models with private data\n",
    "    model_A_twin.fit(X_lab, y_lab,\n",
    "                        batch_size = 32, epochs = 25, shuffle=True, verbose = 0,\n",
    "                        validation_data = [private_test_data[\"X\"], private_test_data[\"y\"]],\n",
    "                        callbacks=[EarlyStopping(monitor='val_accuracy', min_delta=0.001, patience=10)]\n",
    "                    )\n",
    "\n",
    "    print(\"Semi-supervised training done\")\n",
    "\n",
    "    y_pred = model_A_twin.predict(X_unlab).argmax(axis=1)\n",
    "\n",
    "    X_mixed = np.concatenate((X_lab, X_unlab))\n",
    "    y_mixed = np.concatenate((y_lab, y_pred))\n",
    "\n",
    "    private_data[device]['X'] = X_mixed\n",
    "    private_data[device]['y'] = y_mixed\n",
    "\n",
    "    del model_A_twin, X_mixed, y_mixed, X_lab, X_unlab, y_lab, y_unlab, X_train, y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start model initialization: \n",
      "model  0\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:10.756620: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel/dropout/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  1\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:14.498492: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_1/dropout_2/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  2\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:18.532659: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_2/dropout_4/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:22.128518: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_3/dropout_6/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  4\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:27.099917: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_4/dropout_8/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  5\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:31.252723: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_5/dropout_10/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  6\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:34.250820: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_6/dropout_13/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  7\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:37.700072: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_7/dropout_16/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  8\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:41.398662: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_8/dropout_19/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "model  9\n",
      "start full stack training ... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:45.147362: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_9/dropout_22/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full stack training done\n",
      "\n",
      "calculate the theoretical upper bounds for participants: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:06:48.864460: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel/dropout/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:06:52.921467: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_1/dropout_2/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:01.044785: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_2/dropout_4/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:07.477068: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_3/dropout_6/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:16.135549: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_4/dropout_8/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:24.196820: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_5/dropout_10/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:28.831789: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_6/dropout_13/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:34.549084: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_7/dropout_16/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:41.435914: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_8/dropout_19/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n",
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n",
      "2023-05-12 09:07:45.910396: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_9/dropout_22/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the upper bounds are: [0.8722916841506958, 0.8729166388511658, 0.8768749833106995, 0.8799999952316284, 0.8664583563804626, 0.9068750143051147, 0.9150000214576721, 0.9102083444595337, 0.8991666436195374, 0.9106249809265137]\n",
      "round  0\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.21604166666666666\n",
      "0.12166666666666667\n",
      "0.07916666666666666\n",
      "0.15895833333333334\n",
      "0.344375\n",
      "0.2841666666666667\n",
      "0.35875\n",
      "0.35375\n",
      "0.4995833333333333\n",
      "0.4395833333333333\n",
      "[8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:07:53.005811: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_10/dropout/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:07:54.550737: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_11/dropout_2/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:07:56.425078: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_12/dropout_4/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:07:58.133754: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_13/dropout_6/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:08:00.392127: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_14/dropout_8/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:08:02.930338: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_15/dropout_10/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:08:04.669496: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_16/dropout_13/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:08:06.019495: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_17/dropout_16/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:08:07.781305: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_18/dropout_19/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-12 09:08:09.348092: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_19/dropout_22/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  1\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.1675\n",
      "0.12020833333333333\n",
      "0.08041666666666666\n",
      "0.14270833333333333\n",
      "0.2683333333333333\n",
      "0.319375\n",
      "0.35041666666666665\n",
      "0.39958333333333335\n",
      "0.508125\n",
      "0.42520833333333335\n",
      "[8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  2\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.18625\n",
      "0.13\n",
      "0.10541666666666667\n",
      "0.13291666666666666\n",
      "0.486875\n",
      "0.35125\n",
      "0.356875\n",
      "0.41041666666666665\n",
      "0.553125\n",
      "0.49020833333333336\n",
      "[4, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  3\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.18416666666666667\n",
      "0.21770833333333334\n",
      "0.13958333333333334\n",
      "0.18791666666666668\n",
      "0.4997916666666667\n",
      "0.3458333333333333\n",
      "0.37333333333333335\n",
      "0.40541666666666665\n",
      "0.5408333333333334\n",
      "0.48354166666666665\n",
      "[4, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  4\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.201875\n",
      "0.24458333333333335\n",
      "0.11375\n",
      "0.16770833333333332\n",
      "0.498125\n",
      "0.39666666666666667\n",
      "0.4060416666666667\n",
      "0.41125\n",
      "0.559375\n",
      "0.47854166666666664\n",
      "[4, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  5\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.21583333333333332\n",
      "0.24770833333333334\n",
      "0.175625\n",
      "0.19458333333333333\n",
      "0.5125\n",
      "0.425\n",
      "0.413125\n",
      "0.4191666666666667\n",
      "0.556875\n",
      "0.506875\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  6\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.19479166666666667\n",
      "0.3022916666666667\n",
      "0.2675\n",
      "0.19208333333333333\n",
      "0.505\n",
      "0.48291666666666666\n",
      "0.4425\n",
      "0.425625\n",
      "0.5633333333333334\n",
      "0.5222916666666667\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  7\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.198125\n",
      "0.358125\n",
      "0.12791666666666668\n",
      "0.1675\n",
      "0.48604166666666665\n",
      "0.46958333333333335\n",
      "0.44375\n",
      "0.42604166666666665\n",
      "0.5675\n",
      "0.4979166666666667\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  8\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.21291666666666667\n",
      "0.36854166666666666\n",
      "0.17354166666666668\n",
      "0.21354166666666666\n",
      "0.5175\n",
      "0.4820833333333333\n",
      "0.43645833333333334\n",
      "0.43145833333333333\n",
      "0.5760416666666667\n",
      "0.5214583333333334\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  9\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.23020833333333332\n",
      "0.2660416666666667\n",
      "0.170625\n",
      "0.228125\n",
      "0.5147916666666666\n",
      "0.47979166666666667\n",
      "0.453125\n",
      "0.43916666666666665\n",
      "0.578125\n",
      "0.5147916666666666\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  10\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.21895833333333334\n",
      "0.32416666666666666\n",
      "0.17645833333333333\n",
      "0.22208333333333333\n",
      "0.5108333333333334\n",
      "0.488125\n",
      "0.45666666666666667\n",
      "0.43583333333333335\n",
      "0.5804166666666667\n",
      "0.5216666666666666\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  11\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.22291666666666668\n",
      "0.33541666666666664\n",
      "0.17854166666666665\n",
      "0.16708333333333333\n",
      "0.49104166666666665\n",
      "0.4739583333333333\n",
      "0.4764583333333333\n",
      "0.44\n",
      "0.5860416666666667\n",
      "0.5102083333333334\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  12\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.2525\n",
      "0.28791666666666665\n",
      "0.22458333333333333\n",
      "0.19520833333333334\n",
      "0.5129166666666667\n",
      "0.503125\n",
      "0.47125\n",
      "0.453125\n",
      "0.585625\n",
      "0.5077083333333333\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  13\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.24416666666666667\n",
      "0.334375\n",
      "0.21\n",
      "0.22541666666666665\n",
      "0.46958333333333335\n",
      "0.5145833333333333\n",
      "0.48083333333333333\n",
      "0.44208333333333333\n",
      "0.590625\n",
      "0.5422916666666666\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  14\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.2708333333333333\n",
      "0.37645833333333334\n",
      "0.215\n",
      "0.23666666666666666\n",
      "0.5029166666666667\n",
      "0.5108333333333334\n",
      "0.4789583333333333\n",
      "0.4508333333333333\n",
      "0.5885416666666666\n",
      "0.5183333333333333\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  15\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.2791666666666667\n",
      "0.35520833333333335\n",
      "0.22479166666666667\n",
      "0.24541666666666667\n",
      "0.5347916666666667\n",
      "0.5225\n",
      "0.48354166666666665\n",
      "0.45895833333333336\n",
      "0.585\n",
      "0.52\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  16\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.24895833333333334\n",
      "0.32666666666666666\n",
      "0.17291666666666666\n",
      "0.23125\n",
      "0.5258333333333334\n",
      "0.5270833333333333\n",
      "0.47875\n",
      "0.46208333333333335\n",
      "0.59\n",
      "0.5027083333333333\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  17\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.26708333333333334\n",
      "0.32229166666666664\n",
      "0.21104166666666666\n",
      "0.2275\n",
      "0.5104166666666666\n",
      "0.5235416666666667\n",
      "0.48354166666666665\n",
      "0.4710416666666667\n",
      "0.596875\n",
      "0.5389583333333333\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  18\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.26125\n",
      "0.336875\n",
      "0.223125\n",
      "0.22708333333333333\n",
      "0.5204166666666666\n",
      "0.544375\n",
      "0.48375\n",
      "0.465625\n",
      "0.5920833333333333\n",
      "0.5272916666666667\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  19\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.23770833333333333\n",
      "0.32979166666666665\n",
      "0.14916666666666667\n",
      "0.205625\n",
      "0.5114583333333333\n",
      "0.5460416666666666\n",
      "0.485\n",
      "0.46520833333333333\n",
      "0.6004166666666667\n",
      "0.5291666666666667\n",
      "[4, 5, 6, 7, 8, 9]\n",
      "updates models ...\n",
      "model 0 starting alignment with public logits... \n",
      "model 0 done alignment\n",
      "model 0 starting training with private data... \n",
      "model 0 done private training. \n",
      "\n",
      "model 1 starting alignment with public logits... \n",
      "model 1 done alignment\n",
      "model 1 starting training with private data... \n",
      "model 1 done private training. \n",
      "\n",
      "model 2 starting alignment with public logits... \n",
      "model 2 done alignment\n",
      "model 2 starting training with private data... \n",
      "model 2 done private training. \n",
      "\n",
      "model 3 starting alignment with public logits... \n",
      "model 3 done alignment\n",
      "model 3 starting training with private data... \n",
      "model 3 done private training. \n",
      "\n",
      "model 4 starting alignment with public logits... \n",
      "model 4 done alignment\n",
      "model 4 starting training with private data... \n",
      "model 4 done private training. \n",
      "\n",
      "model 5 starting alignment with public logits... \n",
      "model 5 done alignment\n",
      "model 5 starting training with private data... \n",
      "model 5 done private training. \n",
      "\n",
      "model 6 starting alignment with public logits... \n",
      "model 6 done alignment\n",
      "model 6 starting training with private data... \n",
      "model 6 done private training. \n",
      "\n",
      "model 7 starting alignment with public logits... \n",
      "model 7 done alignment\n",
      "model 7 starting training with private data... \n",
      "model 7 done private training. \n",
      "\n",
      "model 8 starting alignment with public logits... \n",
      "model 8 done alignment\n",
      "model 8 starting training with private data... \n",
      "model 8 done private training. \n",
      "\n",
      "model 9 starting alignment with public logits... \n",
      "model 9 done alignment\n",
      "model 9 starting training with private data... \n",
      "model 9 done private training. \n",
      "\n",
      "round  20\n",
      "update logits ... \n",
      "test performance ... \n",
      "0.24208333333333334\n",
      "0.380625\n",
      "0.20583333333333334\n",
      "0.23541666666666666\n",
      "0.513125\n",
      "0.5497916666666667\n",
      "0.48625\n",
      "0.47520833333333334\n",
      "0.6029166666666667\n",
      "0.523125\n",
      "[4, 5, 6, 7, 8, 9]\n"
     ]
    }
   ],
   "source": [
    "fedmd = FedMD(parties, \n",
    "                public_dataset = public_dataset,\n",
    "                private_data = private_data, \n",
    "                total_private_data = total_private_data,\n",
    "                private_test_data = private_test_data,\n",
    "                N_rounds = N_rounds,\n",
    "                N_alignment = N_alignment, \n",
    "                N_logits_matching_round = N_logits_matching_round,\n",
    "                logits_matching_batchsize = logits_matching_batchsize, \n",
    "                N_private_training_round = N_private_training_round, \n",
    "                private_training_batchsize = private_training_batchsize)\n",
    "\n",
    "initialization_result = fedmd.init_result\n",
    "pooled_train_result = fedmd.pooled_train_result\n",
    "\n",
    "collaboration_performance = fedmd.collaborative_training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ult-tf",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}