99_upload_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os, math, numpy as np, pickle, torch \n",
    "cur_dir = os.getcwd()\n",
    "while not os.getcwd().endswith('-analysis'): os.chdir('..')\n",
    "\n",
    "from pprint import pprint\n",
    "from safetensors import safe_open\n",
    "from modeling_jonberta import JonbertaForSequenceClassification, add_features_to_model\n",
    "from transformers import AutoConfig\n",
    "\n",
    "results_dir = os.path.abspath('notebooks/paper/results/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Uploading Models to HuggingFace\n",
    "Uploading 320/5 models and setting model cards. \n",
    "\n",
    "- I have 320 models, but 5x less if we only consider the median one per training split. \n",
    "- Default model cards were added to shame you for not filling them out, but there are no APIs for filling them out programmatically. There's no way I'm doing 320 cards by hand though. \n",
    "- I'm not registering the `JonBERTa` architecture on the hub, as there is much room for improvement. To load the model, see `modeling_jonberta.py`, and refer to the `get_model` function in this notebook to do it properly. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(model_path):\n",
    "    ''' Load in a JonBERTa-head/attn model '''\n",
    "\n",
    "    config = AutoConfig.from_pretrained(model_path)\n",
    "    model = JonbertaForSequenceClassification(config)\n",
    "    if hasattr(config, 'add_head') and config.add_head: \n",
    "        add_features_to_model(model, config)\n",
    "\n",
    "    state_dict = {} \n",
    "    with safe_open(os.path.join(model_path, 'model.safetensors'), framework='pt') as f: \n",
    "        for key in f.keys():\n",
    "            state_dict[key] = f.get_tensor(key)\n",
    "    new_layers = model.load_state_dict(state_dict, strict=False)\n",
    "\n",
    "    pprint(new_layers)\n",
    "    return model  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Retrieving paths to models in my `results` dir\n",
    "And, filtering to keep only the median-performing model per group (hyperparam combination)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1311462/4266769283.py:8: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  model_scores = model_scores.groupby('Group').apply(lambda x: x.iloc[len(x) // 2]).reset_index(drop=True)\n",
      "/tmp/ipykernel_1311462/4266769283.py:12: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  layer_attn_scores = layer_attn_scores.groupby('Group').apply(lambda x: x.iloc[len(x) // 2]).reset_index(drop=True)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# get the median scoring model per group to maintain some ordinality between them\n",
    "model_scores = pd.read_csv(os.path.join(results_dir, 'model_scores.csv'))\n",
    "\n",
    "# per Group, sort on test macro avg, and remove all but the median model \n",
    "model_scores = model_scores.sort_values('test macro avg', ascending=False)\n",
    "model_scores = model_scores.groupby('Group').apply(lambda x: x.iloc[len(x) // 2]).reset_index(drop=True)\n",
    "\n",
    "# do the same for the 3-layer attention scores which for some reason I stored elsewhere \n",
    "layer_attn_scores = pd.read_csv(os.path.join(results_dir, 'model_scores_layers.csv')).sort_values('test macro avg', ascending=False)\n",
    "layer_attn_scores = layer_attn_scores.groupby('Group').apply(lambda x: x.iloc[len(x) // 2]).reset_index(drop=True)\n",
    "\n",
    "median_models = set(model_scores['Name']).union(set(layer_attn_scores['Name']))\n",
    "# median_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading in a dictionary of the model paths from my messy results directory \n",
    "model_dirs = {\n",
    "    'CodeBERTa': sorted([p.path for p in os.scandir(\n",
    "                        os.path.join(results_dir, '12_codeberta/huggingface/CodeBERTa-small-v1/model'))]),\n",
    "    'JonBERTa-head': sorted([p.path for p in os.scandir(\n",
    "                        os.path.join(results_dir, '13_jonberta/12_codeberta-biased-2e-05lr--0/model'))\n",
    "                        if 'HEAD' in p.path and os.path.basename(p.path).startswith('-')]),\n",
    "    'JonBERTa-attn': sorted([p.path for p in os.scandir(\n",
    "                        os.path.join(results_dir, '13_jonberta/12_codeberta-biased-2e-05lr--0/model'))\n",
    "                        if 'ATTN' in p.path]),\n",
    "}\n",
    "\n",
    "# remove the test models \n",
    "model_dirs = {model_type: [model for model in models if 'test' not in model and 'TEST' not in model] \n",
    "              for model_type, models in model_dirs.items()} \n",
    "\n",
    "# keep only the median models \n",
    "model_dirs = {model_type: [model for model in models if os.path.basename(model) in median_models] \n",
    "              for model_type, models in model_dirs.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Renaming models \n",
    "Remapping the extremely verbose names as follows: \n",
    "\n",
    "- `CodeBERTa` &rarr; `CodeBERTa-ft-coco-[1,2,5]e-05lr--[0-4]`\n",
    "    - e.g. `CodeBERTa-ft-coco-1e-05lr--0`\n",
    "- `JonBERTa-head` &rarr; `JonBERTa-head-ft-(dense-proj-reinit)--[0-4]` (all have `2e-05` learning rate)\n",
    "    - e.g. `JonBERTa-head-ft-(dense-proj-)--1`\n",
    "- `JonBERTa-attn` &rarr; `JonBERTa-attn-ft-(0,1,2,3,4,5L)--[0-4]`\n",
    "    - e.g. `JonBERTa-attn-ft-(0,1,2L)--0` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1mCodeBERTa\u001b[0m\n",
      "38 \tCodeBERTa-ft-coco-1e-05lr \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/12_codeberta/huggingface/CodeBERTa-small-v1/model/12_codeberta-biased-1e-05lr--1\n",
      "38 \tCodeBERTa-ft-coco-2e-05lr \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/12_codeberta/huggingface/CodeBERTa-small-v1/model/12_codeberta-biased-2e-05lr--4\n",
      "38 \tCodeBERTa-ft-coco-5e-05lr \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/12_codeberta/huggingface/CodeBERTa-small-v1/model/12_codeberta-biased-5e-05lr--3\n",
      "\u001b[1mJonBERTa-head\u001b[0m\n",
      "34 \tJonBERTa-head-ft-coco \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD---)-2e-05lr-4\n",
      "41 \tJonBERTa-head-ft-coco-reinit \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD---reinit)-2e-05lr-3\n",
      "39 \tJonBERTa-head-ft-coco-proj \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD--proj-)-2e-05lr-4\n",
      "46 \tJonBERTa-head-ft-coco-proj-reinit \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD--proj-reinit)-2e-05lr-1\n",
      "40 \tJonBERTa-head-ft-coco-dense \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense--)-2e-05lr-4\n",
      "47 \tJonBERTa-head-ft-coco-dense-reinit \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense--reinit)-2e-05lr-2\n",
      "45 \tJonBERTa-head-ft-coco-dense-proj \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense-proj-)-2e-05lr-4\n",
      "52 \tJonBERTa-head-ft-coco-dense-proj-reinit \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense-proj-reinit)-2e-05lr-4\n",
      "\u001b[1mJonBERTa-attn\u001b[0m\n",
      "39 \tJonBERTa-attn-ft-coco-012L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 2]L)-2e-05lr-3\n",
      "39 \tJonBERTa-attn-ft-coco-013L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 3]L)-2e-05lr-3\n",
      "39 \tJonBERTa-attn-ft-coco-014L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 4]L)-2e-05lr-4\n",
      "39 \tJonBERTa-attn-ft-coco-015L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 5]L)-2e-05lr-1\n",
      "38 \tJonBERTa-attn-ft-coco-01L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1]L)-2e-05lr-0\n",
      "39 \tJonBERTa-attn-ft-coco-023L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2, 3]L)-2e-05lr-0\n",
      "39 \tJonBERTa-attn-ft-coco-024L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2, 4]L)-2e-05lr-1\n",
      "39 \tJonBERTa-attn-ft-coco-025L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2, 5]L)-2e-05lr-3\n",
      "38 \tJonBERTa-attn-ft-coco-02L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2]L)-2e-05lr-4\n",
      "39 \tJonBERTa-attn-ft-coco-034L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 3, 4]L)-2e-05lr-4\n",
      "39 \tJonBERTa-attn-ft-coco-035L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 3, 5]L)-2e-05lr-2\n",
      "38 \tJonBERTa-attn-ft-coco-03L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 3]L)-2e-05lr-3\n",
      "39 \tJonBERTa-attn-ft-coco-045L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 4, 5]L)-2e-05lr-2\n",
      "38 \tJonBERTa-attn-ft-coco-04L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 4]L)-2e-05lr-0\n",
      "38 \tJonBERTa-attn-ft-coco-05L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 5]L)-2e-05lr-2\n",
      "39 \tJonBERTa-attn-ft-coco-123L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2, 3]L)-2e-05lr-0\n",
      "39 \tJonBERTa-attn-ft-coco-124L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2, 4]L)-2e-05lr-2\n",
      "39 \tJonBERTa-attn-ft-coco-125L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2, 5]L)-2e-05lr-2\n",
      "38 \tJonBERTa-attn-ft-coco-12L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2]L)-2e-05lr-0\n",
      "39 \tJonBERTa-attn-ft-coco-134L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 3, 4]L)-2e-05lr-4\n",
      "39 \tJonBERTa-attn-ft-coco-135L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 3, 5]L)-2e-05lr-3\n",
      "38 \tJonBERTa-attn-ft-coco-13L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 3]L)-2e-05lr-3\n",
      "39 \tJonBERTa-attn-ft-coco-145L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 4, 5]L)-2e-05lr-3\n",
      "38 \tJonBERTa-attn-ft-coco-14L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 4]L)-2e-05lr-2\n",
      "38 \tJonBERTa-attn-ft-coco-15L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 5]L)-2e-05lr-0\n",
      "39 \tJonBERTa-attn-ft-coco-234L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 3, 4]L)-2e-05lr-2\n",
      "39 \tJonBERTa-attn-ft-coco-235L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 3, 5]L)-2e-05lr-0\n",
      "38 \tJonBERTa-attn-ft-coco-23L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 3]L)-2e-05lr-4\n",
      "39 \tJonBERTa-attn-ft-coco-245L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 4, 5]L)-2e-05lr-1\n",
      "38 \tJonBERTa-attn-ft-coco-24L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 4]L)-2e-05lr-2\n",
      "38 \tJonBERTa-attn-ft-coco-25L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 5]L)-2e-05lr-1\n",
      "39 \tJonBERTa-attn-ft-coco-345L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3, 4, 5]L)-2e-05lr-3\n",
      "38 \tJonBERTa-attn-ft-coco-34L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3, 4]L)-2e-05lr-4\n",
      "38 \tJonBERTa-attn-ft-coco-35L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3, 5]L)-2e-05lr-1\n",
      "38 \tJonBERTa-attn-ft-coco-45L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[4, 5]L)-2e-05lr-3\n",
      "37 \tJonBERTa-attn-ft-coco-0L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0]L)-2e-05lr--3\n",
      "37 \tJonBERTa-attn-ft-coco-1L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1]L)-2e-05lr--2\n",
      "37 \tJonBERTa-attn-ft-coco-2L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2]L)-2e-05lr--2\n",
      "37 \tJonBERTa-attn-ft-coco-3L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3]L)-2e-05lr--0\n",
      "37 \tJonBERTa-attn-ft-coco-4L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[4]L)-2e-05lr--4\n",
      "37 \tJonBERTa-attn-ft-coco-5L \t/home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[5]L)-2e-05lr--0\n"
     ]
    }
   ],
   "source": [
    "def get_model_name(model_path): \n",
    "    model_name = os.path.basename(model_path)\n",
    "\n",
    "    if model_name.startswith('12_codeberta'):               # CodeBERTa\n",
    "        model_name = f'CodeBERTa-ft-coco-{model_name[-10:]}'\n",
    "\n",
    "    elif 'HEAD' in model_name:                              # JonBERTa with HEAD\n",
    "        # remove everything until HEAD in model_name \n",
    "        model_name = f'JonBERTa-head-ft-coco-({model_name[model_name.index(\"HEAD\")+4:-1]}' + '-' + model_name[-1]\n",
    "        # remove 24th character \n",
    "        model_name = model_name[:23] + model_name[24:]\n",
    "        # remove lr\n",
    "        model_name = model_name[:-11] + model_name[-3:]\n",
    "\n",
    "        # huggingface doens't want -- in the model name \n",
    "        # get everything between () in the model_name\n",
    "        opts = model_name[model_name.index('(')+1:model_name.index(')')].split('-')\n",
    "        opts = '-'.join([opt for opt in opts if opt != ''])\n",
    "\n",
    "        return f'JonBERTa-head-ft-coco-{opts}' if len(opts) > 0 else 'JonBERTa-head-ft-coco'\n",
    "\n",
    "    elif 'ATTN' in model_name:                              # JonBERTa with ATTN\n",
    "        # get the numbers between [] in the model_name \n",
    "        layers = model_name[model_name.index('[')+1:model_name.index(']')]\n",
    "        # remove whitespace between layers \n",
    "        layers = f'{layers.replace(\" \", \"\").replace(\",\",\"\")}L'\n",
    "\n",
    "        # remove everything until and including ) in model_name \n",
    "        model_name = model_name[model_name.index(')')+1:]\n",
    "\n",
    "        # if the third-last character is not a -, insert a - \n",
    "        if model_name[-3] != '-': \n",
    "            model_name = model_name[:-1] + '-' + model_name[-1]\n",
    "        model_name = model_name[-3:]\n",
    "        model_name = f'JonBERTa-attn-ft-coco-{layers}{model_name}'\n",
    "\n",
    "    return model_name[:-3]\n",
    "\n",
    "for model_type, model_paths in model_dirs.items(): \n",
    "    print(f'\\033[1m{model_type}\\033[0m')\n",
    "    for model_path in model_paths: \n",
    "        model_name = get_model_name(model_path)\n",
    "        print(f'{len(model_name) + len(\"AISE-TUDelft/\")} \\t{model_name} \\t{model_path}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CodeBERTa-ft-coco-1e-05lr written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/12_codeberta/huggingface/CodeBERTa-small-v1/model/12_codeberta-biased-1e-05lr--1/readme.md\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CodeBERTa-ft-coco-2e-05lr written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/12_codeberta/huggingface/CodeBERTa-small-v1/model/12_codeberta-biased-2e-05lr--4/readme.md\n",
      "CodeBERTa-ft-coco-5e-05lr written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/12_codeberta/huggingface/CodeBERTa-small-v1/model/12_codeberta-biased-5e-05lr--3/readme.md\n",
      "JonBERTa-head-ft-coco written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD---)-2e-05lr-4/readme.md\n",
      "JonBERTa-head-ft-coco-reinit written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD---reinit)-2e-05lr-3/readme.md\n",
      "JonBERTa-head-ft-coco-proj written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD--proj-)-2e-05lr-4/readme.md\n",
      "JonBERTa-head-ft-coco-proj-reinit written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD--proj-reinit)-2e-05lr-1/readme.md\n",
      "JonBERTa-head-ft-coco-dense written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense--)-2e-05lr-4/readme.md\n",
      "JonBERTa-head-ft-coco-dense-reinit written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense--reinit)-2e-05lr-2/readme.md\n",
      "JonBERTa-head-ft-coco-dense-proj written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense-proj-)-2e-05lr-4/readme.md\n",
      "JonBERTa-head-ft-coco-dense-proj-reinit written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense-proj-reinit)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-012L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 2]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-013L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 3]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-014L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 4]L)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-015L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1, 5]L)-2e-05lr-1/readme.md\n",
      "JonBERTa-attn-ft-coco-01L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 1]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-023L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2, 3]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-024L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2, 4]L)-2e-05lr-1/readme.md\n",
      "JonBERTa-attn-ft-coco-025L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2, 5]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-02L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 2]L)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-034L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 3, 4]L)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-035L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 3, 5]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-03L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 3]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-045L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 4, 5]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-04L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 4]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-05L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0, 5]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-123L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2, 3]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-124L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2, 4]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-125L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2, 5]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-12L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 2]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-134L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 3, 4]L)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-135L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 3, 5]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-13L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 3]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-145L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 4, 5]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-14L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 4]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-15L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1, 5]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-234L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 3, 4]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-235L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 3, 5]L)-2e-05lr-0/readme.md\n",
      "JonBERTa-attn-ft-coco-23L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 3]L)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-245L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 4, 5]L)-2e-05lr-1/readme.md\n",
      "JonBERTa-attn-ft-coco-24L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 4]L)-2e-05lr-2/readme.md\n",
      "JonBERTa-attn-ft-coco-25L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2, 5]L)-2e-05lr-1/readme.md\n",
      "JonBERTa-attn-ft-coco-345L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3, 4, 5]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-34L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3, 4]L)-2e-05lr-4/readme.md\n",
      "JonBERTa-attn-ft-coco-35L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3, 5]L)-2e-05lr-1/readme.md\n",
      "JonBERTa-attn-ft-coco-45L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[4, 5]L)-2e-05lr-3/readme.md\n",
      "JonBERTa-attn-ft-coco-0L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0]L)-2e-05lr--3/readme.md\n",
      "JonBERTa-attn-ft-coco-1L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[1]L)-2e-05lr--2/readme.md\n",
      "JonBERTa-attn-ft-coco-2L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[2]L)-2e-05lr--2/readme.md\n",
      "JonBERTa-attn-ft-coco-3L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[3]L)-2e-05lr--0/readme.md\n",
      "JonBERTa-attn-ft-coco-4L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[4]L)-2e-05lr--4/readme.md\n",
      "JonBERTa-attn-ft-coco-5L written to /home/jovyan/work/code4me-analysis/notebooks/paper/results/13_jonberta/12_codeberta-biased-2e-05lr--0/model/13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[5]L)-2e-05lr--0/readme.md\n"
     ]
    }
   ],
   "source": [
    "model_type_roberta  = '''[RoBERTa](https://huggingface.co/FacebookAI/roberta-base)'''\n",
    "model_type_jonberta = '''[JonBERTa](https://github.com/Ar4l/curating-code-completions/blob/main/modeling_jonberta.py)'''\n",
    "\n",
    "codeberta_hyp = '''\n",
    "num_train_epochs : int = 6\n",
    "learning_rate    : float = search([2e-5, 1e-5, 5e-5])\n",
    "batch_size       : int = 16\n",
    "'''.strip()\n",
    "\n",
    "jonberta_hyp = '''\n",
    "num_train_epochs : int      = 3 \n",
    "learning_rate    : float    = 2e-5\n",
    "batch_size       : int      = 16\n",
    "'''.strip()\n",
    "\n",
    "jonberta_head_conf = '''\n",
    "num_telemetry_features :int = 26\n",
    "add_head              :bool = True\n",
    "add_dense             :bool = search([True, False])\n",
    "add_proj              :bool = search([True, False])\n",
    "reinit_head           :bool = search([True, False])\n",
    "'''.strip()\n",
    "\n",
    "jonberta_attn_conf = '''\n",
    "num_telemetry_features  :int = 26\n",
    "\n",
    "add_feature_embeddings :bool = True \n",
    "feature_hidden_size     :int = num_telemetry_features * 4\n",
    "feature_dropout_prob  :float = 0.1\n",
    "add_feature_bias       :bool = True\n",
    "\n",
    "add_self_attn          :bool = True\n",
    "self_attn_layers  :list[int] = search(sum(\n",
    "    [[i,j,k] for i in range(6) for j in range(6) for k in range(6) if i < j < k], \n",
    "    [[i,j] for j in range(6) for i in range(6) if i < j],\n",
    "    [[i] for i in range(6)],\n",
    "    []\n",
    "))\n",
    "'''.strip()\n",
    "\n",
    "model_card = '''\n",
    "---\n",
    "library_name: transformers\n",
    "tags:\n",
    "- code\n",
    "license: mit\n",
    "---\n",
    "\n",
    "## {}\n",
    "\n",
    "Model for the paper [**\"A Transformer-Based Approach for Smart Invocation of Automatic Code Completion\"**](https://arxiv.org/abs/2405.14753). \n",
    "\n",
    "#### Description\n",
    "This model is fine-tuned on a code-completion dataset collected from the open-source [Code4Me](https://github.com/code4me-me/code4me) plugin. The training objective is to have a small, lightweight transformer model to filter out unnecessary and unhelpful code completions. To this end, we leverage the in-IDE telemetry data, and integrate it with the textual code data in the transformer's attention module. \n",
    "\n",
    "- **Developed by:** [AISE Lab](https://www.linkedin.com/company/aise-tudelft/) @ [SERG](https://se.ewi.tudelft.nl/), Delft University of Technology \n",
    "- **Model type:** {}\n",
    "- **Language:** Code \n",
    "- **Finetuned from model:** [`CodeBERTa-small-v1`](https://huggingface.co/huggingface/CodeBERTa-small-v1). \n",
    "\n",
    "Models are named as follows: \n",
    "\n",
    "- `CodeBERTa` &rarr; `CodeBERTa-ft-coco-[1,2,5]e-05lr`\n",
    "    - e.g. `CodeBERTa-ft-coco-2e-05lr`, which was trained with learning rate of `2e-05`.\n",
    "- `JonBERTa-head` &rarr; `JonBERTa-head-ft-[dense,proj,reinit]` \n",
    "    - e.g. `JonBERTa-head-ft-dense-proj`, where all have `2e-05` learning rate, but may differ in the head layer in which the telemetry features are introduced (either `head` or `proj`, with optional `reinit`ialisation of all its weights).\n",
    "- `JonBERTa-attn` &rarr; `JonBERTa-attn-ft-[0,1,2,3,4,5]L`\n",
    "    - e.g. `JonBERTa-attn-ft-012L` , where all have `2e-05` learning rate, but may differ in the attention layer(s) in which the telemetry features are introduced (either `0`, `1`, `2`, `3`, `4`, or `5L`).\n",
    "\n",
    "Other hyperparameters may be found in the paper or the replication package (see below).\n",
    "\n",
    "#### Sources \n",
    "\n",
    "- **Replication Repository:** [`Ar4l/curating-code-completions`](https://github.com/Ar4l/curating-code-completions/tree/main)\n",
    "- **Paper:** [**\"A Transformer-Based Approach for Smart Invocation of Automatic Code Completion\"**](https://arxiv.org/abs/2405.14753) \n",
    "- **Contact:** https://huggingface.co/Ar4l\n",
    "\n",
    "To cite, please use \n",
    "\n",
    "```bibtex\n",
    "@misc{{de_moor_smart_invocation_2024,\n",
    "\ttitle = {{A {{Transformer}}-{{Based}} {{Approach}} for {{Smart}} {{Invocation}} of {{Automatic}} {{Code}} {{Completion}}}},\n",
    "\turl = {{http://arxiv.org/abs/2405.14753}},\n",
    "\tdoi = {{10.1145/3664646.3664760}},\n",
    "\tauthor = {{de Moor, Aral and van Deursen, Arie and Izadi, Maliheh}},\n",
    "\tmonth = may,\n",
    "\tyear = {{2024}},\n",
    "}}\n",
    "```\n",
    "\n",
    "#### Training Details \n",
    "This model was trained with the following hyperparameters, everything else being `TrainingArguments`' default. The dataset was prepared identically across all models as detailed in the paper. \n",
    "\n",
    "```python\n",
    "{}\n",
    "```\n",
    "\n",
    "{}\n",
    "'''.strip()\n",
    "\n",
    "headconfig = '''\n",
    "#### Model Configuration\n",
    "\n",
    "```python\n",
    "{}\n",
    "```\n",
    "'''.strip()\n",
    "\n",
    "def get_model_card(model_type, model_name): \n",
    "    ''' 1. Model type (JonBERTa-head/attn/CodeBERTa)\n",
    "        2. Model type repo [RoBERTa](https://huggingface.co/FacebookAI/roberta-base) or [JonBERTa](https://github.com/Ar4l/curating-code-completions/blob/main/modeling_jonberta.py)\n",
    "        3. Hyperparameters \n",
    "        4. IF JonBERTa: add JonBERTaConfig head/attn\n",
    "    '''\n",
    "\n",
    "    return model_card.format(\n",
    "\n",
    "        model_name, \n",
    "\n",
    "        model_type_roberta if model_type == 'CodeBERTa' else model_type_jonberta,\n",
    "\n",
    "        codeberta_hyp if model_type == 'CodeBERTa' else jonberta_hyp,\n",
    "\n",
    "        headconfig.format(jonberta_head_conf) if 'head' in model_type else \\\n",
    "        headconfig.format(jonberta_attn_conf) if 'attn' in model_type else \\\n",
    "        ''\n",
    "    )\n",
    "\n",
    "for model_type, model_paths in model_dirs.items():\n",
    "    for model_path in model_paths: \n",
    "\n",
    "        card_file = os.path.join(model_path, 'readme.md')\n",
    "        with open(card_file, 'w') as f:\n",
    "            f.write(get_model_card(model_type, get_model_name(model_path)))\n",
    "            print(get_model_name(model_path), 'written to', card_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### My greatest enemy: HF APIs\n",
    "We just need to (1) load in models, (2) push to hub under `AISE-TUDelft`, (3) add a model card. How difficult can it be? \n",
    "\n",
    "The fact that there is a `ModelCard` library makes a trivial task of uploading a `readme.md` nontrivial. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
      "Token is valid (permission: write).\n",
      "Your token has been saved to /home/jovyan/.cache/huggingface/token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login, Repository, ModelCard, get_collection, add_collection_item, login\n",
    "import tqdm\n",
    "\n",
    "login('token', write_permission=True)\n",
    "\n",
    "org_name = 'AISE-TUDelft' \n",
    "collection_name = 'smart-invocation-of-code-completion-66473ddf6fa6cf6e541f750c'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: both add_dense and add_proj are False, so this head will function like RoBERTa's\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ad692c2f3f44092a3a68ce00d3eb2cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/3.16k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco\n",
      "WARNING: both add_dense and add_proj are False, so this head will function like RoBERTa's\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9adab9a022424913b69ae8ed0b7b689b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/3.17k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-reinit\n",
      "expanding classifier.out_proj\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1cee018503b404ca8c1cd9f43fab9e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/3.16k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-proj\n",
      "expanding classifier.out_proj\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b428b093109143bcaf9051036c733f38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/334M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-proj-reinit\n",
      "expanding classifier.dense\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "453de108424344f4a1d0c8027edceb28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/334M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-dense\n",
      "expanding classifier.dense\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d740b81a5df404484fb68f04ae8eaf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/334M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-dense-reinit\n",
      "expanding both dense and proj\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8892ce7607b644aaaebd2d50b7e1f092",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/334M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-dense-proj\n",
      "expanding both dense and proj\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0182868c1cd4604b9f5c696637d835a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/334M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2/3 [02:27<01:13, 73.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-head-ft-coco-dense-proj-reinit\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 2\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ff476897acb4711bb6a5eab6ee7e6ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-012L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c36489f5f3224caca580b161dc79b505",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-013L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e1cddb0bdf24cb5aacff46f21f8adef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-014L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e8bf8c0f5404048872d6e29c47ed0f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-015L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 1\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4117db50be694dcaaa8b43723cea5433",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-01L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d88eeb6fac904417b2b6f9b86e498453",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-023L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70101211a5594d6caf94fef7bd9fe51f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-024L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e88eba5e19c844ecb2f80f7a3c0cfff7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-025L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 2\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35d3adf03532401892382760fd040310",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-02L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa3a7e0d4241494d922bb1375bb8da95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-034L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc4b0fcaff5a42dba0424d98bbe31f85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-035L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9dcc365323f94707a190f239d3458c1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-03L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 4\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45de18d744524a628cbc43bf90941abc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-045L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c9841156b3b4c50a425f3383cba3498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-04L\n",
      "Adding custom self-attention to layer 0\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3264f9fc54fd42219c5c05be1062d6b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-05L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94f04add45b1402fb240fcb1ea16ae97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-123L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca360d3fd53f4c15b625d5d815091c98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-124L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c521e44440a420bbe591c60d73a4cb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-125L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 2\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c97a5d1c4704b5c8a1d8ff84472f5aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-12L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a38e87c855b4dd6a232c89f7fbf0993",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-134L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98c574320c1647b0abdf601a9c6b6263",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-135L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2c43e4e0d9e479a8c273670570dcb6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-13L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 4\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c9c5bc810c34c459c3f7cb1d899f792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-145L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36a0bb78b1774b5dad469f6cb7a71efe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-14L\n",
      "Adding custom self-attention to layer 1\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4994f7101214c05bcc259ca70b49041",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-15L\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5d1ac3a46fd42ee974c106d247d2666",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-234L\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e82dbec79094cbf85fc7c3cf7467ac9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-235L\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d7823ee02324de1a4c7c246cc87555c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-23L\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 4\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25b2ea67ec0f4d18afb2e2e502904d33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-245L\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56e68351871f496faabae222d8580b5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-24L\n",
      "Adding custom self-attention to layer 2\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d364e2f5857f4ee1be840f1bce1e8d6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-25L\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 4\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c2de95cecff4b81bcc7803188986969",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/338M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-345L\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d80bfa656414227a8499bd799c508f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-34L\n",
      "Adding custom self-attention to layer 3\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6163397b886c47c19da721565cb3aae9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-35L\n",
      "Adding custom self-attention to layer 4\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dac767c91ca64a2197246b78faeadfcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/336M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-45L\n",
      "Adding custom self-attention to layer 0\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af8c5dc052e046c9b3d9fd7f4e4ce2c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-0L\n",
      "Adding custom self-attention to layer 1\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4992554369a8406a9135be1bab25dcb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-1L\n",
      "Adding custom self-attention to layer 2\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52948691893f40ecb8b7ba6d581c7684",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-2L\n",
      "Adding custom self-attention to layer 3\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74ae8a8e593a4c2a90ea55b25fbca632",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-3L\n",
      "Adding custom self-attention to layer 4\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81a8534f6606433c9402af779690571c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-4L\n",
      "Adding custom self-attention to layer 5\n",
      "<All keys matched successfully>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bf99b171f9a41c78f4cbd88f6689b51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/335M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [17:54<00:00, 358.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uploaded JonBERTa-attn-ft-coco-5L\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for model_type, model_paths in tqdm.tqdm(model_dirs.items()):\n",
    "\n",
    "    # already done but my kernel keeps crashing\n",
    "    if model_type == 'CodeBERTa': continue \n",
    "\n",
    "    for model_path in model_paths:\n",
    "\n",
    "        model_name = get_model_name(model_path)\n",
    "        hf_model = get_model(model_path)\n",
    "        hf_model_card = ModelCard(get_model_card(model_type, model_name))\n",
    "\n",
    "        hf_path = os.path.join(org_name, model_name)\n",
    "        hf_model.push_to_hub(repo_id=hf_path)\n",
    "        hf_model_card.push_to_hub(repo_id=hf_path)\n",
    "\n",
    "        hf_collection = os.path.join(org_name, collection_name)\n",
    "        add_collection_item(hf_collection, hf_path, 'model', exists_ok=True)\n",
    "\n",
    "        print(f'uploaded {model_name}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "temp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}