false_preds.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "false_preds.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "05608bf184994a75bc199b2f021c3a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aaade71f67ca4b639c1a71b5838ceb41",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e5e7f7f8fd564988b47f4824789f1515",
              "IPY_MODEL_ae5eebb5c4b4442190ff9f9a38399179"
            ]
          }
        },
        "aaade71f67ca4b639c1a71b5838ceb41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5e7f7f8fd564988b47f4824789f1515": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5f840fa9d0a54c4eadbf3e72e7b4c287",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2cbdf7eaa2a04b2a8d8a0632e9503733"
          }
        },
        "ae5eebb5c4b4442190ff9f9a38399179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_41aeeee051d544c1baec618c34122575",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 742kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7db68dfa5a8f4f15bc4772118af38db9"
          }
        },
        "5f840fa9d0a54c4eadbf3e72e7b4c287": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2cbdf7eaa2a04b2a8d8a0632e9503733": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41aeeee051d544c1baec618c34122575": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7db68dfa5a8f4f15bc4772118af38db9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e39c2c1f0be48a6b47f5bb85cad14e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_409cccdda6a5430395c40973b7c7f0a6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_af172542e11b4b3ba1f056f9b7626107",
              "IPY_MODEL_3637544406c243259bf257e7ec927740"
            ]
          }
        },
        "409cccdda6a5430395c40973b7c7f0a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af172542e11b4b3ba1f056f9b7626107": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_373e6aab65a140afb71614e032826ecd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c51d97e17854aac824294588a0ea365"
          }
        },
        "3637544406c243259bf257e7ec927740": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_96982c2bf23348b18ed3248511f43d7f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 162B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_994bc0589f0545e6acd1a2dfa4c5cda8"
          }
        },
        "373e6aab65a140afb71614e032826ecd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c51d97e17854aac824294588a0ea365": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "96982c2bf23348b18ed3248511f43d7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "994bc0589f0545e6acd1a2dfa4c5cda8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5de3ee0ec5804f93a4d3703b1c2bdcc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5a5ba25025ab4849a6b507f397f3b303",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4ec40b4e7943463899e518df57e28539",
              "IPY_MODEL_c7d5137bef0445b2970d9116f7ae4e2d"
            ]
          }
        },
        "5a5ba25025ab4849a6b507f397f3b303": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4ec40b4e7943463899e518df57e28539": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0e00ace740ca41b985867e4ec9d47dd9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c5cd6bb8df449b2b88a7dab94a4854e"
          }
        },
        "c7d5137bef0445b2970d9116f7ae4e2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dd713ec8698f42aeb594bf44346cd1e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 5.42MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d7d0756f0ff3483f82bb28fb8d08a6d2"
          }
        },
        "0e00ace740ca41b985867e4ec9d47dd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c5cd6bb8df449b2b88a7dab94a4854e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd713ec8698f42aeb594bf44346cd1e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d7d0756f0ff3483f82bb28fb8d08a6d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e8ffa37c79c4fe198d4b0d62f18a6b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_443938508bf644b0a6ba144cf7dc6e2e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3aad3a4836bc495282de665dcab6a172",
              "IPY_MODEL_3ae36a6f499d4216b2708fffb6d44e38"
            ]
          }
        },
        "443938508bf644b0a6ba144cf7dc6e2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3aad3a4836bc495282de665dcab6a172": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3a5245690ed64be7b1a9b2ff546215ed",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_09c840112e7d4551a0e28590eeac9989"
          }
        },
        "3ae36a6f499d4216b2708fffb6d44e38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_04ee752d3cbe459bae28263dc06e4083",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:11&lt;00:00, 38.7B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b6ab6933072e4d4db6cb70cc0e7d077a"
          }
        },
        "3a5245690ed64be7b1a9b2ff546215ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "09c840112e7d4551a0e28590eeac9989": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "04ee752d3cbe459bae28263dc06e4083": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b6ab6933072e4d4db6cb70cc0e7d077a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0e52308776d7490baefddac6f7a9b8bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b7a6e3b9548d4347808a13133e50196a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9627a312be8c446eb7bba6f15ae8377e",
              "IPY_MODEL_46a6f27e23eb4a87820a7444f90ddd38"
            ]
          }
        },
        "b7a6e3b9548d4347808a13133e50196a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9627a312be8c446eb7bba6f15ae8377e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_300d453d198a4f98b00e175df2ab4e79",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b44c37b0689f43d98088fae94e290815"
          }
        },
        "46a6f27e23eb4a87820a7444f90ddd38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f253a90bcd7a48648e8a6e1e8fa26b10",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:10&lt;00:00, 40.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b27c8b0ebf764710a5496da5cff9fd3a"
          }
        },
        "300d453d198a4f98b00e175df2ab4e79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b44c37b0689f43d98088fae94e290815": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f253a90bcd7a48648e8a6e1e8fa26b10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b27c8b0ebf764710a5496da5cff9fd3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CJ0cpSzaYquf"
      },
      "source": [
        "# This Notebook looks at false predictions our models make and compares them to find commonalities among them. \n",
        "\n",
        "We also specifically look at cases where the text_only model makes mistakes, which are corrected by our multimodal MMBT model. Even though the MMBT model has slightly lower accuracy than the text-only model, we do find some cases where it in fact makes correct predictions, while the text-only model does not. Since the text remains contant in these cases we hypothesize that there must be some cross-modal interactions that explain this behaviour."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4I-tEWc-aAu5"
      },
      "source": [
        "## Skip Unless running in Google Colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gbcc0iwU2Z9n",
        "outputId": "f5225334-c1ca-4637-96fd-993836515fe4"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "wAwUUimTyz58",
        "outputId": "ac45dc57-6c60-4354-d783-98b895758b43"
      },
      "source": [
        "%pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kARAPI3_y9u7",
        "outputId": "1f2e8664-d19f-4197-c62b-95f44265c4be"
      },
      "source": [
        "%cd /content/drive/MyDrive/LAP_MMBT"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/.shortcut-targets-by-id/1gwgx4ZApTKz5fN6SG9YkiVjVCZ0WNGeH/LAP_MMBT\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v31DXo7y6GIo",
        "outputId": "5f00dda7-19b8-4d5e-9e7d-d83e37204eaa"
      },
      "source": [
        "%ls"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "baseline_experiments_results.ipynb  \u001b[0m\u001b[01;34mmmbt_output_major_findings_10epochs\u001b[0m/\n",
            "\u001b[01;34mdata\u001b[0m/                               \u001b[01;34m__pycache__\u001b[0m/\n",
            "false_preds.ipynb                   \u001b[01;34mresults\u001b[0m/\n",
            "image_submodel.ipynb                run_bert_text_only.ipynb\n",
            "\u001b[01;34mintegrated_gradients\u001b[0m/               run_mmbt.ipynb\n",
            "\u001b[01;34mMMBT\u001b[0m/                               run_mmbt_masked_text_eval.ipynb\n",
            "\u001b[01;34mmmbt_experiment_logs_results\u001b[0m/       \u001b[01;34mruns\u001b[0m/\n",
            "\u001b[01;34mmmbt_output_findings_10epochs\u001b[0m/      textBert_utils.py\n",
            "\u001b[01;34mmmbt_output_findings_10epochs_n\u001b[0m/    \u001b[01;34mtext_only_experiment_logs_results\u001b[0m/\n",
            "\u001b[01;34mmmbt_output_findings_4epochs\u001b[0m/       \u001b[01;34mwandb\u001b[0m/\n",
            "\u001b[01;34mmmbt_output_major_8epochs\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RRP3UXaaMwe"
      },
      "source": [
        "## Configure device"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3_3rHDZCzCTD",
        "outputId": "e877f97c-869d-49f5-c069-c6d0bb6e5992"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FCGE7pX9aTDS"
      },
      "source": [
        "## Install Transformers library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5x-N8RUNzoDC",
        "outputId": "78416b31-bbed-4d75-b13a-54133190ad44"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/d5/f4157a376b8a79489a76ce6cfe147f4f3be1e029b7144fa7b8432e8acb26/transformers-4.4.2-py3-none-any.whl (2.0MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0MB 18.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 47.8MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 51.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=a562485331a409bcbb159bcaed49b73ed427ea22bed6885335466cc912395df4\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.4.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VLo91qJ-7DRn"
      },
      "source": [
        "## Resolve current directory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L_OONLqE6jNC",
        "outputId": "02b8fb62-2c3b-4264-d028-72cc4a4d0512"
      },
      "source": [
        "from pathlib import Path\n",
        "import os\n",
        "\n",
        "try:\n",
        "    FILE_DIR = os.path.dirname(os.path.abspath(__file__))\n",
        "except NameError:\n",
        "    print('__file__ does not exist for notebook, use current directory instead')\n",
        "    FILE_DIR = Path().resolve()\n",
        "    \n",
        "print(f'current directory is: {FILE_DIR}')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__file__ does not exist for notebook, use current directory instead\n",
            "current directory is: /content/drive/.shortcut-targets-by-id/1gwgx4ZApTKz5fN6SG9YkiVjVCZ0WNGeH/LAP_MMBT\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DrQ55zFH7SeJ"
      },
      "source": [
        "## Load Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cSn6wbzhz-Gl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5638d233-e046-4cfd-9a49-94eb3029249c"
      },
      "source": [
        "import pandas as pd\n",
        "pd.set_option('display.max_colwidth', -1)\n",
        "pd.options.display.max_rows = 580"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L18XRia4z_jc",
        "outputId": "303db935-1ae7-4e86-d582-ae1777830a01"
      },
      "source": [
        "DATA_DIR = os.path.join(FILE_DIR, 'data')\n",
        "\n",
        "CSV_PATH = os.path.join(DATA_DIR, 'csv')\n",
        "\n",
        "major = '/image_labels_major_findings_frontal_test.csv'\n",
        "baseline = '/image_labels_findings_frontal_test.csv'\n",
        "\n",
        "major = pd.read_csv(CSV_PATH+major)\n",
        "baseline = pd.read_csv(CSV_PATH+baseline)\n",
        "\n",
        "# all the test sets are of equal length\n",
        "print(f\"There are {len(baseline)} items in the dataset.\")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 570 items in the dataset.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qh6iqoFBi333"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cyx0BCy9_z8u"
      },
      "source": [
        "# sentences and labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xLSXvlOc_zT8"
      },
      "source": [
        "major_sentences = major.text.values\n",
        "major_labels = major.label.values\n",
        "\n",
        "base_sentences = baseline.text.values\n",
        "base_labels = baseline.label.values"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "edE59UKxAp0O"
      },
      "source": [
        "# Max sentence length (training)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181,
          "referenced_widgets": [
            "05608bf184994a75bc199b2f021c3a42",
            "aaade71f67ca4b639c1a71b5838ceb41",
            "e5e7f7f8fd564988b47f4824789f1515",
            "ae5eebb5c4b4442190ff9f9a38399179",
            "5f840fa9d0a54c4eadbf3e72e7b4c287",
            "2cbdf7eaa2a04b2a8d8a0632e9503733",
            "41aeeee051d544c1baec618c34122575",
            "7db68dfa5a8f4f15bc4772118af38db9",
            "5e39c2c1f0be48a6b47f5bb85cad14e9",
            "409cccdda6a5430395c40973b7c7f0a6",
            "af172542e11b4b3ba1f056f9b7626107",
            "3637544406c243259bf257e7ec927740",
            "373e6aab65a140afb71614e032826ecd",
            "3c51d97e17854aac824294588a0ea365",
            "96982c2bf23348b18ed3248511f43d7f",
            "994bc0589f0545e6acd1a2dfa4c5cda8",
            "5de3ee0ec5804f93a4d3703b1c2bdcc3",
            "5a5ba25025ab4849a6b507f397f3b303",
            "4ec40b4e7943463899e518df57e28539",
            "c7d5137bef0445b2970d9116f7ae4e2d",
            "0e00ace740ca41b985867e4ec9d47dd9",
            "4c5cd6bb8df449b2b88a7dab94a4854e",
            "dd713ec8698f42aeb594bf44346cd1e7",
            "d7d0756f0ff3483f82bb28fb8d08a6d2"
          ]
        },
        "id": "grUOixuaA9B7",
        "outputId": "2f00cb1a-0cf0-438f-fa07-73c8b5123ae3"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "05608bf184994a75bc199b2f021c3a42",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5e39c2c1f0be48a6b47f5bb85cad14e9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5de3ee0ec5804f93a4d3703b1c2bdcc3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KQ93D7WPApIm",
        "outputId": "31c027a8-9479-4c8c-ce70-ac5499250b3c"
      },
      "source": [
        "max_len = 0\n",
        "\n",
        "# For every sentence...\n",
        "for sent in major_sentences:\n",
        "\n",
        "    # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n",
        "    input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "\n",
        "    # Update the maximum sentence length.\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max sentence length:  165\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NpAQZo4SDl-Z"
      },
      "source": [
        "#default sentence length\n",
        "MAX_SENT_LEN = max_len # default: 256"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s5xvzVhnbMRF"
      },
      "source": [
        "## Helper functions for labeling and multiclass criterion"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pkoi56y3EX4e"
      },
      "source": [
        "# Tokenize and Encode data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aafjfvosB1yy"
      },
      "source": [
        "def tokenize_and_encode_data(sentences_iterable, tokenizer_encoder, max_sent_len, labels_iterable):\n",
        "    # Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent, label in zip(sentences_iterable, labels_iterable):\n",
        "        # `encode_plus` will:\n",
        "        #   (1) Tokenize the sentence.\n",
        "        #   (2) Prepend the `[CLS]` token to the start.\n",
        "        #   (3) Append the `[SEP]` token to the end.\n",
        "        #   (4) Map tokens to their IDs.\n",
        "        #   (5) Pad or truncate the sentence to `max_length`\n",
        "        #   (6) Create attention masks for [PAD] tokens.\n",
        "        encoded_dict = tokenizer_encoder.encode_plus(\n",
        "                            sent,                      # Sentence to encode.\n",
        "                            add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                            max_length = max_sent_len,           # Pad & truncate all sentences.\n",
        "                            padding = 'max_length',\n",
        "                            return_attention_mask = True,   # Construct attn. masks.\n",
        "                            return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                    )\n",
        "        \n",
        "        # Add the encoded sentence to the list.    \n",
        "        input_ids.append(encoded_dict['input_ids'])\n",
        "        \n",
        "        # And its attention mask (simply differentiates padding from non-padding).\n",
        "        attention_masks.append(encoded_dict['attention_mask'])\n",
        "        \n",
        "\n",
        "    # Convert the lists into tensors.\n",
        "    input_ids = torch.cat(input_ids, dim=0)\n",
        "    attention_masks = torch.cat(attention_masks, dim=0)\n",
        "    input_labels = torch.tensor(labels_iterable)\n",
        "\n",
        "    # Print sentence 0, now as a list of IDs.\n",
        "    print('Original: ', sentences_iterable[0])\n",
        "    print('Token IDs:', input_ids[0])\n",
        "\n",
        "    return input_ids, attention_masks, input_labels"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QXmGl8hlBEn3",
        "outputId": "80d5b0c2-acd9-4486-a127-f6ea392700d0"
      },
      "source": [
        "major_input_ids, major_attention_masks, major_labels_tensors = tokenize_and_encode_data(major_sentences, tokenizer, MAX_SENT_LEN, major_labels)\n",
        "base_input_ids, base_attention_masks, base_labels_tensors = tokenize_and_encode_data(base_sentences, tokenizer, MAX_SENT_LEN, base_labels) "
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:   Lungs are clear without focal consolidation, effusion or pneumothorax. Normal heart size. Bony thorax and soft tissues unremarkable\n",
            "Token IDs: tensor([  101,  8948,  2024,  3154,  2302, 15918, 17439,  1010,  1041,  4246,\n",
            "        14499,  2030,  1052,  2638,  2819, 29288,  2527,  2595,  1012,  3671,\n",
            "         2540,  2946,  1012, 22678, 15321,  8528,  1998,  3730, 14095,  4895,\n",
            "        28578, 17007,  3085,   102,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0])\n",
            "Original:   Lungs are clear without focal consolidation, effusion or pneumothorax. Normal heart size. Bony thorax and soft tissues unremarkable\n",
            "Token IDs: tensor([  101,  8948,  2024,  3154,  2302, 15918, 17439,  1010,  1041,  4246,\n",
            "        14499,  2030,  1052,  2638,  2819, 29288,  2527,  2595,  1012,  3671,\n",
            "         2540,  2946,  1012, 22678, 15321,  8528,  1998,  3730, 14095,  4895,\n",
            "        28578, 17007,  3085,   102,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "49DUwTrmEl43"
      },
      "source": [
        "# Torch dataset and dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8-QTtgDFEkwD"
      },
      "source": [
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "major_dataset = TensorDataset(major_input_ids, major_attention_masks, major_labels_tensors)\n",
        "base_dataset = TensorDataset(base_input_ids, base_attention_masks, base_labels_tensors)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ray80zREF5nx"
      },
      "source": [
        "Create an iterator for the dataset using the torch DataLoader class. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MctnKe2mF1zf"
      },
      "source": [
        "from torch.utils.data import DataLoader, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it \n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch \n",
        "# size of 16 or 32.\n",
        "batch_size = 32\n",
        "\n",
        "\n",
        "\n",
        "# Dataloader for the major label based dataset\n",
        "major_dataloader = DataLoader(\n",
        "            major_dataset,  \n",
        "            sampler = SequentialSampler(major_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "            )\n",
        "\n",
        "# Dataloader for the impression label based dataset\n",
        "base_dataloader = DataLoader(\n",
        "            base_dataset,  \n",
        "            sampler = SequentialSampler(base_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "            )\n",
        "\n"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zARn-95bGbYc"
      },
      "source": [
        "# Fine Tune BERT for Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2QGJ_7wSbbNi"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cDh0rU6SL0a5"
      },
      "source": [
        "import os\n",
        "import random\n",
        "import sys\n",
        "import numpy as np\n",
        "import transformers\n",
        "from transformers import (\n",
        "    WEIGHTS_NAME,\n",
        "    AdamW,\n",
        "    AutoConfig,\n",
        "    AutoModelForSequenceClassification,\n",
        "    AutoTokenizer,\n",
        "    DataCollatorWithPadding,\n",
        "    EvalPrediction,\n",
        "    HfArgumentParser,\n",
        "    PretrainedConfig,\n",
        "    Trainer,\n",
        "    TrainingArguments,\n",
        "    default_data_collator,\n",
        "    set_seed,\n",
        "    get_linear_schedule_with_warmup,\n",
        ")\n",
        "from transformers.trainer_utils import get_last_checkpoint, is_main_process\n",
        "\n",
        "from tqdm import tqdm, trange"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f0NTxYqzbhVq"
      },
      "source": [
        "## Model Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w58sidteRwj_"
      },
      "source": [
        "model_name = \"bert-base-uncased\"\n",
        "tokenizer_name = \"bert-base-uncased\"\n",
        "max_seq_length = max_len\n",
        "\n",
        "config_name = None\n",
        "\n",
        "n_gpu = torch.cuda.device_count() if torch.cuda.is_available() else 0\n",
        "\n",
        "my_seed = 42\n",
        "\n",
        "eval_batch_size = 32"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_jQZ9RmAbmU9"
      },
      "source": [
        "## Evaluation function\n",
        "\n",
        "This function is modified from the original eval function used in the text_only notebook. Instead of returning the eval results, it returns a dictionary containing the indices in the dataset as keys, where the prediction did not match the target. As values for the keys, it returns the prediction and target labels associated to these examples."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dw6mRRWfdjaL"
      },
      "source": [
        "def evaluate(model, tokenizer, dataloader):\n",
        "   \n",
        "    eval_loss = 0.0\n",
        "    nb_eval_steps = 0\n",
        "    preds = []\n",
        "    out_label_ids = []\n",
        "    for batch in tqdm(dataloader, desc=\"Batch Evaluating\"):\n",
        "        model.eval()\n",
        "        #batch = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            batch = tuple(t.to(device) for t in batch)\n",
        "            b_input_ids = batch[0]\n",
        "            b_input_mask = batch[1]\n",
        "            b_labels = batch[2]\n",
        "            \n",
        "            result = model(b_input_ids,\n",
        "                            token_type_ids=None,\n",
        "                            attention_mask=b_input_mask,\n",
        "                            labels=b_labels,\n",
        "                            return_dict=True)\n",
        "\n",
        "            logits = result.logits  # model outputs are always tuple in transformers (see doc)\n",
        "            tmp_eval_loss = result.loss\n",
        "\n",
        "            eval_loss += tmp_eval_loss.mean().item()\n",
        "        nb_eval_steps += 1\n",
        "        # Move logits and labels to CPU\n",
        "        pred = torch.nn.functional.softmax(logits, dim=1).argmax(dim=1).cpu().detach().numpy()\n",
        "        \n",
        "        out_label_id = b_labels.detach().cpu().numpy()\n",
        "        preds.append(pred)\n",
        "        out_label_ids.append(out_label_id)\n",
        "        \n",
        "    eval_loss = eval_loss / nb_eval_steps\n",
        "\n",
        "    \n",
        "    preds = [l for sl in preds for l in sl]\n",
        "    out_label_ids = [l for sl in out_label_ids for l in sl]\n",
        "    \n",
        "    idxs = {}\n",
        "    for i, (pred, label) in enumerate(zip(preds, out_label_ids)):    \n",
        "      if pred != label:\n",
        "        idxs.update({i: (pred, label)})\n",
        "\n",
        "    return idxs"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-9tiXIyycVhF"
      },
      "source": [
        "## Set random seed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hnCRpiQjT-w3"
      },
      "source": [
        "def set_seed(seed):\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    if n_gpu > 0:\n",
        "        torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "set_seed(my_seed)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y3HT1vZmofkp"
      },
      "source": [
        "# Evaluation on Test set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vOur0pn-oiay"
      },
      "source": [
        "## Basic model configurations\n",
        "\n",
        "+ path to pretrained model weights and dataloaders\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 220,
          "referenced_widgets": [
            "3e8ffa37c79c4fe198d4b0d62f18a6b0",
            "443938508bf644b0a6ba144cf7dc6e2e",
            "3aad3a4836bc495282de665dcab6a172",
            "3ae36a6f499d4216b2708fffb6d44e38",
            "3a5245690ed64be7b1a9b2ff546215ed",
            "09c840112e7d4551a0e28590eeac9989",
            "04ee752d3cbe459bae28263dc06e4083",
            "b6ab6933072e4d4db6cb70cc0e7d077a",
            "0e52308776d7490baefddac6f7a9b8bb",
            "b7a6e3b9548d4347808a13133e50196a",
            "9627a312be8c446eb7bba6f15ae8377e",
            "46a6f27e23eb4a87820a7444f90ddd38",
            "300d453d198a4f98b00e175df2ab4e79",
            "b44c37b0689f43d98088fae94e290815",
            "f253a90bcd7a48648e8a6e1e8fa26b10",
            "b27c8b0ebf764710a5496da5cff9fd3a"
          ]
        },
        "id": "tFrWAi5AvjWs",
        "outputId": "01781eb3-7061-4161-e8a3-ffa995b59bc2"
      },
      "source": [
        "MODELS_DIR = os.path.join(DATA_DIR, 'models')\n",
        "TEXT_ONLY_DIR = os.path.join(MODELS_DIR, 'text_only')\n",
        "\n",
        "models = [\"text_only_findings.bin\", \"text_only_major_findings.bin\"]\n",
        "dataloaders = [base_dataloader, major_dataloader]\n",
        "\n",
        "transformer_config = AutoConfig.from_pretrained(model_name)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\n",
        "        tokenizer_name if tokenizer_name else model_name,\n",
        "        do_lower_case=True,\n",
        "        cache_dir=None,\n",
        "    )\n",
        "transformer_model = AutoModelForSequenceClassification.from_pretrained(model_name, config=transformer_config)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e8ffa37c79c4fe198d4b0d62f18a6b0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0e52308776d7490baefddac6f7a9b8bb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gx1fHMOTeCZy"
      },
      "source": [
        "## Run base model\n",
        "\n",
        "Loads the pretrained weights and evaluates model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_tqvu4GppgRE",
        "outputId": "f09fadf0-49f6-49c1-d033-1f78d026a3ad"
      },
      "source": [
        "base_checkpoint = os.path.join(TEXT_ONLY_DIR, models[0])\n",
        "transformer_model.load_state_dict(torch.load(base_checkpoint))\n",
        "transformer_model.to(device)\n",
        "base_result = evaluate(transformer_model, tokenizer, dataloaders[0])\n",
        "base_idxs = list(base_result.keys())"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Batch Evaluating: 100%|██████████| 18/18 [00:05<00:00,  3.36it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1AST489OeM9O"
      },
      "source": [
        "## False predictions\n",
        "\n",
        "- Key: index in dataset\n",
        "- Value: tuple (prediction, target)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J40jFblh5M6N",
        "outputId": "04e16ed4-255a-4f5e-f2aa-2b421dcbf512"
      },
      "source": [
        "base_result"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{5: (0, 1),\n",
              " 10: (0, 1),\n",
              " 15: (0, 1),\n",
              " 19: (0, 1),\n",
              " 22: (0, 1),\n",
              " 49: (1, 0),\n",
              " 54: (1, 0),\n",
              " 56: (1, 0),\n",
              " 80: (1, 0),\n",
              " 86: (0, 1),\n",
              " 88: (0, 1),\n",
              " 91: (0, 1),\n",
              " 98: (0, 1),\n",
              " 99: (0, 1),\n",
              " 100: (1, 0),\n",
              " 106: (0, 1),\n",
              " 107: (0, 1),\n",
              " 108: (0, 1),\n",
              " 113: (0, 1),\n",
              " 115: (0, 1),\n",
              " 128: (1, 0),\n",
              " 136: (1, 0),\n",
              " 140: (0, 1),\n",
              " 143: (0, 1),\n",
              " 156: (0, 1),\n",
              " 185: (0, 1),\n",
              " 204: (1, 0),\n",
              " 213: (1, 0),\n",
              " 219: (0, 1),\n",
              " 224: (0, 1),\n",
              " 229: (0, 1),\n",
              " 241: (0, 1),\n",
              " 242: (1, 0),\n",
              " 260: (1, 0),\n",
              " 271: (0, 1),\n",
              " 273: (1, 0),\n",
              " 291: (0, 1),\n",
              " 296: (0, 1),\n",
              " 297: (0, 1),\n",
              " 305: (1, 0),\n",
              " 309: (1, 0),\n",
              " 313: (0, 1),\n",
              " 337: (1, 0),\n",
              " 357: (0, 1),\n",
              " 382: (1, 0),\n",
              " 410: (0, 1),\n",
              " 421: (0, 1),\n",
              " 422: (1, 0),\n",
              " 434: (0, 1),\n",
              " 437: (0, 1),\n",
              " 440: (0, 1),\n",
              " 442: (1, 0),\n",
              " 443: (0, 1),\n",
              " 458: (1, 0),\n",
              " 485: (1, 0),\n",
              " 487: (1, 0),\n",
              " 488: (0, 1),\n",
              " 493: (1, 0),\n",
              " 498: (0, 1),\n",
              " 505: (0, 1),\n",
              " 520: (1, 0),\n",
              " 526: (0, 1),\n",
              " 530: (1, 0),\n",
              " 531: (0, 1),\n",
              " 533: (0, 1),\n",
              " 542: (0, 1),\n",
              " 548: (0, 1),\n",
              " 553: (1, 0),\n",
              " 564: (0, 1),\n",
              " 567: (0, 1)}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qd46QK-75Peb",
        "outputId": "cd160f7b-6a89-4cb7-d0e2-c035dfc2fa7a"
      },
      "source": [
        "len(base_idxs)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "70"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WVojk8G6etAv"
      },
      "source": [
        "## Visualize false predictions from the dataset\n",
        "\n",
        "label = target label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "IGPX1IW7-_kk",
        "outputId": "f4afa589-b239-4e1f-bd5c-ea71859c20f8"
      },
      "source": [
        "base_false_preds = [baseline.iloc[idx] for idx in base_idxs]\n",
        "base_false_preds = pd.DataFrame(base_false_preds)\n",
        "base_false_preds"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>img</th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2672</td>\n",
              "      <td>CXR3205_IM-1513-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>No pneumothorax. Heart size is normal. No large pleural effusions. No focal airspace opacities. No definite visualized rib fractures.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>2029</td>\n",
              "      <td>CXR299_IM-1377-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Cardiac and mediastinal contours are within normal limits. The lungs are clear. Bony structures are intact. Small hiatal hernia.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>2280</td>\n",
              "      <td>CXR1439_IM-0282-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Lungs are XXXX. XXXX opacities are present in the left lung base. Heart size normal. Mediastinum normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>2380</td>\n",
              "      <td>CXR1911_IM-0593-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Cardiac and mediastinal contours are unremarkable. Pulmonary vascularity is within normal limits. No focal air space opacities, pleural effusion, or pneumothorax. XXXX are grossly unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2775</td>\n",
              "      <td>CXR3695_IM-1845-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The cardiac silhouette size is at the upper limits of normal. Central vascular markings are mildly prominent. The lungs are normally inflated with no focal airspace disease, pleural effusion, or pneumothorax. No acute bony abnormality.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>1628</td>\n",
              "      <td>CXR3355_IM-1609-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Cardiomediastinal silhouettes are within normal limits. Lungs are hyperexpanded. Lungs are clear without focal consolidation, pneumothorax, or pleural effusion. Bony thorax is unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>1707</td>\n",
              "      <td>CXR3494_IM-1699-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>There are low lung volumes with bronchovascular crowding. There is no focal consolidation. No visualized pneumothorax. Heart size is within normal limits. The cardiomediastinal contours is grossly normal in size and contour.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>50</td>\n",
              "      <td>CXR102_IM-0016-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal heart size. Clear, hyperaerated lungs. No pneumothorax. No pleural effusion. XXXX substernal density may be related to a pectus deformity.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>330</td>\n",
              "      <td>CXR655_IM-2231-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal heart size and mediastinal contours. The lungs are hyperinflated but clear. No pneumothorax or pleural effusion. No acute bony abnormalities.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>2493</td>\n",
              "      <td>CXR2390_IM-0944-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is borderline in size. The mediastinum is stable with changes of XXXX sternotomy and bypass graft. Aorta is atherosclerotic. There are postsurgical changes of the left hemithorax with mild left-sided volume loss as evidenced by diaphragm elevation. Left post thoracotomy rib changes are noted. The right lung is clear. There is no pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>2473</td>\n",
              "      <td>CXR2314_IM-0889-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The lungs are XXXX. XXXX opacities are present in the right costophrenic XXXX. No focal infiltrates. Heart size normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>2018</td>\n",
              "      <td>CXR257_IM-1072-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Midline sternotomy XXXX identified. Heart size and cardiomediastinal silhouette are grossly normal. Airspace opacity in posterior segment on the lateral view. Osseous structures are grossly intact.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>2138</td>\n",
              "      <td>CXR856_IM-2377-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Limited evaluation of the lateral view due to rotation and frontal view due to motion artifact. Stable mild cardiomegaly. Normal pulmonary vascularity. The lungs are clear. No focal consolidation, visible pneumothorax or large pleural effusions. XXXX XXXX opacities are related to overlying soft tissues. The posterior sulci are clear. Degenerative changes of the spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>2271</td>\n",
              "      <td>CXR1403_IM-0258-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Heart size is normal. Mild XXXX XXXX atelectasis. Lungs are otherwise clear. No pleural effusions or pneumothoraces. The hilar and mediastinal contours are normal. Normal pulmonary vascularity.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>1396</td>\n",
              "      <td>CXR2904_IM-1308-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Stable cardiomediastinal silhouette with borderline heart size and aortic ectasia/tortuosity. Stable hyperinflation without focal alveolar consolidation. No definite pleural effusion seen. No typical findings of pulmonary edema. Healed left rib fractures again noted.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>106</th>\n",
              "      <td>2827</td>\n",
              "      <td>CXR3917_IM-1992-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The lungs are clear bilaterally. Specifically, no evidence of focal consolidation, pneumothorax, or pleural effusion.. Mild cardiomegaly without acute cardiac abnormality. Visualized osseous structures of the thorax are without acute abnormality.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107</th>\n",
              "      <td>2038</td>\n",
              "      <td>CXR341_IM-1648-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Two-view chest. Both lungs are clear and expanded. Heart and mediastinum normal. Right foot. Hindfoot, midfoot, forefoot XXXX are intact with no fractures or bone destruction.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>2043</td>\n",
              "      <td>CXR357_IM-1753-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Cardiac size, mediastinal contour, and pulmonary vascularity are within normal limits. The right heart XXXX appears obscured and there are streaky right medial basilar airspace opacities, possibly due to airspace disease or atelectasis. Otherwise, no focal consolidation, pleural effusion, or pneumothorax. The visualized osseous structures appear intact.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>113</th>\n",
              "      <td>2741</td>\n",
              "      <td>CXR3529_IM-1725-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Status post midline sternotomy with intact XXXX XXXX. Stable mild cardiomegaly. Normal lung vascularity. The lungs are clear.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>115</th>\n",
              "      <td>2836</td>\n",
              "      <td>CXR3961_IM-2026-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is normal. The mediastinal contour is within normal limits. There is a streaky opacity within the right upper lobe. There are no nodules or masses. No visible pneumothorax. No visible pleural fluid. The XXXX are grossly normal. There is no visible free intraperitoneal air under the diaphragm.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>128</th>\n",
              "      <td>1817</td>\n",
              "      <td>CXR3708_IM-1852-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Lung volumes are decreased from XXXX, and there is resultant bronchovascular crowding. No evidence of focal airspace disease. No definite pleural effusion or pneumothorax. Cardiomediastinal silhouette is within normal limits given the low lung volumes. No free subdiaphragmatic air. Grossly stable mild degenerative changes of the right lower thoracic spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>136</th>\n",
              "      <td>6</td>\n",
              "      <td>CXR14_IM-0256-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Heart size within normal limits, stable mediastinal and hilar contours. Mild hyperinflation appears similar to prior. No focal alveolar consolidation, no definite pleural effusion seen. Scattered chronic appearing irregular interstitial markings, no typical findings of pulmonary edema.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>140</th>\n",
              "      <td>2028</td>\n",
              "      <td>CXR293_IM-1332-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Sternotomy XXXX mediastinal clips noted. Heart size within normal limits. Negative for focal pulmonary consolidation, pleural effusion, or pneumothorax. There is XXXX deformity of T6, XXXX since X-XXXX thoracic spine XXXX, XXXX. Mild XXXX deformity of T12, stable. Prior cholecystectomy.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>143</th>\n",
              "      <td>2622</td>\n",
              "      <td>CXR3005_IM-1388-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is normal in size. The mediastinum is stable. Aorta is tortuous. Calcified lymph XXXX are again identified. There is mild prominence of the right paratracheal soft tissues, stable in appearance from prior studies. There is no acute infiltrate or pleural effusion. Osteopenia and degenerative changes are identified. XXXX deformity of T9 appears worse than prior study.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>156</th>\n",
              "      <td>2429</td>\n",
              "      <td>CXR2153_IM-0773-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size and pulmonary vascularity appear within normal limits. The lungs are free of focal airspace disease. No pleural effusion or pneumothorax is seen. Left hemidiaphragm is elevated.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>185</th>\n",
              "      <td>2508</td>\n",
              "      <td>CXR2448_IM-0983-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>204</th>\n",
              "      <td>220</td>\n",
              "      <td>CXR455_IM-2086-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>There are low lung volumes with associated bronchovascular crowding and basilar subsegmental atelectasis. There is stable prominence of the right cardiac silhouette. The mediastinum is unremarkable. There is no pleural effusion, pneumothorax, or focal airspace disease. There are stable chronic degenerative changes of the thoracic spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>81</td>\n",
              "      <td>CXR172_IM-0474-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Elevated right hemidiaphragm. Clear lungs. No pleural effusions or pneumothoraces. heart size is upper limits of normal with tortuosity and ectasia of the aorta. Generative changes within the spine. In the upper lumbar spine there is an age-indeterminate wedge XXXX of a vertebral body.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>219</th>\n",
              "      <td>2359</td>\n",
              "      <td>CXR1816_IM-0528-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart and mediastinum are unremarkable. Again identified are numerous calcified mediastinal lymph XXXX as well as large calcifications within the left upper and left lower lobes. These appear similar to the patient's previous chest CT and are XXXX the sequela of prior granulomatous disease. The lungs are otherwise clear without infiltrate. There is no effusion or pneumothorax.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>1980</td>\n",
              "      <td>CXR108_IM-0056-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is normal in size. The mediastinum is Within normal limits the lungs are hypoinflated. There is mild increase in perihilar markings XXXX related to patient's history bronchitis. No acute infiltrate or pleural effusion are seen.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>229</th>\n",
              "      <td>2549</td>\n",
              "      <td>CXR2614_IM-1105-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is within normal limits. Prominent right paratracheal soft tissues XXXX representing adenopathy. No focal airspace consolidation, pleural effusions or pneumothorax. No acute bony abnormalities.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>1974</td>\n",
              "      <td>CXR75_IM-2303-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is stable. The aorta is ectatic and atherosclerotic but stable. XXXX sternotomy XXXX are again noted. The scarring in the left lower lobe is again noted and unchanged from prior exam. There are mild bilateral prominent lung interstitial opacities consistent with emphysematous disease. The calcified granulomas are stable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>242</th>\n",
              "      <td>396</td>\n",
              "      <td>CXR776_IM-2319-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Cardiomediastinal silhouette is within normal limits in size and appearance. Pulmonary vascularity is unremarkable. There are prominent coarse interstitial markings throughout the lungs, with more focal streaky bibasilar opacities, seen only on the frontal XXXX, XXXX atelectasis. Negative for focal airspace disease or consolidation. Negative for pneumothorax or pleural effusion. Limited evaluation reveals the XXXX XXXX are grossly intact.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>260</th>\n",
              "      <td>1260</td>\n",
              "      <td>CXR2605_IM-1095-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Single view of chest was obtained in AP projection. Limited study secondary to body habitus. The cardiomediastinal silhouette is not enlarged. Lungs demonstrate no focal infiltrates. There is no effusion or pneumothorax.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>271</th>\n",
              "      <td>2531</td>\n",
              "      <td>CXR2527_IM-1043-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The cardiomediastinal silhouette is within normal limits for size and contour. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. No acute osseus abnormality.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>273</th>\n",
              "      <td>774</td>\n",
              "      <td>CXR1555_IM-0362-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>XXXX XXXX and lateral chest examination was obtained. The heart silhouette and mediastinal contours are not enlarged. There is elevated right hemidiaphragm and evidence of right upper lobectomy. Lungs demonstrate no acute findings. There is no effusion or pneumothorax.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>291</th>\n",
              "      <td>2520</td>\n",
              "      <td>CXR2487_IM-1014-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Chest: The cardiomediastinal silhouette is within normal limits for size and contour. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. Thoracic spine: Mild dextro curvature the upper thoracic spine. Evaluation of the upper thoracic bodies is limited secondary to osseous overlap. Vertebral body XXXX and disc spaces are maintained. Mild degenerative endplate changes. Lumbar spine: There are 5 nonrib-bearing lumbar type vertebral bodies. Alignment is within normal limits. Vertebral body XXXX and disc spaces are maintained. Mild degenerative change without acute displaced fracture or dislocation. Moderate amount of stool..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>296</th>\n",
              "      <td>2539</td>\n",
              "      <td>CXR2578_IM-1078-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The lungs are clear. No suspicious pulmonary mass or nodule is identified. There is no pleural effusion or pneumothorax. Heart size and mediastinal contour are normal. There are sclerotic lesions within the XXXX, better visualized on the comparison XXXX scan. There are several bilateral rib fractures with evidence of the callus formation. The appearance is similar to the prior chest radiograph.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297</th>\n",
              "      <td>2509</td>\n",
              "      <td>CXR2451_IM-0987-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Chest. Heart size within normal limits. No focal airspace disease. No pneumothorax or effusions. Mild degenerative changes throughout the thoracic spine. Right knee. There has been prior ligamentous repair. There is tricompartmental joint space narrowing and marginal osteophyte formation which is severe in the medial compartment. No knee joint effusion. No fractures or dislocations.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>305</th>\n",
              "      <td>47</td>\n",
              "      <td>CXR98_IM-2467-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Frontal and lateral views of the chest show an unchanged cardiomediastinal silhouette. Reduced lung volumes with basilar atelectasis. No XXXX focal airspace consolidation or pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>309</th>\n",
              "      <td>72</td>\n",
              "      <td>CXR152_IM-0335-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Stable cardiomediastinal silhouette with mild cardiomegaly and aortic ectasia and tortuosity. No alveolar consolidation, no findings of pleural effusion. Chronic appearing bilateral rib contour deformities compatible with old fractures. No pneumothorax.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>313</th>\n",
              "      <td>2174</td>\n",
              "      <td>CXR1041_IM-0033-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Lucency crosses the 10th left posterior rib. Visualized portions of the thoracic spine are unremarkable. Mediastinal contours are normal. Lungs are clear. There is no pneumothorax or large pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>337</th>\n",
              "      <td>1410</td>\n",
              "      <td>CXR2934_IM-1337-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart is normal in size. The mediastinum is unremarkable. The lungs are mildly hyperlucent but clear. There is denser lumbar scoliosis.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>357</th>\n",
              "      <td>2164</td>\n",
              "      <td>CXR991_IM-2476-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is mild streaky perihilar opacity without confluent airspace opacity to suggest a bacterial pneumonia.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>382</th>\n",
              "      <td>527</td>\n",
              "      <td>CXR1037_IM-0029-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart is normal size. The mediastinum is unremarkable. A tortuous, calcified thoracic aorta is present. The lungs are hyperexpanded, consistent with emphysema. There is no pleural effusion, pneumothorax, or focal airspace disease. The XXXX are unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>410</th>\n",
              "      <td>2302</td>\n",
              "      <td>CXR1563_IM-0368-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Heart size within normal limits. No focal airspace opacities. No pneumothorax. No effusions. Mild degenerative changes of the thoracic spine. No XXXX deformities. Emphysematous changes.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>421</th>\n",
              "      <td>2554</td>\n",
              "      <td>CXR2631_IM-1118-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The cardiomediastinal silhouette is normal in size and contour. No focal consolidation, pneumothorax or large pleural effusion. Interval placement of right humeral prosthesis, incompletely evaluated. Incompletely evaluated the lumbar spine fusion XXXX. XXXX cholecystectomy.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>422</th>\n",
              "      <td>1397</td>\n",
              "      <td>CXR2908_IM-1312-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Heart size is upper limits of normal but stable. Mediastinal contours are within normal limits.. Chronically increased interstitial markings without focal airspace consolidation, pleural effusion, pneumothorax. Degenerative changes of the spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>434</th>\n",
              "      <td>2611</td>\n",
              "      <td>CXR2946_IM-1346-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The cardiomediastinal silhouette and pulmonary vasculature are within normal limits in size. Lungs are clear of focal airspace disease. There is no pneumothorax. There is mild blunting of the right costophrenic XXXX, without definite pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>437</th>\n",
              "      <td>2579</td>\n",
              "      <td>CXR2759_IM-1207-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The cardiomediastinal silhouette is within normal limits for appearance. No focal air is prominent consolidation. No pneumothorax. No large pleural effusion. Minimal degenerative changes of the thoracic spine with a mild wedge XXXX deformity of a midthoracic vertebral body. This is age-indeterminate.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>440</th>\n",
              "      <td>2117</td>\n",
              "      <td>CXR744_IM-2299-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size and pulmonary vascularity appear within normal limits. Left XXXX-a-XXXX is in XXXX. No pleural effusion or pneumothorax is seen. Right upper lobe area of dense opacity is seen in the medial right apex. On a previous outside XXXX scan (XXXX), the right upper lobe was consolidated. Comparison to the XXXX XXXX from that exam shows this opacity to have decreased. No films were available, however, for direct comparison.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>442</th>\n",
              "      <td>1152</td>\n",
              "      <td>CXR2364_IM-0927-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The lungs are clear. There is hyperexpansion of the lungs suggesting underlying emphysema. The heart and pulmonary XXXX appear normal. Pleural spaces are clear. Mediastinal contours are normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>443</th>\n",
              "      <td>2629</td>\n",
              "      <td>CXR3066_IM-1430-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is normal in size. The mediastinum is unremarkable. Mild emphysematous changes without focal consolidation. There is no pleural effusion. XXXX lingular scarring or atelectasis noted.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>458</th>\n",
              "      <td>620</td>\n",
              "      <td>CXR1262_IM-0178-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The cardiac silhouette mediastinal contours are within normal limits. There are low lung volumes. There is no focal consolidation. There is pneumothorax. There is no large pleural effusion. There is prominent right anterior first rib end.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>485</th>\n",
              "      <td>1194</td>\n",
              "      <td>CXR2465_IM-0997-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Heart size is mildly enlarged but stable.. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. Osseous structures are within normal limits for patient age..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>487</th>\n",
              "      <td>1609</td>\n",
              "      <td>CXR3316_IM-1586-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Heart size is normal. No focal airspace disease. Emphysema. Bibasilar pleural scarring. No pneumothorax or effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>488</th>\n",
              "      <td>2806</td>\n",
              "      <td>CXR3843_IM-1944-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>A right-sided chest XXXX remains in XXXX with the distal tip at the level of the mid SVC. The cardiomediastinal silhouette is within normal limits for appearance. No focal areas of pulmonary consolidation. No pulmonary nodules or mass lesions identified. No pneumothorax or pleural effusion. Mild degenerative changes of the thoracic spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>493</th>\n",
              "      <td>442</td>\n",
              "      <td>CXR864_IM-2384-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Stable cardiomediastinal silhouette with tortuous thoracic aorta. No pneumothorax, pleural effusion or suspicious focal air space opacity. Stable right lung base scarring.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>498</th>\n",
              "      <td>2136</td>\n",
              "      <td>CXR849_IM-2371-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Normal heart size and mediastinal contours. The lungs are free of any focal airspace disease. In the left lung base, there is a 9 mm nodule that not definitively calcified. No pneumothorax or pleural effusion. No acute bony abnormalities.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>505</th>\n",
              "      <td>2631</td>\n",
              "      <td>CXR3072_IM-1433-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Left-sided medication injection XXXX has its tip projecting at the cavoatrial junction. The trachea is midline. Extensive bilateral bronchiectasis, cystic changes, and scarring represents sequela from the patient's cystic fibrosis. No evidence of focal pulmonary infiltrate or pleural effusion. No large pneumothorax has developed in the interim. The overlying bony structures reveal no acute abnormalities. The heart size is normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>520</th>\n",
              "      <td>586</td>\n",
              "      <td>CXR1184_IM-0124-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart size and mediastinal contours appear within normal limits. Low lung volumes on the AP view with bronchovascular crowding and bibasilar atelectasis. No focal airspace consolidation, pleural effusions or pneumothorax. No acute bony abnormalities.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>526</th>\n",
              "      <td>2193</td>\n",
              "      <td>CXR1089_IM-0061-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size and pulmonary vascularity appear within normal limits. The descending thoracic aorta is tortuous. Central venous catheter is again noted. The lungs are free of focal airspace disease. The left hemidiaphragm remains elevated. No pneumothorax or pleural effusion is seen.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>530</th>\n",
              "      <td>1136</td>\n",
              "      <td>CXR2340_IM-0907-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Chest. Lung volumes are low, but no focal infiltrates are present. Heart and mediastinum remain normal. Abdomen. Multiple slightly distended loops are present from stomach to rectum. Formed stool is present in the rectum.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>531</th>\n",
              "      <td>2829</td>\n",
              "      <td>CXR3921_IM-1995-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The lungs are clear bilaterally. Specifically, no evidence of focal consolidation, pneumothorax, or pleural effusion.. Cardio mediastinal silhouette is unremarkable. Visualized osseous structures of the thorax are without acute abnormality. Left-sided PICC line has been placed in the interval with tip XXXX in the innominate vein.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>533</th>\n",
              "      <td>2209</td>\n",
              "      <td>CXR1126_IM-0082-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>XXXX sternotomy XXXX and mediastinal surgical clips remain in XXXX. The cardiomediastinal silhouette is stable in appearance. The thoracic aorta is tortuous and calcified with stable appearance since XXXX exam. No focal areas of pulmonary consolidation. Scattered right basilar subsegmental atelectasis. The left lung appears clear. No pneumothorax or pleural effusion present. Moderate degenerative changes of the thoracic spine. Osteopenia. Mild loss of XXXX of a mid thoracic vertebral body.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>542</th>\n",
              "      <td>2340</td>\n",
              "      <td>CXR1729_IM-0480-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Chronic bilateral emphysematous changes. The heart size and mediastinal silhouette are within normal limits for contour. The lungs are clear. No pneumothorax or pleural effusions. The XXXX are intact.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>548</th>\n",
              "      <td>2484</td>\n",
              "      <td>CXR2363_IM-0926-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is right basilar air space opacity.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>553</th>\n",
              "      <td>954</td>\n",
              "      <td>CXR1924_IM-0598-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Rotated apical lordotic frontal projection, mild bronchovascular crowding and scattered chronic appearing irregular interstitial markings. No definite focal alveolar consolidation or pleural effusion seen. Accounting for technical factors heart size XXXX within normal limits, heavily calcified and mildly tortuous aorta. No typical findings of pulmonary edema.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>564</th>\n",
              "      <td>2093</td>\n",
              "      <td>CXR605_IM-2194-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is normal in size. The mediastinum is unremarkable. Small nodular opacity left upper lobe may represent early infiltrate. The lungs are otherwise clear. There is no pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>567</th>\n",
              "      <td>2341</td>\n",
              "      <td>CXR1735_IM-0484-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The trachea is midline. The cardiomediastinal silhouette is normal in size and unchanged from prior examinations with sternotomy XXXX and surgical clips overlying. There is blunting of the right costophrenic XXXX which appears unchanged from prior examination and may be secondary to scarring or pleural thickening of the right lung base. There is no evidence of acute infiltrate. There is no pneumothorax. Visualized bony structures reveal no acute abnormalities.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               text\n",
              "5    2672        ...   No pneumothorax. Heart size is normal. No large pleural effusions. No focal airspace opacities. No definite visualized rib fractures.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "10   2029        ...   Cardiac and mediastinal contours are within normal limits. The lungs are clear. Bony structures are intact. Small hiatal hernia.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "15   2280        ...   Lungs are XXXX. XXXX opacities are present in the left lung base. Heart size normal. Mediastinum normal.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
              "19   2380        ...   Cardiac and mediastinal contours are unremarkable. Pulmonary vascularity is within normal limits. No focal air space opacities, pleural effusion, or pneumothorax. XXXX are grossly unremarkable.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "22   2775        ...   The cardiac silhouette size is at the upper limits of normal. Central vascular markings are mildly prominent. The lungs are normally inflated with no focal airspace disease, pleural effusion, or pneumothorax. No acute bony abnormality.                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "49   1628        ...   Cardiomediastinal silhouettes are within normal limits. Lungs are hyperexpanded. Lungs are clear without focal consolidation, pneumothorax, or pleural effusion. Bony thorax is unremarkable.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "54   1707        ...   There are low lung volumes with bronchovascular crowding. There is no focal consolidation. No visualized pneumothorax. Heart size is within normal limits. The cardiomediastinal contours is grossly normal in size and contour.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "56   50          ...   Normal heart size. Clear, hyperaerated lungs. No pneumothorax. No pleural effusion. XXXX substernal density may be related to a pectus deformity.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "80   330         ...   Normal heart size and mediastinal contours. The lungs are hyperinflated but clear. No pneumothorax or pleural effusion. No acute bony abnormalities.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
              "86   2493        ...   The heart is borderline in size. The mediastinum is stable with changes of XXXX sternotomy and bypass graft. Aorta is atherosclerotic. There are postsurgical changes of the left hemithorax with mild left-sided volume loss as evidenced by diaphragm elevation. Left post thoracotomy rib changes are noted. The right lung is clear. There is no pleural effusion.                                                                                                                                                                                                                                                                                                                                          \n",
              "88   2473        ...   The lungs are XXXX. XXXX opacities are present in the right costophrenic XXXX. No focal infiltrates. Heart size normal.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "91   2018        ...   Midline sternotomy XXXX identified. Heart size and cardiomediastinal silhouette are grossly normal. Airspace opacity in posterior segment on the lateral view. Osseous structures are grossly intact.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "98   2138        ...   Limited evaluation of the lateral view due to rotation and frontal view due to motion artifact. Stable mild cardiomegaly. Normal pulmonary vascularity. The lungs are clear. No focal consolidation, visible pneumothorax or large pleural effusions. XXXX XXXX opacities are related to overlying soft tissues. The posterior sulci are clear. Degenerative changes of the spine.                                                                                                                                                                                                                                                                                                                              \n",
              "99   2271        ...   Heart size is normal. Mild XXXX XXXX atelectasis. Lungs are otherwise clear. No pleural effusions or pneumothoraces. The hilar and mediastinal contours are normal. Normal pulmonary vascularity.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "100  1396        ...   Stable cardiomediastinal silhouette with borderline heart size and aortic ectasia/tortuosity. Stable hyperinflation without focal alveolar consolidation. No definite pleural effusion seen. No typical findings of pulmonary edema. Healed left rib fractures again noted.                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "106  2827        ...   The lungs are clear bilaterally. Specifically, no evidence of focal consolidation, pneumothorax, or pleural effusion.. Mild cardiomegaly without acute cardiac abnormality. Visualized osseous structures of the thorax are without acute abnormality.                                                                                                                                                                                                                                                                                                                                                                                                                                                          \n",
              "107  2038        ...   Two-view chest. Both lungs are clear and expanded. Heart and mediastinum normal. Right foot. Hindfoot, midfoot, forefoot XXXX are intact with no fractures or bone destruction.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "108  2043        ...   Cardiac size, mediastinal contour, and pulmonary vascularity are within normal limits. The right heart XXXX appears obscured and there are streaky right medial basilar airspace opacities, possibly due to airspace disease or atelectasis. Otherwise, no focal consolidation, pleural effusion, or pneumothorax. The visualized osseous structures appear intact.                                                                                                                                                                                                                                                                                                                                             \n",
              "113  2741        ...   Status post midline sternotomy with intact XXXX XXXX. Stable mild cardiomegaly. Normal lung vascularity. The lungs are clear.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "115  2836        ...   The heart size is normal. The mediastinal contour is within normal limits. There is a streaky opacity within the right upper lobe. There are no nodules or masses. No visible pneumothorax. No visible pleural fluid. The XXXX are grossly normal. There is no visible free intraperitoneal air under the diaphragm.                                                                                                                                                                                                                                                                                                                                                                                            \n",
              "128  1817        ...   Lung volumes are decreased from XXXX, and there is resultant bronchovascular crowding. No evidence of focal airspace disease. No definite pleural effusion or pneumothorax. Cardiomediastinal silhouette is within normal limits given the low lung volumes. No free subdiaphragmatic air. Grossly stable mild degenerative changes of the right lower thoracic spine.                                                                                                                                                                                                                                                                                                                                          \n",
              "136  6           ...   Heart size within normal limits, stable mediastinal and hilar contours. Mild hyperinflation appears similar to prior. No focal alveolar consolidation, no definite pleural effusion seen. Scattered chronic appearing irregular interstitial markings, no typical findings of pulmonary edema.                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "140  2028        ...   Sternotomy XXXX mediastinal clips noted. Heart size within normal limits. Negative for focal pulmonary consolidation, pleural effusion, or pneumothorax. There is XXXX deformity of T6, XXXX since X-XXXX thoracic spine XXXX, XXXX. Mild XXXX deformity of T12, stable. Prior cholecystectomy.                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "143  2622        ...   The heart is normal in size. The mediastinum is stable. Aorta is tortuous. Calcified lymph XXXX are again identified. There is mild prominence of the right paratracheal soft tissues, stable in appearance from prior studies. There is no acute infiltrate or pleural effusion. Osteopenia and degenerative changes are identified. XXXX deformity of T9 appears worse than prior study.                                                                                                                                                                                                                                                                                                                      \n",
              "156  2429        ...   The heart size and pulmonary vascularity appear within normal limits. The lungs are free of focal airspace disease. No pleural effusion or pneumothorax is seen. Left hemidiaphragm is elevated.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "185  2508        ...   Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "204  220         ...   There are low lung volumes with associated bronchovascular crowding and basilar subsegmental atelectasis. There is stable prominence of the right cardiac silhouette. The mediastinum is unremarkable. There is no pleural effusion, pneumothorax, or focal airspace disease. There are stable chronic degenerative changes of the thoracic spine.                                                                                                                                                                                                                                                                                                                                                              \n",
              "213  81          ...   Elevated right hemidiaphragm. Clear lungs. No pleural effusions or pneumothoraces. heart size is upper limits of normal with tortuosity and ectasia of the aorta. Generative changes within the spine. In the upper lumbar spine there is an age-indeterminate wedge XXXX of a vertebral body.                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "219  2359        ...   The heart and mediastinum are unremarkable. Again identified are numerous calcified mediastinal lymph XXXX as well as large calcifications within the left upper and left lower lobes. These appear similar to the patient's previous chest CT and are XXXX the sequela of prior granulomatous disease. The lungs are otherwise clear without infiltrate. There is no effusion or pneumothorax.                                                                                                                                                                                                                                                                                                                 \n",
              "224  1980        ...   The heart is normal in size. The mediastinum is Within normal limits the lungs are hypoinflated. There is mild increase in perihilar markings XXXX related to patient's history bronchitis. No acute infiltrate or pleural effusion are seen.                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "229  2549        ...   The heart size is within normal limits. Prominent right paratracheal soft tissues XXXX representing adenopathy. No focal airspace consolidation, pleural effusions or pneumothorax. No acute bony abnormalities.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "241  1974        ...   The heart size is stable. The aorta is ectatic and atherosclerotic but stable. XXXX sternotomy XXXX are again noted. The scarring in the left lower lobe is again noted and unchanged from prior exam. There are mild bilateral prominent lung interstitial opacities consistent with emphysematous disease. The calcified granulomas are stable.                                                                                                                                                                                                                                                                                                                                                               \n",
              "242  396         ...   Cardiomediastinal silhouette is within normal limits in size and appearance. Pulmonary vascularity is unremarkable. There are prominent coarse interstitial markings throughout the lungs, with more focal streaky bibasilar opacities, seen only on the frontal XXXX, XXXX atelectasis. Negative for focal airspace disease or consolidation. Negative for pneumothorax or pleural effusion. Limited evaluation reveals the XXXX XXXX are grossly intact.                                                                                                                                                                                                                                                      \n",
              "260  1260        ...   Single view of chest was obtained in AP projection. Limited study secondary to body habitus. The cardiomediastinal silhouette is not enlarged. Lungs demonstrate no focal infiltrates. There is no effusion or pneumothorax.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "271  2531        ...   The cardiomediastinal silhouette is within normal limits for size and contour. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. No acute osseus abnormality.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \n",
              "273  774         ...   XXXX XXXX and lateral chest examination was obtained. The heart silhouette and mediastinal contours are not enlarged. There is elevated right hemidiaphragm and evidence of right upper lobectomy. Lungs demonstrate no acute findings. There is no effusion or pneumothorax.                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "291  2520        ...   Chest: The cardiomediastinal silhouette is within normal limits for size and contour. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. Thoracic spine: Mild dextro curvature the upper thoracic spine. Evaluation of the upper thoracic bodies is limited secondary to osseous overlap. Vertebral body XXXX and disc spaces are maintained. Mild degenerative endplate changes. Lumbar spine: There are 5 nonrib-bearing lumbar type vertebral bodies. Alignment is within normal limits. Vertebral body XXXX and disc spaces are maintained. Mild degenerative change without acute displaced fracture or dislocation. Moderate amount of stool..\n",
              "296  2539        ...   The lungs are clear. No suspicious pulmonary mass or nodule is identified. There is no pleural effusion or pneumothorax. Heart size and mediastinal contour are normal. There are sclerotic lesions within the XXXX, better visualized on the comparison XXXX scan. There are several bilateral rib fractures with evidence of the callus formation. The appearance is similar to the prior chest radiograph.                                                                                                                                                                                                                                                                                                   \n",
              "297  2509        ...   Chest. Heart size within normal limits. No focal airspace disease. No pneumothorax or effusions. Mild degenerative changes throughout the thoracic spine. Right knee. There has been prior ligamentous repair. There is tricompartmental joint space narrowing and marginal osteophyte formation which is severe in the medial compartment. No knee joint effusion. No fractures or dislocations.                                                                                                                                                                                                                                                                                                               \n",
              "305  47          ...   Frontal and lateral views of the chest show an unchanged cardiomediastinal silhouette. Reduced lung volumes with basilar atelectasis. No XXXX focal airspace consolidation or pleural effusion.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "309  72          ...   Stable cardiomediastinal silhouette with mild cardiomegaly and aortic ectasia and tortuosity. No alveolar consolidation, no findings of pleural effusion. Chronic appearing bilateral rib contour deformities compatible with old fractures. No pneumothorax.                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "313  2174        ...   Lucency crosses the 10th left posterior rib. Visualized portions of the thoracic spine are unremarkable. Mediastinal contours are normal. Lungs are clear. There is no pneumothorax or large pleural effusion.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "337  1410        ...   The heart is normal in size. The mediastinum is unremarkable. The lungs are mildly hyperlucent but clear. There is denser lumbar scoliosis.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "357  2164        ...   The heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is mild streaky perihilar opacity without confluent airspace opacity to suggest a bacterial pneumonia.                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "382  527         ...   The heart is normal size. The mediastinum is unremarkable. A tortuous, calcified thoracic aorta is present. The lungs are hyperexpanded, consistent with emphysema. There is no pleural effusion, pneumothorax, or focal airspace disease. The XXXX are unremarkable.                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "410  2302        ...   Heart size within normal limits. No focal airspace opacities. No pneumothorax. No effusions. Mild degenerative changes of the thoracic spine. No XXXX deformities. Emphysematous changes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
              "421  2554        ...   The cardiomediastinal silhouette is normal in size and contour. No focal consolidation, pneumothorax or large pleural effusion. Interval placement of right humeral prosthesis, incompletely evaluated. Incompletely evaluated the lumbar spine fusion XXXX. XXXX cholecystectomy.                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "422  1397        ...   Heart size is upper limits of normal but stable. Mediastinal contours are within normal limits.. Chronically increased interstitial markings without focal airspace consolidation, pleural effusion, pneumothorax. Degenerative changes of the spine.                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "434  2611        ...   The cardiomediastinal silhouette and pulmonary vasculature are within normal limits in size. Lungs are clear of focal airspace disease. There is no pneumothorax. There is mild blunting of the right costophrenic XXXX, without definite pleural effusion.                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "437  2579        ...   The cardiomediastinal silhouette is within normal limits for appearance. No focal air is prominent consolidation. No pneumothorax. No large pleural effusion. Minimal degenerative changes of the thoracic spine with a mild wedge XXXX deformity of a midthoracic vertebral body. This is age-indeterminate.                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "440  2117        ...   The heart size and pulmonary vascularity appear within normal limits. Left XXXX-a-XXXX is in XXXX. No pleural effusion or pneumothorax is seen. Right upper lobe area of dense opacity is seen in the medial right apex. On a previous outside XXXX scan (XXXX), the right upper lobe was consolidated. Comparison to the XXXX XXXX from that exam shows this opacity to have decreased. No films were available, however, for direct comparison.                                                                                                                                                                                                                                                               \n",
              "442  1152        ...   The lungs are clear. There is hyperexpansion of the lungs suggesting underlying emphysema. The heart and pulmonary XXXX appear normal. Pleural spaces are clear. Mediastinal contours are normal.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "443  2629        ...   The heart is normal in size. The mediastinum is unremarkable. Mild emphysematous changes without focal consolidation. There is no pleural effusion. XXXX lingular scarring or atelectasis noted.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "458  620         ...   The cardiac silhouette mediastinal contours are within normal limits. There are low lung volumes. There is no focal consolidation. There is pneumothorax. There is no large pleural effusion. There is prominent right anterior first rib end.                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "485  1194        ...   Heart size is mildly enlarged but stable.. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. Osseous structures are within normal limits for patient age..                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "487  1609        ...   Heart size is normal. No focal airspace disease. Emphysema. Bibasilar pleural scarring. No pneumothorax or effusion.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
              "488  2806        ...   A right-sided chest XXXX remains in XXXX with the distal tip at the level of the mid SVC. The cardiomediastinal silhouette is within normal limits for appearance. No focal areas of pulmonary consolidation. No pulmonary nodules or mass lesions identified. No pneumothorax or pleural effusion. Mild degenerative changes of the thoracic spine.                                                                                                                                                                                                                                                                                                                                                            \n",
              "493  442         ...   Stable cardiomediastinal silhouette with tortuous thoracic aorta. No pneumothorax, pleural effusion or suspicious focal air space opacity. Stable right lung base scarring.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "498  2136        ...   Normal heart size and mediastinal contours. The lungs are free of any focal airspace disease. In the left lung base, there is a 9 mm nodule that not definitively calcified. No pneumothorax or pleural effusion. No acute bony abnormalities.                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "505  2631        ...   Left-sided medication injection XXXX has its tip projecting at the cavoatrial junction. The trachea is midline. Extensive bilateral bronchiectasis, cystic changes, and scarring represents sequela from the patient's cystic fibrosis. No evidence of focal pulmonary infiltrate or pleural effusion. No large pneumothorax has developed in the interim. The overlying bony structures reveal no acute abnormalities. The heart size is normal.                                                                                                                                                                                                                                                               \n",
              "520  586         ...   The heart size and mediastinal contours appear within normal limits. Low lung volumes on the AP view with bronchovascular crowding and bibasilar atelectasis. No focal airspace consolidation, pleural effusions or pneumothorax. No acute bony abnormalities.                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "526  2193        ...   The heart size and pulmonary vascularity appear within normal limits. The descending thoracic aorta is tortuous. Central venous catheter is again noted. The lungs are free of focal airspace disease. The left hemidiaphragm remains elevated. No pneumothorax or pleural effusion is seen.                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "530  1136        ...   Chest. Lung volumes are low, but no focal infiltrates are present. Heart and mediastinum remain normal. Abdomen. Multiple slightly distended loops are present from stomach to rectum. Formed stool is present in the rectum.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "531  2829        ...   The lungs are clear bilaterally. Specifically, no evidence of focal consolidation, pneumothorax, or pleural effusion.. Cardio mediastinal silhouette is unremarkable. Visualized osseous structures of the thorax are without acute abnormality. Left-sided PICC line has been placed in the interval with tip XXXX in the innominate vein.                                                                                                                                                                                                                                                                                                                                                                     \n",
              "533  2209        ...   XXXX sternotomy XXXX and mediastinal surgical clips remain in XXXX. The cardiomediastinal silhouette is stable in appearance. The thoracic aorta is tortuous and calcified with stable appearance since XXXX exam. No focal areas of pulmonary consolidation. Scattered right basilar subsegmental atelectasis. The left lung appears clear. No pneumothorax or pleural effusion present. Moderate degenerative changes of the thoracic spine. Osteopenia. Mild loss of XXXX of a mid thoracic vertebral body.                                                                                                                                                                                                  \n",
              "542  2340        ...   Chronic bilateral emphysematous changes. The heart size and mediastinal silhouette are within normal limits for contour. The lungs are clear. No pneumothorax or pleural effusions. The XXXX are intact.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
              "548  2484        ...   Heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is right basilar air space opacity.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \n",
              "553  954         ...   Rotated apical lordotic frontal projection, mild bronchovascular crowding and scattered chronic appearing irregular interstitial markings. No definite focal alveolar consolidation or pleural effusion seen. Accounting for technical factors heart size XXXX within normal limits, heavily calcified and mildly tortuous aorta. No typical findings of pulmonary edema.                                                                                                                                                                                                                                                                                                                                       \n",
              "564  2093        ...   The heart is normal in size. The mediastinum is unremarkable. Small nodular opacity left upper lobe may represent early infiltrate. The lungs are otherwise clear. There is no pleural effusion.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "567  2341        ...   The trachea is midline. The cardiomediastinal silhouette is normal in size and unchanged from prior examinations with sternotomy XXXX and surgical clips overlying. There is blunting of the right costophrenic XXXX which appears unchanged from prior examination and may be secondary to scarring or pleural thickening of the right lung base. There is no evidence of acute infiltrate. There is no pneumothorax. Visualized bony structures reveal no acute abnormalities.                                                                                                                                                                                                                                \n",
              "\n",
              "[70 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aYrKhqptfC2C"
      },
      "source": [
        "## Run major model\n",
        "\n",
        "Loads the pretrained weights and evaluates model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b2UtCbokp8oh",
        "outputId": "6ea48efa-9c92-4177-8b5a-6b2b650eaef8"
      },
      "source": [
        "major_checkpoint = os.path.join(TEXT_ONLY_DIR, models[1])\n",
        "transformer_model.load_state_dict(torch.load(major_checkpoint))\n",
        "transformer_model.to(device)\n",
        "major_result = evaluate(transformer_model, tokenizer, dataloaders[1])\n",
        "major_idxs = list(major_result.keys())"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Batch Evaluating: 100%|██████████| 18/18 [00:05<00:00,  3.48it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQC_W-T0fUgq"
      },
      "source": [
        "## False predictions\n",
        "\n",
        "- Key: index in dataset\n",
        "- Value: tuple (prediction, target)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P1FxXk2_qf2e",
        "outputId": "969d7a74-90dd-4cb1-b387-1057a7357358"
      },
      "source": [
        "major_result"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{48: (0, 1),\n",
              " 63: (1, 0),\n",
              " 135: (1, 0),\n",
              " 166: (1, 0),\n",
              " 185: (0, 1),\n",
              " 277: (0, 1),\n",
              " 282: (0, 1),\n",
              " 291: (1, 0),\n",
              " 306: (1, 0),\n",
              " 340: (1, 0),\n",
              " 403: (0, 1),\n",
              " 424: (1, 0),\n",
              " 431: (1, 0),\n",
              " 448: (1, 0),\n",
              " 473: (0, 1),\n",
              " 479: (0, 1),\n",
              " 547: (1, 0)}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bl3g6OH_qhSd",
        "outputId": "d0b7c692-5860-450d-c8ef-c04236c86730"
      },
      "source": [
        "len(major_idxs)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jt6FUEOofc8V"
      },
      "source": [
        "## Visualize false predictions from the dataset\n",
        "\n",
        "label = target label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "QyoG1sgkqihT",
        "outputId": "1063fc3c-01aa-46a5-a220-64f732201deb"
      },
      "source": [
        "major_false_preds = [major.iloc[idx] for idx in major_idxs]\n",
        "major_false_preds = pd.DataFrame(major_false_preds)\n",
        "major_false_preds"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>img</th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>1909</td>\n",
              "      <td>CXR3898_IM-1978-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The cardiomediastinal silhouette is normal in size and contour. No focal consolidation, pneumothorax or large pleural effusion. Normal XXXX.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>1417</td>\n",
              "      <td>CXR2942_IM-1343-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Postsurgical changes of XXXX sternotomy with screw fixation of anterior XXXX plates. Heart size and cardiomediastinal silhouette are normal. No focal consolidation, suspicious bony opacity, pneumothorax, or pleural effusion. No acute osseous abnormality.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>135</th>\n",
              "      <td>107</td>\n",
              "      <td>CXR226_IM-0851-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The cardiac contours are normal. The lungs are clear. Thoracic spondylosis.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>166</th>\n",
              "      <td>1135</td>\n",
              "      <td>CXR2339_IM-0905-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart size and pulmonary vascularity appear within normal limits. The lungs are free of focal airspace disease. No pleural effusion or pneumothorax is seen. Multiple XXXX-filled loops of bowel are present. Gastrostomy is noted.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>185</th>\n",
              "      <td>2508</td>\n",
              "      <td>CXR2448_IM-0983-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>277</th>\n",
              "      <td>1843</td>\n",
              "      <td>CXR3751_IM-1875-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The lungs are clear without evidence of focal airspace disease. There is no evidence of pneumothorax or large pleural effusion. The cardiac and mediastinal contours are within normal limits. The XXXX are unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>282</th>\n",
              "      <td>683</td>\n",
              "      <td>CXR1392_IM-0251-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The aortic XXXX, cardiac apex, and stomach are left-sided. Cardiomediastinal silhouette is within normal limits in overall size and appearance. Pulmonary vascular markings are symmetric and within normal limits. The lungs are normally inflated with no focal airspace disease, pleural effusion, or pneumothorax. No acute bony abnormality.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>291</th>\n",
              "      <td>2520</td>\n",
              "      <td>CXR2487_IM-1014-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Chest: The cardiomediastinal silhouette is within normal limits for size and contour. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. Thoracic spine: Mild dextro curvature the upper thoracic spine. Evaluation of the upper thoracic bodies is limited secondary to osseous overlap. Vertebral body XXXX and disc spaces are maintained. Mild degenerative endplate changes. Lumbar spine: There are 5 nonrib-bearing lumbar type vertebral bodies. Alignment is within normal limits. Vertebral body XXXX and disc spaces are maintained. Mild degenerative change without acute displaced fracture or dislocation. Moderate amount of stool..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>306</th>\n",
              "      <td>490</td>\n",
              "      <td>CXR965_IM-2455-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The lungs appear clear. Heart and pulmonary XXXX appear normal. Mediastinal contours are normal. Pleural spaces are clear. There appears to the contrast XXXX within small colonic diverticula in the splenic flexure region.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>340</th>\n",
              "      <td>546</td>\n",
              "      <td>CXR1091_IM-0062-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart and lungs have XXXX XXXX in the interval. Both lungs are clear and expanded. Heart and mediastinum normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>403</th>\n",
              "      <td>1945</td>\n",
              "      <td>CXR3975_IM-2035-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>No focal consolidation. No visualized pneumothorax. No large pleural effusions. Heart size normal. Cardiomediastinal silhouette is unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>424</th>\n",
              "      <td>521</td>\n",
              "      <td>CXR1028_IM-0022-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Cardiac and mediastinal contours are within normal limits. The lungs are clear. Acromioclavicular arthritis is present, XXXX severe.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>431</th>\n",
              "      <td>137</td>\n",
              "      <td>CXR284_IM-1252-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The cardiomediastinal silhouette is normal in size and contour. No focal consolidation, pneumothorax or large pleural effusion. Negative for acute displaced rib fracture. No discrete XXXX projectile visualized. Contrast within the bilateral renal collecting systems. Contrast also probably within the left colon.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>448</th>\n",
              "      <td>171</td>\n",
              "      <td>CXR358_IM-1759-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Lungs are clear bilaterally. Cardiac and mediastinal silhouettes are normal. Pulmonary vasculature is normal. No pneumothorax or pleural effusion. Limbus vertebra noted within the partial visualized lumbar vertebral body.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>473</th>\n",
              "      <td>1171</td>\n",
              "      <td>CXR2406_IM-0954-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Both lungs remain clear and expanded. Heart and pulmonary XXXX are normal. No change in the large hiatus hernia.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>479</th>\n",
              "      <td>547</td>\n",
              "      <td>CXR1092_IM-0063-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Heart XXXX, mediastinum, XXXX, bony structures and lung XXXX are unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>547</th>\n",
              "      <td>280</td>\n",
              "      <td>CXR563_IM-2164-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The XXXX examination consists of frontal and lateral radiographs of the chest. Cardiac silhouette is at top limits of normal. Aortic and mediastinal contours are within normal limits. Pulmonary vascularity is within normal limits. No focal consolidation, pleural effusion, or pneumothorax identified. Partially visualized ORIF XXXX involving the left proximal humerus. Deformity of the left distal clavicle compatible with remote XXXX. No displaced rib fractures on this chest examination.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               text\n",
              "48   1909        ...   The cardiomediastinal silhouette is normal in size and contour. No focal consolidation, pneumothorax or large pleural effusion. Normal XXXX.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "63   1417        ...   Postsurgical changes of XXXX sternotomy with screw fixation of anterior XXXX plates. Heart size and cardiomediastinal silhouette are normal. No focal consolidation, suspicious bony opacity, pneumothorax, or pleural effusion. No acute osseous abnormality.                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "135  107         ...   The cardiac contours are normal. The lungs are clear. Thoracic spondylosis.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "166  1135        ...   The heart size and pulmonary vascularity appear within normal limits. The lungs are free of focal airspace disease. No pleural effusion or pneumothorax is seen. Multiple XXXX-filled loops of bowel are present. Gastrostomy is noted.                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "185  2508        ...   Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "277  1843        ...   The lungs are clear without evidence of focal airspace disease. There is no evidence of pneumothorax or large pleural effusion. The cardiac and mediastinal contours are within normal limits. The XXXX are unremarkable.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
              "282  683         ...   The aortic XXXX, cardiac apex, and stomach are left-sided. Cardiomediastinal silhouette is within normal limits in overall size and appearance. Pulmonary vascular markings are symmetric and within normal limits. The lungs are normally inflated with no focal airspace disease, pleural effusion, or pneumothorax. No acute bony abnormality.                                                                                                                                                                                                                                                                                                                                                               \n",
              "291  2520        ...   Chest: The cardiomediastinal silhouette is within normal limits for size and contour. The lungs are normally inflated without evidence of focal airspace disease, pleural effusion, or pneumothorax. Thoracic spine: Mild dextro curvature the upper thoracic spine. Evaluation of the upper thoracic bodies is limited secondary to osseous overlap. Vertebral body XXXX and disc spaces are maintained. Mild degenerative endplate changes. Lumbar spine: There are 5 nonrib-bearing lumbar type vertebral bodies. Alignment is within normal limits. Vertebral body XXXX and disc spaces are maintained. Mild degenerative change without acute displaced fracture or dislocation. Moderate amount of stool..\n",
              "306  490         ...   The lungs appear clear. Heart and pulmonary XXXX appear normal. Mediastinal contours are normal. Pleural spaces are clear. There appears to the contrast XXXX within small colonic diverticula in the splenic flexure region.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "340  546         ...   The heart and lungs have XXXX XXXX in the interval. Both lungs are clear and expanded. Heart and mediastinum normal.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
              "403  1945        ...   No focal consolidation. No visualized pneumothorax. No large pleural effusions. Heart size normal. Cardiomediastinal silhouette is unremarkable.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "424  521         ...   Cardiac and mediastinal contours are within normal limits. The lungs are clear. Acromioclavicular arthritis is present, XXXX severe.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
              "431  137         ...   The cardiomediastinal silhouette is normal in size and contour. No focal consolidation, pneumothorax or large pleural effusion. Negative for acute displaced rib fracture. No discrete XXXX projectile visualized. Contrast within the bilateral renal collecting systems. Contrast also probably within the left colon.                                                                                                                                                                                                                                                                                                                                                                                        \n",
              "448  171         ...   Lungs are clear bilaterally. Cardiac and mediastinal silhouettes are normal. Pulmonary vasculature is normal. No pneumothorax or pleural effusion. Limbus vertebra noted within the partial visualized lumbar vertebral body.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "473  1171        ...   Both lungs remain clear and expanded. Heart and pulmonary XXXX are normal. No change in the large hiatus hernia.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "479  547         ...   Heart XXXX, mediastinum, XXXX, bony structures and lung XXXX are unremarkable.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "547  280         ...   The XXXX examination consists of frontal and lateral radiographs of the chest. Cardiac silhouette is at top limits of normal. Aortic and mediastinal contours are within normal limits. Pulmonary vascularity is within normal limits. No focal consolidation, pleural effusion, or pneumothorax identified. Partially visualized ORIF XXXX involving the left proximal humerus. Deformity of the left distal clavicle compatible with remote XXXX. No displaced rib fractures on this chest examination.                                                                                                                                                                                                       \n",
              "\n",
              "[17 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SZl3tsA8-qfx"
      },
      "source": [
        "## MMBT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "msGBmxeLBrHH"
      },
      "source": [
        "## Configure Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lpzRSM5T-p15"
      },
      "source": [
        "import json\n",
        "import os\n",
        "from collections import Counter\n",
        "import numpy as np\n",
        "from matplotlib.pyplot import imshow\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from PIL import Image\n",
        "from torch.utils.data import Dataset"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XFjnCYb3BpX4"
      },
      "source": [
        "from transformers import (\n",
        "    WEIGHTS_NAME,\n",
        "    AdamW,\n",
        "    AutoConfig,\n",
        "    AutoModel,\n",
        "    AutoTokenizer,\n",
        "    get_linear_schedule_with_warmup,\n",
        ")"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Zg7V3dwg1Xf"
      },
      "source": [
        "## Files and Paths"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "khdbQ-aVB1rW"
      },
      "source": [
        "img_data_dir = os.path.join(DATA_DIR, \"NLMCXR_png_frontal\")\n",
        "jsonl_data_dir = os.path.join(DATA_DIR, \"json\")\n",
        "saved_chexnet = \"saved_chexnet.pt\"\n",
        "major_file = \"image_labels_major_findings_frontal_test.jsonl\"\n",
        "baseline_file = \"image_labels_findings_frontal_test.jsonl\""
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gd1VXKz3g_wS"
      },
      "source": [
        "## Define Image Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9yum2zgoB5o7"
      },
      "source": [
        "# mapping number of image embeddings to AdaptiveAvgPool2d output size\n",
        "POOLING_BREAKDOWN = {1: (1, 1), 2: (2, 1), 3: (3, 1), 4: (2, 2), 5: (5, 1), 6: (3, 2), 7: (7, 1), 8: (4, 2), 9: (3, 3)}\n",
        "\n",
        "class ImageEncoderDenseNet(nn.Module):\n",
        "    def __init__(self, num_image_embeds, saved_model=True, path=os.path.join(MODELS_DIR, saved_chexnet)):\n",
        "        super().__init__()\n",
        "        if saved_model:\n",
        "            print(f\"Loading a saved model from {path}\")\n",
        "            model = torch.load(path)\n",
        "        else:\n",
        "            print(\"No saved model found. Loading a pretrained Densenet121 model from Pytorch repositories.\")\n",
        "            model = torchvision.models.densenet121(pretrained=True)\n",
        "\n",
        "        # DenseNet architecture last layer is the classifier; we only want everything before that\n",
        "        modules = list(model.children())[:-1] \n",
        "        self.model = nn.Sequential(*modules)\n",
        "        # self.model same as original DenseNet self.features part of the forward function\n",
        "        self.pool = nn.AdaptiveAvgPool2d(POOLING_BREAKDOWN[num_image_embeds])\n",
        "\n",
        "    def forward(self, input_modal):\n",
        "        # Bx3x224x224 -> Bx1024x7x7 -> Bx1024xN -> BxNx1024\n",
        "        features = self.model(input_modal)\n",
        "        out = F.relu(features, inplace=True)\n",
        "        out = self.pool(out)\n",
        "        out = torch.flatten(out, start_dim=2)\n",
        "        out = out.transpose(1, 2).contiguous()\n",
        "\n",
        "        return out  # BxNx1024"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xU6s6YW5hDQF"
      },
      "source": [
        "## Define custom JsonDataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YvF5N8Z2B6qL"
      },
      "source": [
        "class JsonlDataset(Dataset):\n",
        "    def __init__(self, jsonl_data_path, img_dir, tokenizer, transforms, labels, max_seq_length):\n",
        "        self.data = [json.loads(l) for l in open(jsonl_data_path)]\n",
        "        # self.data_dir = os.path.dirname(data_path)\n",
        "        self.img_data_dir = img_dir\n",
        "        self.tokenizer = tokenizer\n",
        "        self.labels = labels\n",
        "        self.n_classes = len(labels)\n",
        "        self.max_seq_length = max_seq_length\n",
        "\n",
        "        # for image normalization for DenseNet\n",
        "        self.transforms = transforms\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        sentence = torch.LongTensor(self.tokenizer.encode(self.data[index][\"text\"], add_special_tokens=True))\n",
        "        start_token, sentence, end_token = sentence[0], sentence[1:-1], sentence[-1]\n",
        "        sentence = sentence[:self.max_seq_length]\n",
        "        label = torch.LongTensor([self.labels.index(self.data[index][\"label\"])])\n",
        "\n",
        "        image = Image.open(os.path.join(self.img_data_dir, self.data[index][\"img\"])).convert(\"RGB\")\n",
        "        image = self.transforms(image)\n",
        "\n",
        "        return {\n",
        "            \"image_start_token\": start_token,\n",
        "            \"image_end_token\": end_token,\n",
        "            \"sentence\": sentence,\n",
        "            \"image\": image,\n",
        "            \"label\": label,\n",
        "        }\n",
        "\n",
        "\n",
        "def collate_fn(batch):\n",
        "    \"\"\"\n",
        "    Specify batching for the torch Dataloader function\n",
        "\n",
        "    :param batch: each batch of the JsonlDataset\n",
        "    :return: text tensor, attention mask tensor, img tensor, modal start token, modal end token, label\n",
        "    \"\"\"\n",
        "    lens = [len(row[\"sentence\"]) for row in batch]\n",
        "    bsz, max_seq_len = len(batch), max(lens)\n",
        "\n",
        "    mask_tensor = torch.zeros(bsz, max_seq_len, dtype=torch.long)\n",
        "    text_tensor = torch.zeros(bsz, max_seq_len, dtype=torch.long)\n",
        "\n",
        "    for i_batch, (input_row, length) in enumerate(zip(batch, lens)):\n",
        "        text_tensor[i_batch, :length] = input_row[\"sentence\"]\n",
        "        mask_tensor[i_batch, :length] = 1\n",
        "\n",
        "    img_tensor = torch.stack([row[\"image\"] for row in batch])\n",
        "    tgt_tensor = torch.stack([row[\"label\"] for row in batch])\n",
        "    img_start_token = torch.stack([row[\"image_start_token\"] for row in batch])\n",
        "    img_end_token = torch.stack([row[\"image_end_token\"] for row in batch])\n",
        "\n",
        "    return text_tensor, mask_tensor, img_tensor, img_start_token, img_end_token, tgt_tensor"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BDh9fFFjhMxi"
      },
      "source": [
        "## Define other helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e2UOehS_B93V"
      },
      "source": [
        "def get_labels():\n",
        "    \"\"\"\n",
        "    0: normal\n",
        "    1: abnormal\n",
        "\n",
        "    :return: label classes\n",
        "    \"\"\"\n",
        "\n",
        "    return [0, 1]\n",
        "\n",
        "# mean and std specific for DenseNet architecture\n",
        "def get_image_transforms():\n",
        "    return transforms.Compose(\n",
        "        [\n",
        "            transforms.Resize(256),\n",
        "            transforms.CenterCrop(224),\n",
        "            transforms.ToTensor(),\n",
        "            transforms.Normalize(\n",
        "                mean=[0.485, 0.456, 0.406], \n",
        "                std=[0.229, 0.224, 0.225]\n",
        "            )\n",
        "        ]\n",
        "    )\n",
        "\n",
        "def load_examples(tokenizer, max_seq_len, num_image_embeds, test_file, evaluate=False, test=False, data_dir=jsonl_data_dir, img_dir=img_data_dir):\n",
        "    if evaluate and not test:\n",
        "        path = os.path.join(data_dir, val_file)\n",
        "    elif evaluate and test:\n",
        "        path = os.path.join(data_dir, test_file)\n",
        "    elif not evaluate and not test:\n",
        "        path = os.path.join(data_dir, train_file)\n",
        "    else:\n",
        "        # shouldn't get here not evaluate and test?\n",
        "        raise ValueError(\"invalid data file option!!\")\n",
        "\n",
        "    img_transforms = get_image_transforms()\n",
        "    labels = get_labels()\n",
        "    dataset = JsonlDataset(path, img_dir, tokenizer, img_transforms, labels, max_seq_len - num_image_embeds - 2)\n",
        "    return dataset"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "76WD6ayChR-0"
      },
      "source": [
        "## Create Datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1LlwYJDLCBS2"
      },
      "source": [
        "img_transforms = get_image_transforms()\n",
        "labels = get_labels()\n",
        "num_labels = len(labels)\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\", do_lower_case=True, cache_dir=None)\n",
        "\n",
        "major_dataset = load_examples(tokenizer, max_seq_len=256, num_image_embeds=3, test_file=major_file, evaluate=True, test=True)\n",
        "base_dataset = load_examples(tokenizer, max_seq_len=256, num_image_embeds=3, test_file=baseline_file, evaluate=True, test=True)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xZ2DYbxGCXbf"
      },
      "source": [
        "## Modifying ModalEmbeddings, MMBTModel and MMBTForClassification classes\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qe7jlDMzCJP4"
      },
      "source": [
        "from torch.nn import CrossEntropyLoss, MSELoss\n",
        "from transformers.modeling_outputs import BaseModelOutputWithPooling, SequenceClassifierOutput\n",
        "from transformers.modeling_utils import ModuleUtilsMixin\n",
        "from transformers.models.bert.modeling_bert import BertPreTrainedModel\n",
        "from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings\n",
        "from transformers import MMBTConfig"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lwUcuzyjCfqb"
      },
      "source": [
        "## ModalEmbeddings Modification \n",
        "\n",
        "The ModalEmbeddings class needs **hidden_modal_size**, which is part of the MMBTConfig and also MMBTConfig2. However, originally this attribute gets passed as part of the 'config' argument, which now has only the BertConfig and not MMBTConfig attributes.\n",
        "\n",
        "So the modiciations in ModalEmbeddings2 has both BertConfig (self.config) and MMBTConfig attributes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TchluMhVCfDo"
      },
      "source": [
        "_CONFIG_FOR_DOC = \"MMBTConfig\"\n",
        "\n",
        "\n",
        "class ModalEmbeddings2(nn.Module):\n",
        "    \"\"\"Generic Modal Embeddings which takes in an encoder, and a transformer embedding.\"\"\"\n",
        "\n",
        "    def __init__(self, config, modal_hidden_size, encoder, embeddings):\n",
        "        super().__init__()\n",
        "        self.config = config\n",
        "        self.encoder = encoder\n",
        "        self.proj_embeddings = nn.Linear(modal_hidden_size, config.hidden_size)\n",
        "        self.position_embeddings = embeddings.position_embeddings\n",
        "        self.token_type_embeddings = embeddings.token_type_embeddings\n",
        "        self.word_embeddings = embeddings.word_embeddings\n",
        "        self.LayerNorm = embeddings.LayerNorm\n",
        "        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)\n",
        "\n",
        "    def forward(self, input_modal, start_token=None, end_token=None, position_ids=None, token_type_ids=None):\n",
        "        token_embeddings = self.proj_embeddings(self.encoder(input_modal))\n",
        "        seq_length = token_embeddings.size(1)\n",
        "\n",
        "        if start_token is not None:\n",
        "            start_token_embeds = self.word_embeddings(start_token)\n",
        "            seq_length += 1\n",
        "            token_embeddings = torch.cat([start_token_embeds.unsqueeze(1), token_embeddings], dim=1)\n",
        "\n",
        "        if end_token is not None:\n",
        "            end_token_embeds = self.word_embeddings(end_token)\n",
        "            seq_length += 1\n",
        "            token_embeddings = torch.cat([token_embeddings, end_token_embeds.unsqueeze(1)], dim=1)\n",
        "\n",
        "        if position_ids is None:\n",
        "            position_ids = torch.arange(seq_length, dtype=torch.long, device=input_modal.device)\n",
        "            position_ids = position_ids.unsqueeze(0).expand(input_modal.size(0), seq_length)\n",
        "\n",
        "        if token_type_ids is None:\n",
        "            token_type_ids = torch.zeros(\n",
        "                (input_modal.size(0), seq_length), dtype=torch.long, device=input_modal.device\n",
        "            )\n",
        "\n",
        "        position_embeddings = self.position_embeddings(position_ids)\n",
        "        token_type_embeddings = self.token_type_embeddings(token_type_ids)\n",
        "        embeddings = token_embeddings + position_embeddings + token_type_embeddings\n",
        "        embeddings = self.LayerNorm(embeddings)\n",
        "        embeddings = self.dropout(embeddings)\n",
        "        return embeddings\n",
        "\n",
        "\n",
        "MMBT_START_DOCSTRING = r\"\"\"\n",
        "    MMBT model was proposed in `Supervised Multimodal Bitransformers for Classifying Images and Text\n",
        "    <https://github.com/facebookresearch/mmbt>`__ by Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.\n",
        "    It's a supervised multimodal bitransformer model that fuses information from text and other image encoders, and\n",
        "    obtain state-of-the-art performance on various multimodal classification benchmark tasks.\n",
        "    This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic\n",
        "    methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,\n",
        "    pruning heads etc.)\n",
        "    This model is also a PyTorch `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__\n",
        "    subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to\n",
        "    general usage and behavior.\n",
        "    Parameters:\n",
        "        config (:class:`~transformers.MMBTConfig`): Model configuration class with all the parameters of the model.\n",
        "            Initializing with a config file does not load the weights associated with the model, only the\n",
        "            configuration.\n",
        "        transformer (:class: `~nn.Module`): A text transformer that is used by MMBT.\n",
        "            It should have embeddings, encoder, and pooler attributes.\n",
        "        encoder (:class: `~nn.Module`): Encoder for the second modality.\n",
        "            It should take in a batch of modal inputs and return k, n dimension embeddings.\n",
        "\"\"\"\n",
        "\n",
        "MMBT_INPUTS_DOCSTRING = r\"\"\"\n",
        "    Args:\n",
        "        input_modal (``torch.FloatTensor`` of shape ``(batch_size, ***)``):\n",
        "            The other modality data. It will be the shape that the encoder for that type expects. e.g. With an Image\n",
        "            Encoder, the shape would be (batch_size, channels, height, width)\n",
        "        input_ids (``torch.LongTensor`` of shape ``(batch_size, sequence_length)``):\n",
        "            Indices of input sequence tokens in the vocabulary. It does not expect [CLS] token to be added as it's\n",
        "            appended to the end of other modality embeddings. Indices can be obtained using\n",
        "            :class:`~transformers.BertTokenizer`. See :meth:`transformers.PreTrainedTokenizer.encode` and\n",
        "            :meth:`transformers.PreTrainedTokenizer.__call__` for details.\n",
        "            `What are input IDs? <../glossary.html#input-ids>`__\n",
        "        modal_start_tokens (``torch.LongTensor`` of shape ``(batch_size,)``, `optional`):\n",
        "            Optional start token to be added to Other Modality Embedding. [CLS] Most commonly used for classification\n",
        "            tasks.\n",
        "        modal_end_tokens (``torch.LongTensor`` of shape ``(batch_size,)``, `optional`):\n",
        "            Optional end token to be added to Other Modality Embedding. [SEP] Most commonly used.\n",
        "        attention_mask (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:\n",
        "            Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``:\n",
        "            - 1 for tokens that are **not masked**,\n",
        "            - 0 for tokens that are **masked**.\n",
        "            `What are attention masks? <../glossary.html#attention-mask>`__\n",
        "        token_type_ids (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:\n",
        "            Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0,\n",
        "            1]``:\n",
        "            - 0 corresponds to a `sentence A` token,\n",
        "            - 1 corresponds to a `sentence B` token.\n",
        "            `What are token type IDs? <../glossary.html#token-type-ids>`_\n",
        "        modal_token_type_ids (`optional`) ``torch.LongTensor`` of shape ``(batch_size, modal_sequence_length)``:\n",
        "            Segment token indices to indicate different portions of the non-text modality. The embeddings from these\n",
        "            tokens will be summed with the respective token embeddings for the non-text modality.\n",
        "        position_ids (``torch.LongTensor`` of shape ``(batch_size, sequence_length)``, `optional`):\n",
        "            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0,\n",
        "            config.max_position_embeddings - 1]``.\n",
        "            `What are position IDs? <../glossary.html#position-ids>`__\n",
        "        modal_position_ids (``torch.LongTensor`` of shape ``(batch_size, modal_sequence_length)``, `optional`):\n",
        "            Indices of positions of each input sequence tokens in the position embeddings for the non-text modality.\n",
        "            Selected in the range ``[0, config.max_position_embeddings - 1]``.\n",
        "            `What are position IDs? <../glossary.html#position-ids>`__\n",
        "        head_mask (``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``, `optional`):\n",
        "            Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``:\n",
        "            - 1 indicates the head is **not masked**,\n",
        "            - 0 indicates the head is **masked**.\n",
        "        inputs_embeds (``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``, `optional`):\n",
        "            Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.\n",
        "            This is useful if you want more control over how to convert :obj:`input_ids` indices into associated\n",
        "            vectors than the model's internal embedding lookup matrix.\n",
        "        encoder_hidden_states (``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``, `optional`):\n",
        "            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if\n",
        "            the model is configured as a decoder.\n",
        "        encoder_attention_mask (``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``, `optional`):\n",
        "            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in\n",
        "            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:\n",
        "            - 1 for tokens that are **not masked**,\n",
        "            - 0 for tokens that are **masked**.\n",
        "        output_attentions (:obj:`bool`, `optional`):\n",
        "            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned\n",
        "            tensors for more detail.\n",
        "        output_hidden_states (:obj:`bool`, `optional`):\n",
        "            Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for\n",
        "            more detail.\n",
        "        return_dict (:obj:`bool`, `optional`):\n",
        "            Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.\n",
        "\"\"\""
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kb4Ej3bwCrPd"
      },
      "source": [
        "@add_start_docstrings(\n",
        "    \"The bare MMBT Model outputting raw hidden-states without any specific head on top.\",\n",
        "    MMBT_START_DOCSTRING,\n",
        ")\n",
        "class MMBTModel2(nn.Module, ModuleUtilsMixin):\n",
        "    def __init__(self, config, mmbt_config):\n",
        "        super().__init__()\n",
        "        self.config = config\n",
        "        self.transformer = mmbt_config.transformer\n",
        "        self.modal_encoder = ModalEmbeddings2(config, \n",
        "                                             mmbt_config.modal_hidden_size, \n",
        "                                             mmbt_config.encoder, \n",
        "                                             mmbt_config.transformer.embeddings)\n",
        "\n",
        "    @add_start_docstrings_to_model_forward(MMBT_INPUTS_DOCSTRING)\n",
        "    @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC)\n",
        "    def forward(\n",
        "        self,\n",
        "        input_modal,\n",
        "        input_ids=None,\n",
        "        modal_start_tokens=None,\n",
        "        modal_end_tokens=None,\n",
        "        attention_mask=None,\n",
        "        token_type_ids=None,\n",
        "        modal_token_type_ids=None,\n",
        "        position_ids=None,\n",
        "        modal_position_ids=None,\n",
        "        head_mask=None,\n",
        "        inputs_embeds=None,\n",
        "        encoder_hidden_states=None,\n",
        "        encoder_attention_mask=None,\n",
        "        output_attentions=None,\n",
        "        output_hidden_states=None,\n",
        "        return_dict=None,\n",
        "    ):\n",
        "        r\"\"\"\n",
        "        Returns:\n",
        "        Examples::\n",
        "            # For example purposes. Not runnable.\n",
        "            transformer = BertModel.from_pretrained('bert-base-uncased')\n",
        "            encoder = ImageEncoder(args)\n",
        "            mmbt = MMBTModel(config, transformer, encoder)\n",
        "        \"\"\"\n",
        "        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions\n",
        "        output_hidden_states = (\n",
        "            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states\n",
        "        )\n",
        "        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n",
        "\n",
        "        if input_ids is not None and inputs_embeds is not None:\n",
        "            raise ValueError(\"You cannot specify both input_ids and inputs_embeds at the same time\")\n",
        "        elif input_ids is not None:\n",
        "            input_txt_shape = input_ids.size()\n",
        "        elif inputs_embeds is not None:\n",
        "            input_txt_shape = inputs_embeds.size()[:-1]\n",
        "        else:\n",
        "            raise ValueError(\"You have to specify either input_ids or inputs_embeds\")\n",
        "\n",
        "        device = input_ids.device if input_ids is not None else inputs_embeds.device\n",
        "\n",
        "        modal_embeddings = self.modal_encoder(\n",
        "            input_modal,\n",
        "            start_token=modal_start_tokens,\n",
        "            end_token=modal_end_tokens,\n",
        "            position_ids=modal_position_ids,\n",
        "            token_type_ids=modal_token_type_ids,\n",
        "        )\n",
        "\n",
        "        input_modal_shape = modal_embeddings.size()[:-1]\n",
        "\n",
        "        if token_type_ids is None:\n",
        "            token_type_ids = torch.ones(input_txt_shape, dtype=torch.long, device=device)\n",
        "\n",
        "        txt_embeddings = self.transformer.embeddings(\n",
        "            input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds\n",
        "        )\n",
        "\n",
        "        embedding_output = torch.cat([modal_embeddings, txt_embeddings], 1)\n",
        "\n",
        "        input_shape = embedding_output.size()[:-1]\n",
        "\n",
        "        if attention_mask is None:\n",
        "            attention_mask = torch.ones(input_shape, device=device)\n",
        "        else:\n",
        "            attention_mask = torch.cat(\n",
        "                [torch.ones(input_modal_shape, device=device, dtype=torch.long), attention_mask], dim=1\n",
        "            )\n",
        "        if encoder_attention_mask is None:\n",
        "            encoder_attention_mask = torch.ones(input_shape, device=device)\n",
        "        else:\n",
        "            encoder_attention_mask = torch.cat(\n",
        "                [torch.ones(input_modal_shape, device=device), encoder_attention_mask], dim=1\n",
        "            )\n",
        "\n",
        "        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, self.device)\n",
        "        encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)\n",
        "        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)\n",
        "\n",
        "        encoder_outputs = self.transformer.encoder(\n",
        "            embedding_output,\n",
        "            attention_mask=extended_attention_mask,\n",
        "            head_mask=head_mask,\n",
        "            encoder_hidden_states=encoder_hidden_states,\n",
        "            encoder_attention_mask=encoder_extended_attention_mask,\n",
        "            output_attentions=output_attentions,\n",
        "            output_hidden_states=output_hidden_states,\n",
        "            return_dict=return_dict,\n",
        "        )\n",
        "\n",
        "        sequence_output = encoder_outputs[0]\n",
        "        pooled_output = self.transformer.pooler(sequence_output)\n",
        "\n",
        "        if not return_dict:\n",
        "            return (sequence_output, pooled_output) + encoder_outputs[1:]\n",
        "\n",
        "        return BaseModelOutputWithPooling(\n",
        "            last_hidden_state=sequence_output,\n",
        "            pooler_output=pooled_output,\n",
        "            hidden_states=encoder_outputs.hidden_states,\n",
        "            attentions=encoder_outputs.attentions,\n",
        "        )\n",
        "\n",
        "    def get_input_embeddings(self):\n",
        "        return self.embeddings.word_embeddings\n",
        "\n",
        "    def set_input_embeddings(self, value):\n",
        "        self.embeddings.word_embeddings = value"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7GDw3QgjCuen"
      },
      "source": [
        "class MMBTConfig2(object):\n",
        "    \"\"\"\n",
        "    This is the configuration class to store the configuration of a :class:`~transformers.MMBTModel`. It is used to\n",
        "    instantiate a MMBT model according to the specified arguments, defining the model architecture.\n",
        "    Args:\n",
        "        config (:class:`~transformers.PreTrainedConfig`):\n",
        "            Config of the underlying Transformer models. Its values are copied over to use a single config.\n",
        "        num_labels (:obj:`int`, `optional`):\n",
        "            Size of final Linear layer for classification.\n",
        "        modal_hidden_size (:obj:`int`, `optional`, defaults to 2048):\n",
        "            Embedding dimension of the non-text modality encoder.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, transformer, encoder, num_labels=None, modal_hidden_size=1024):\n",
        "        self.transformer = transformer\n",
        "        self.encoder = encoder\n",
        "        self.modal_hidden_size = modal_hidden_size\n",
        "        if num_labels:\n",
        "            self.num_labels = num_labels"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7su3ipiaCxhc"
      },
      "source": [
        "\n",
        "\n",
        "@add_start_docstrings(\n",
        "    \"\"\"\n",
        "    MMBT Model with a sequence classification/regression head on top (a linear layer on top of the pooled output)\n",
        "    \"\"\",\n",
        "    MMBT_START_DOCSTRING,\n",
        "    MMBT_INPUTS_DOCSTRING,\n",
        ")\n",
        "class MMBTForClassification2(BertPreTrainedModel):\n",
        "    r\"\"\"\n",
        "        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:\n",
        "            Labels for computing the sequence classification/regression loss. Indices should be in ``[0, ...,\n",
        "            config.num_labels - 1]``. If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),\n",
        "            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).\n",
        "    Returns: `Tuple` comprising various elements depending on the configuration (config) and inputs: **loss**:\n",
        "    (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: Classification (or\n",
        "    regression if config.num_labels==1) loss. **logits**: ``torch.FloatTensor`` of shape ``(batch_size,\n",
        "    config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax).\n",
        "    **hidden_states**: (`optional`, returned when ``output_hidden_states=True``) list of ``torch.FloatTensor`` (one for\n",
        "    the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``:\n",
        "    Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**:\n",
        "    (`optional`, returned when ``output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape\n",
        "    ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used\n",
        "    to compute the weighted average in the self-attention heads.\n",
        "    Examples::\n",
        "        # For example purposes. Not runnable.\n",
        "        transformer = BertModel.from_pretrained('bert-base-uncased')\n",
        "        encoder = ImageEncoder(args)\n",
        "        model = MMBTForClassification(config, transformer, encoder)\n",
        "        outputs = model(input_modal, input_ids, labels=labels)\n",
        "        loss, logits = outputs[:2]\n",
        "    \"\"\"\n",
        "    def __init__(self, config, mmbt_config):\n",
        "        super().__init__(config)\n",
        "        self.num_labels = config.num_labels\n",
        "\n",
        "        self.mmbt = MMBTModel2(config, mmbt_config)\n",
        "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
        "        self.classifier = nn.Linear(config.hidden_size, config.num_labels)\n",
        "\n",
        "        self.init_weights()\n",
        "\n",
        "    def forward(\n",
        "        self,\n",
        "        input_modal,\n",
        "        input_ids=None,\n",
        "        modal_start_tokens=None,\n",
        "        modal_end_tokens=None,\n",
        "        attention_mask=None,\n",
        "        token_type_ids=None,\n",
        "        modal_token_type_ids=None,\n",
        "        position_ids=None,\n",
        "        modal_position_ids=None,\n",
        "        head_mask=None,\n",
        "        inputs_embeds=None,\n",
        "        labels=None,\n",
        "        return_dict=None,\n",
        "    ):\n",
        "        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n",
        "\n",
        "        outputs = self.mmbt(\n",
        "            input_modal=input_modal,\n",
        "            input_ids=input_ids,\n",
        "            modal_start_tokens=modal_start_tokens,\n",
        "            modal_end_tokens=modal_end_tokens,\n",
        "            attention_mask=attention_mask,\n",
        "            token_type_ids=token_type_ids,\n",
        "            modal_token_type_ids=modal_token_type_ids,\n",
        "            position_ids=position_ids,\n",
        "            modal_position_ids=modal_position_ids,\n",
        "            head_mask=head_mask,\n",
        "            inputs_embeds=inputs_embeds,\n",
        "            return_dict=return_dict,\n",
        "            output_attentions=True\n",
        "        )\n",
        "\n",
        "        pooled_output = outputs[1]\n",
        "\n",
        "        pooled_output = self.dropout(pooled_output)\n",
        "        logits = self.classifier(pooled_output)\n",
        "\n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "            if self.num_labels == 1:\n",
        "                #  We are doing regression\n",
        "                loss_fct = MSELoss()\n",
        "                loss = loss_fct(logits.view(-1), labels.view(-1))\n",
        "            else:\n",
        "                loss_fct = CrossEntropyLoss()\n",
        "                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n",
        "\n",
        "        if not return_dict:\n",
        "            output = (logits,) + outputs[2:]\n",
        "            return ((loss,) + output) if loss is not None else output\n",
        "\n",
        "        return SequenceClassifierOutput(\n",
        "            loss=loss,\n",
        "            logits=logits,\n",
        "            hidden_states=outputs.hidden_states,\n",
        "            attentions=outputs.attentions,\n",
        "        )"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fNiugA5VhcJl"
      },
      "source": [
        "## Model Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Na3pNQEoC2Wx"
      },
      "source": [
        "model_name = \"bert-base-uncased\"\n",
        "tokenizer_name = \"bert-base-uncased\"\n",
        "max_seq_length = 256\n",
        "num_img_embeddings = 3\n",
        "batch_size = 32\n",
        "\n",
        "config_name = None"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E1tvDhT-hfMZ"
      },
      "source": [
        "## Evaluation function\n",
        "\n",
        "This function is modified from the original eval function used in the mmbt notebook. Instead of returning the eval results, it returns a dictionary containing the indices in the dataset as keys, where the prediction did not match the target. As values for the keys, it returns the prediction and target labels associated to these examples."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qUqObrhtDkl_"
      },
      "source": [
        "def mmbt_evaluate(model, tokenizer, dataloader):\n",
        "    eval_loss = 0.0\n",
        "    nb_eval_steps = 0\n",
        "    preds = []\n",
        "    out_label_ids = []\n",
        "    for batch in tqdm(dataloader, desc=\"Evaluating\"):\n",
        "        model.eval()\n",
        "        #batch = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            batch = tuple(t.to(device) for t in batch)\n",
        "            labels = batch[5]\n",
        "            input_ids = batch[0]\n",
        "            input_modal = batch[2]\n",
        "            attention_mask = batch[1]\n",
        "            modal_start_tokens = batch[3]\n",
        "            modal_end_tokens = batch[4]\n",
        "                   \n",
        "            outputs = model(\n",
        "                input_modal,\n",
        "                input_ids=input_ids,\n",
        "                modal_start_tokens=modal_start_tokens,\n",
        "                modal_end_tokens=modal_end_tokens,\n",
        "                attention_mask=attention_mask,\n",
        "                token_type_ids=None,\n",
        "                modal_token_type_ids=None,\n",
        "                position_ids=None,\n",
        "                modal_position_ids=None,\n",
        "                head_mask=None,\n",
        "                inputs_embeds=None,\n",
        "                labels=labels,\n",
        "                return_dict=True\n",
        "            )\n",
        "           \n",
        "            logits = outputs.logits\n",
        "            tmp_eval_loss = outputs.loss              \n",
        "            eval_loss += tmp_eval_loss.mean().item()\n",
        "        nb_eval_steps += 1\n",
        "        # Move logits and labels to CPU\n",
        "        pred = torch.nn.functional.softmax(logits, dim=1).argmax(dim=1).cpu().detach().numpy()      \n",
        "        out_label_id = labels.detach().cpu().numpy()\n",
        "        preds.append(pred)\n",
        "        out_label_ids.append(out_label_id)\n",
        "\n",
        "    eval_loss = eval_loss / nb_eval_steps\n",
        "\n",
        "    preds = [l for sl in preds for l in sl]\n",
        "    out_label_ids = [l for sl in out_label_ids for l in sl]\n",
        "\n",
        "    idxs = {}\n",
        "    for i, (pred, label) in enumerate(zip(preds, out_label_ids)):\n",
        "      if pred != label:\n",
        "        idxs.update({i: (pred, label)})\n",
        "\n",
        "    return idxs"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nwoa-WFphsS1"
      },
      "source": [
        "## Configure model and create dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LT1OThIrEow5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "346e9cb5-66e9-4bdf-b7c3-24870a73680e"
      },
      "source": [
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm\n",
        "\n",
        "#\n",
        "major_dataloader = DataLoader(\n",
        "            major_dataset,  \n",
        "            sampler = SequentialSampler(major_dataset),\n",
        "            batch_size = batch_size,\n",
        "            collate_fn=collate_fn\n",
        "            )\n",
        "\n",
        "#\n",
        "base_dataloader = DataLoader(\n",
        "            base_dataset,  \n",
        "            sampler = SequentialSampler(base_dataset), \n",
        "            batch_size = batch_size,\n",
        "            collate_fn=collate_fn\n",
        "            )\n",
        "\n",
        "transformer_config = AutoConfig.from_pretrained(config_name if config_name else model_name)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\n",
        "        tokenizer_name if tokenizer_name else model_name,\n",
        "        do_lower_case=True,\n",
        "        cache_dir=None,\n",
        "    )\n",
        "transformer = AutoModel.from_pretrained(model_name, config=transformer_config, cache_dir=None)\n",
        "img_encoder = ImageEncoderDenseNet(num_image_embeds=num_img_embeddings)\n",
        "multimodal_config = MMBTConfig2(transformer, img_encoder, num_labels=num_labels, modal_hidden_size=1024)\n",
        "mmbt_model = MMBTForClassification2(transformer_config, multimodal_config)\n",
        "\n",
        "mmbt_model.to(device)\n",
        "\n",
        "MODELS_DIR = os.path.join(DATA_DIR, 'models')\n",
        "MODELS_MMBT_DIR = os.path.join(MODELS_DIR, 'mmbt')\n",
        "\n",
        "models = [\"mmbt_findings.bin\", \"mmbt_major.bin\"]\n",
        "dataloaders = [base_dataloader, major_dataloader]"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading a saved model from /content/drive/.shortcut-targets-by-id/1gwgx4ZApTKz5fN6SG9YkiVjVCZ0WNGeH/LAP_MMBT/data/models/saved_chexnet.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ob-oSbd5h_kQ"
      },
      "source": [
        "## Run base MMBT model\n",
        "\n",
        "Loads the pretrained weights and evaluates model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_8o7B3v0rWfU",
        "outputId": "826b614c-9faf-46d5-9faa-d2f0ec58202c"
      },
      "source": [
        "mmbt_base_checkpoint = os.path.join(MODELS_MMBT_DIR, models[0])\n",
        "mmbt_model.load_state_dict(torch.load(mmbt_base_checkpoint, map_location=device))\n",
        "\n",
        "mmbt_base_result = mmbt_evaluate(mmbt_model, tokenizer, dataloaders[0])\n",
        "mmbt_base_idxs = list(mmbt_base_result.keys())"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluating: 100%|██████████| 18/18 [03:10<00:00, 10.59s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FLDNVYbhiqqA"
      },
      "source": [
        "## False predictions\n",
        "\n",
        "- Key: index in dataset\n",
        "- Value: tuple (prediction, target)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-KkRJEhrKu_n",
        "outputId": "5eddacb5-cc26-4974-e134-fc24ff760c2c"
      },
      "source": [
        "mmbt_base_result"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{2: (0, array([1])),\n",
              " 5: (0, array([1])),\n",
              " 10: (0, array([1])),\n",
              " 15: (0, array([1])),\n",
              " 19: (0, array([1])),\n",
              " 22: (0, array([1])),\n",
              " 26: (1, array([0])),\n",
              " 35: (0, array([1])),\n",
              " 52: (1, array([0])),\n",
              " 90: (0, array([1])),\n",
              " 91: (0, array([1])),\n",
              " 98: (0, array([1])),\n",
              " 99: (0, array([1])),\n",
              " 100: (1, array([0])),\n",
              " 104: (0, array([1])),\n",
              " 106: (0, array([1])),\n",
              " 107: (0, array([1])),\n",
              " 108: (0, array([1])),\n",
              " 113: (0, array([1])),\n",
              " 124: (0, array([1])),\n",
              " 140: (0, array([1])),\n",
              " 150: (1, array([0])),\n",
              " 156: (0, array([1])),\n",
              " 160: (0, array([1])),\n",
              " 161: (1, array([0])),\n",
              " 178: (0, array([1])),\n",
              " 200: (0, array([1])),\n",
              " 204: (1, array([0])),\n",
              " 208: (0, array([1])),\n",
              " 211: (0, array([1])),\n",
              " 218: (0, array([1])),\n",
              " 219: (0, array([1])),\n",
              " 221: (1, array([0])),\n",
              " 224: (0, array([1])),\n",
              " 229: (0, array([1])),\n",
              " 242: (1, array([0])),\n",
              " 247: (0, array([1])),\n",
              " 248: (1, array([0])),\n",
              " 260: (1, array([0])),\n",
              " 263: (0, array([1])),\n",
              " 271: (0, array([1])),\n",
              " 273: (1, array([0])),\n",
              " 291: (0, array([1])),\n",
              " 296: (0, array([1])),\n",
              " 297: (0, array([1])),\n",
              " 298: (1, array([0])),\n",
              " 300: (0, array([1])),\n",
              " 313: (0, array([1])),\n",
              " 321: (0, array([1])),\n",
              " 330: (1, array([0])),\n",
              " 332: (0, array([1])),\n",
              " 343: (1, array([0])),\n",
              " 358: (1, array([0])),\n",
              " 372: (0, array([1])),\n",
              " 377: (0, array([1])),\n",
              " 378: (0, array([1])),\n",
              " 380: (1, array([0])),\n",
              " 381: (0, array([1])),\n",
              " 392: (0, array([1])),\n",
              " 393: (1, array([0])),\n",
              " 410: (0, array([1])),\n",
              " 421: (0, array([1])),\n",
              " 427: (0, array([1])),\n",
              " 434: (0, array([1])),\n",
              " 437: (0, array([1])),\n",
              " 443: (0, array([1])),\n",
              " 458: (1, array([0])),\n",
              " 485: (1, array([0])),\n",
              " 486: (0, array([1])),\n",
              " 487: (1, array([0])),\n",
              " 488: (0, array([1])),\n",
              " 492: (0, array([1])),\n",
              " 498: (0, array([1])),\n",
              " 520: (1, array([0])),\n",
              " 524: (0, array([1])),\n",
              " 530: (1, array([0])),\n",
              " 531: (0, array([1])),\n",
              " 533: (0, array([1])),\n",
              " 539: (0, array([1])),\n",
              " 542: (0, array([1])),\n",
              " 552: (0, array([1])),\n",
              " 562: (1, array([0])),\n",
              " 564: (0, array([1])),\n",
              " 565: (0, array([1]))}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cb-GDRj6Kwoz",
        "outputId": "c54ece9b-b5da-4074-d532-921dd0273699"
      },
      "source": [
        "len(mmbt_base_idxs)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "84"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vd9Vx4B_Lnuo",
        "outputId": "3dfce957-78ed-40a1-c10d-a1ad222c02ad"
      },
      "source": [
        "len(base_idxs)"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "70"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sJckD1JXjLYB"
      },
      "source": [
        "## Investigate if there are cases where MMBT corrects our unimodal model.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7N6SvPqqlDL9"
      },
      "source": [
        "mmbt_base_corrected = []\n",
        "for idx in base_idxs:\n",
        "  if idx not in mmbt_base_idxs:\n",
        "    mmbt_base_corrected.append(idx)"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lVPzKq5NjdeS"
      },
      "source": [
        "## Visualize those cases where MMBT corrected.\n",
        "\n",
        "label = target label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "IwjEi8G7mGM-",
        "outputId": "2d7e4d7f-7c19-4f65-e9ad-08ba78f7ccac"
      },
      "source": [
        "df_mmbt_base_corrected = [baseline.iloc[idx] for idx in mmbt_base_corrected]\n",
        "df_mmbt_base_corrected = pd.DataFrame(df_mmbt_base_corrected)\n",
        "df_mmbt_base_corrected"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>img</th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>1628</td>\n",
              "      <td>CXR3355_IM-1609-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Cardiomediastinal silhouettes are within normal limits. Lungs are hyperexpanded. Lungs are clear without focal consolidation, pneumothorax, or pleural effusion. Bony thorax is unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>1707</td>\n",
              "      <td>CXR3494_IM-1699-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>There are low lung volumes with bronchovascular crowding. There is no focal consolidation. No visualized pneumothorax. Heart size is within normal limits. The cardiomediastinal contours is grossly normal in size and contour.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>50</td>\n",
              "      <td>CXR102_IM-0016-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal heart size. Clear, hyperaerated lungs. No pneumothorax. No pleural effusion. XXXX substernal density may be related to a pectus deformity.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>330</td>\n",
              "      <td>CXR655_IM-2231-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal heart size and mediastinal contours. The lungs are hyperinflated but clear. No pneumothorax or pleural effusion. No acute bony abnormalities.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>2493</td>\n",
              "      <td>CXR2390_IM-0944-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is borderline in size. The mediastinum is stable with changes of XXXX sternotomy and bypass graft. Aorta is atherosclerotic. There are postsurgical changes of the left hemithorax with mild left-sided volume loss as evidenced by diaphragm elevation. Left post thoracotomy rib changes are noted. The right lung is clear. There is no pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>2473</td>\n",
              "      <td>CXR2314_IM-0889-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The lungs are XXXX. XXXX opacities are present in the right costophrenic XXXX. No focal infiltrates. Heart size normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>115</th>\n",
              "      <td>2836</td>\n",
              "      <td>CXR3961_IM-2026-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is normal. The mediastinal contour is within normal limits. There is a streaky opacity within the right upper lobe. There are no nodules or masses. No visible pneumothorax. No visible pleural fluid. The XXXX are grossly normal. There is no visible free intraperitoneal air under the diaphragm.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>128</th>\n",
              "      <td>1817</td>\n",
              "      <td>CXR3708_IM-1852-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Lung volumes are decreased from XXXX, and there is resultant bronchovascular crowding. No evidence of focal airspace disease. No definite pleural effusion or pneumothorax. Cardiomediastinal silhouette is within normal limits given the low lung volumes. No free subdiaphragmatic air. Grossly stable mild degenerative changes of the right lower thoracic spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>136</th>\n",
              "      <td>6</td>\n",
              "      <td>CXR14_IM-0256-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Heart size within normal limits, stable mediastinal and hilar contours. Mild hyperinflation appears similar to prior. No focal alveolar consolidation, no definite pleural effusion seen. Scattered chronic appearing irregular interstitial markings, no typical findings of pulmonary edema.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>143</th>\n",
              "      <td>2622</td>\n",
              "      <td>CXR3005_IM-1388-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart is normal in size. The mediastinum is stable. Aorta is tortuous. Calcified lymph XXXX are again identified. There is mild prominence of the right paratracheal soft tissues, stable in appearance from prior studies. There is no acute infiltrate or pleural effusion. Osteopenia and degenerative changes are identified. XXXX deformity of T9 appears worse than prior study.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>185</th>\n",
              "      <td>2508</td>\n",
              "      <td>CXR2448_IM-0983-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>81</td>\n",
              "      <td>CXR172_IM-0474-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Elevated right hemidiaphragm. Clear lungs. No pleural effusions or pneumothoraces. heart size is upper limits of normal with tortuosity and ectasia of the aorta. Generative changes within the spine. In the upper lumbar spine there is an age-indeterminate wedge XXXX of a vertebral body.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>1974</td>\n",
              "      <td>CXR75_IM-2303-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is stable. The aorta is ectatic and atherosclerotic but stable. XXXX sternotomy XXXX are again noted. The scarring in the left lower lobe is again noted and unchanged from prior exam. There are mild bilateral prominent lung interstitial opacities consistent with emphysematous disease. The calcified granulomas are stable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>305</th>\n",
              "      <td>47</td>\n",
              "      <td>CXR98_IM-2467-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Frontal and lateral views of the chest show an unchanged cardiomediastinal silhouette. Reduced lung volumes with basilar atelectasis. No XXXX focal airspace consolidation or pleural effusion.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>309</th>\n",
              "      <td>72</td>\n",
              "      <td>CXR152_IM-0335-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Stable cardiomediastinal silhouette with mild cardiomegaly and aortic ectasia and tortuosity. No alveolar consolidation, no findings of pleural effusion. Chronic appearing bilateral rib contour deformities compatible with old fractures. No pneumothorax.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>337</th>\n",
              "      <td>1410</td>\n",
              "      <td>CXR2934_IM-1337-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart is normal in size. The mediastinum is unremarkable. The lungs are mildly hyperlucent but clear. There is denser lumbar scoliosis.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>357</th>\n",
              "      <td>2164</td>\n",
              "      <td>CXR991_IM-2476-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is mild streaky perihilar opacity without confluent airspace opacity to suggest a bacterial pneumonia.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>382</th>\n",
              "      <td>527</td>\n",
              "      <td>CXR1037_IM-0029-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The heart is normal size. The mediastinum is unremarkable. A tortuous, calcified thoracic aorta is present. The lungs are hyperexpanded, consistent with emphysema. There is no pleural effusion, pneumothorax, or focal airspace disease. The XXXX are unremarkable.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>422</th>\n",
              "      <td>1397</td>\n",
              "      <td>CXR2908_IM-1312-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Heart size is upper limits of normal but stable. Mediastinal contours are within normal limits.. Chronically increased interstitial markings without focal airspace consolidation, pleural effusion, pneumothorax. Degenerative changes of the spine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>440</th>\n",
              "      <td>2117</td>\n",
              "      <td>CXR744_IM-2299-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size and pulmonary vascularity appear within normal limits. Left XXXX-a-XXXX is in XXXX. No pleural effusion or pneumothorax is seen. Right upper lobe area of dense opacity is seen in the medial right apex. On a previous outside XXXX scan (XXXX), the right upper lobe was consolidated. Comparison to the XXXX XXXX from that exam shows this opacity to have decreased. No films were available, however, for direct comparison.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>442</th>\n",
              "      <td>1152</td>\n",
              "      <td>CXR2364_IM-0927-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>The lungs are clear. There is hyperexpansion of the lungs suggesting underlying emphysema. The heart and pulmonary XXXX appear normal. Pleural spaces are clear. Mediastinal contours are normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>493</th>\n",
              "      <td>442</td>\n",
              "      <td>CXR864_IM-2384-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Stable cardiomediastinal silhouette with tortuous thoracic aorta. No pneumothorax, pleural effusion or suspicious focal air space opacity. Stable right lung base scarring.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>505</th>\n",
              "      <td>2631</td>\n",
              "      <td>CXR3072_IM-1433-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Left-sided medication injection XXXX has its tip projecting at the cavoatrial junction. The trachea is midline. Extensive bilateral bronchiectasis, cystic changes, and scarring represents sequela from the patient's cystic fibrosis. No evidence of focal pulmonary infiltrate or pleural effusion. No large pneumothorax has developed in the interim. The overlying bony structures reveal no acute abnormalities. The heart size is normal.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>526</th>\n",
              "      <td>2193</td>\n",
              "      <td>CXR1089_IM-0061-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The heart size and pulmonary vascularity appear within normal limits. The descending thoracic aorta is tortuous. Central venous catheter is again noted. The lungs are free of focal airspace disease. The left hemidiaphragm remains elevated. No pneumothorax or pleural effusion is seen.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>548</th>\n",
              "      <td>2484</td>\n",
              "      <td>CXR2363_IM-0926-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is right basilar air space opacity.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>553</th>\n",
              "      <td>954</td>\n",
              "      <td>CXR1924_IM-0598-1001.png</td>\n",
              "      <td>0</td>\n",
              "      <td>Rotated apical lordotic frontal projection, mild bronchovascular crowding and scattered chronic appearing irregular interstitial markings. No definite focal alveolar consolidation or pleural effusion seen. Accounting for technical factors heart size XXXX within normal limits, heavily calcified and mildly tortuous aorta. No typical findings of pulmonary edema.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>567</th>\n",
              "      <td>2341</td>\n",
              "      <td>CXR1735_IM-0484-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The trachea is midline. The cardiomediastinal silhouette is normal in size and unchanged from prior examinations with sternotomy XXXX and surgical clips overlying. There is blunting of the right costophrenic XXXX which appears unchanged from prior examination and may be secondary to scarring or pleural thickening of the right lung base. There is no evidence of acute infiltrate. There is no pneumothorax. Visualized bony structures reveal no acute abnormalities.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                               text\n",
              "49   1628        ...   Cardiomediastinal silhouettes are within normal limits. Lungs are hyperexpanded. Lungs are clear without focal consolidation, pneumothorax, or pleural effusion. Bony thorax is unremarkable.                                                                                                                                                                                                                                                                                   \n",
              "54   1707        ...   There are low lung volumes with bronchovascular crowding. There is no focal consolidation. No visualized pneumothorax. Heart size is within normal limits. The cardiomediastinal contours is grossly normal in size and contour.                                                                                                                                                                                                                                                \n",
              "56   50          ...   Normal heart size. Clear, hyperaerated lungs. No pneumothorax. No pleural effusion. XXXX substernal density may be related to a pectus deformity.                                                                                                                                                                                                                                                                                                                               \n",
              "80   330         ...   Normal heart size and mediastinal contours. The lungs are hyperinflated but clear. No pneumothorax or pleural effusion. No acute bony abnormalities.                                                                                                                                                                                                                                                                                                                            \n",
              "86   2493        ...   The heart is borderline in size. The mediastinum is stable with changes of XXXX sternotomy and bypass graft. Aorta is atherosclerotic. There are postsurgical changes of the left hemithorax with mild left-sided volume loss as evidenced by diaphragm elevation. Left post thoracotomy rib changes are noted. The right lung is clear. There is no pleural effusion.                                                                                                          \n",
              "88   2473        ...   The lungs are XXXX. XXXX opacities are present in the right costophrenic XXXX. No focal infiltrates. Heart size normal.                                                                                                                                                                                                                                                                                                                                                         \n",
              "115  2836        ...   The heart size is normal. The mediastinal contour is within normal limits. There is a streaky opacity within the right upper lobe. There are no nodules or masses. No visible pneumothorax. No visible pleural fluid. The XXXX are grossly normal. There is no visible free intraperitoneal air under the diaphragm.                                                                                                                                                            \n",
              "128  1817        ...   Lung volumes are decreased from XXXX, and there is resultant bronchovascular crowding. No evidence of focal airspace disease. No definite pleural effusion or pneumothorax. Cardiomediastinal silhouette is within normal limits given the low lung volumes. No free subdiaphragmatic air. Grossly stable mild degenerative changes of the right lower thoracic spine.                                                                                                          \n",
              "136  6           ...   Heart size within normal limits, stable mediastinal and hilar contours. Mild hyperinflation appears similar to prior. No focal alveolar consolidation, no definite pleural effusion seen. Scattered chronic appearing irregular interstitial markings, no typical findings of pulmonary edema.                                                                                                                                                                                  \n",
              "143  2622        ...   The heart is normal in size. The mediastinum is stable. Aorta is tortuous. Calcified lymph XXXX are again identified. There is mild prominence of the right paratracheal soft tissues, stable in appearance from prior studies. There is no acute infiltrate or pleural effusion. Osteopenia and degenerative changes are identified. XXXX deformity of T9 appears worse than prior study.                                                                                      \n",
              "185  2508        ...   Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "213  81          ...   Elevated right hemidiaphragm. Clear lungs. No pleural effusions or pneumothoraces. heart size is upper limits of normal with tortuosity and ectasia of the aorta. Generative changes within the spine. In the upper lumbar spine there is an age-indeterminate wedge XXXX of a vertebral body.                                                                                                                                                                                  \n",
              "241  1974        ...   The heart size is stable. The aorta is ectatic and atherosclerotic but stable. XXXX sternotomy XXXX are again noted. The scarring in the left lower lobe is again noted and unchanged from prior exam. There are mild bilateral prominent lung interstitial opacities consistent with emphysematous disease. The calcified granulomas are stable.                                                                                                                               \n",
              "305  47          ...   Frontal and lateral views of the chest show an unchanged cardiomediastinal silhouette. Reduced lung volumes with basilar atelectasis. No XXXX focal airspace consolidation or pleural effusion.                                                                                                                                                                                                                                                                                 \n",
              "309  72          ...   Stable cardiomediastinal silhouette with mild cardiomegaly and aortic ectasia and tortuosity. No alveolar consolidation, no findings of pleural effusion. Chronic appearing bilateral rib contour deformities compatible with old fractures. No pneumothorax.                                                                                                                                                                                                                   \n",
              "337  1410        ...   The heart is normal in size. The mediastinum is unremarkable. The lungs are mildly hyperlucent but clear. There is denser lumbar scoliosis.                                                                                                                                                                                                                                                                                                                                     \n",
              "357  2164        ...   The heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is mild streaky perihilar opacity without confluent airspace opacity to suggest a bacterial pneumonia.                                                                                                                                                                                                               \n",
              "382  527         ...   The heart is normal size. The mediastinum is unremarkable. A tortuous, calcified thoracic aorta is present. The lungs are hyperexpanded, consistent with emphysema. There is no pleural effusion, pneumothorax, or focal airspace disease. The XXXX are unremarkable.                                                                                                                                                                                                           \n",
              "422  1397        ...   Heart size is upper limits of normal but stable. Mediastinal contours are within normal limits.. Chronically increased interstitial markings without focal airspace consolidation, pleural effusion, pneumothorax. Degenerative changes of the spine.                                                                                                                                                                                                                           \n",
              "440  2117        ...   The heart size and pulmonary vascularity appear within normal limits. Left XXXX-a-XXXX is in XXXX. No pleural effusion or pneumothorax is seen. Right upper lobe area of dense opacity is seen in the medial right apex. On a previous outside XXXX scan (XXXX), the right upper lobe was consolidated. Comparison to the XXXX XXXX from that exam shows this opacity to have decreased. No films were available, however, for direct comparison.                               \n",
              "442  1152        ...   The lungs are clear. There is hyperexpansion of the lungs suggesting underlying emphysema. The heart and pulmonary XXXX appear normal. Pleural spaces are clear. Mediastinal contours are normal.                                                                                                                                                                                                                                                                               \n",
              "493  442         ...   Stable cardiomediastinal silhouette with tortuous thoracic aorta. No pneumothorax, pleural effusion or suspicious focal air space opacity. Stable right lung base scarring.                                                                                                                                                                                                                                                                                                     \n",
              "505  2631        ...   Left-sided medication injection XXXX has its tip projecting at the cavoatrial junction. The trachea is midline. Extensive bilateral bronchiectasis, cystic changes, and scarring represents sequela from the patient's cystic fibrosis. No evidence of focal pulmonary infiltrate or pleural effusion. No large pneumothorax has developed in the interim. The overlying bony structures reveal no acute abnormalities. The heart size is normal.                               \n",
              "526  2193        ...   The heart size and pulmonary vascularity appear within normal limits. The descending thoracic aorta is tortuous. Central venous catheter is again noted. The lungs are free of focal airspace disease. The left hemidiaphragm remains elevated. No pneumothorax or pleural effusion is seen.                                                                                                                                                                                    \n",
              "548  2484        ...   Heart size is upper limits of normal. The pulmonary XXXX and mediastinum are within normal limits. There is no pleural effusion or pneumothorax. There is right basilar air space opacity.                                                                                                                                                                                                                                                                                      \n",
              "553  954         ...   Rotated apical lordotic frontal projection, mild bronchovascular crowding and scattered chronic appearing irregular interstitial markings. No definite focal alveolar consolidation or pleural effusion seen. Accounting for technical factors heart size XXXX within normal limits, heavily calcified and mildly tortuous aorta. No typical findings of pulmonary edema.                                                                                                       \n",
              "567  2341        ...   The trachea is midline. The cardiomediastinal silhouette is normal in size and unchanged from prior examinations with sternotomy XXXX and surgical clips overlying. There is blunting of the right costophrenic XXXX which appears unchanged from prior examination and may be secondary to scarring or pleural thickening of the right lung base. There is no evidence of acute infiltrate. There is no pneumothorax. Visualized bony structures reveal no acute abnormalities.\n",
              "\n",
              "[27 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OEXmnLYuFUYn",
        "outputId": "6adaa898-4dad-45d1-92e6-c63ab32ceb5c"
      },
      "source": [
        "len(df_mmbt_base_corrected)"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "27"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_5NVULwpjuAC"
      },
      "source": [
        "## Run major MMBT model\n",
        "\n",
        "Loads the pretrained weights and evaluates model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ig69cwGytXd9",
        "outputId": "552460ca-d643-452d-d98c-66d6c29c3b2d"
      },
      "source": [
        "mmbt_major_checkpoint = os.path.join(MODELS_MMBT_DIR, models[1])\n",
        "mmbt_model.load_state_dict(torch.load(mmbt_major_checkpoint, map_location=device))\n",
        "\n",
        "mmbt_major_result = mmbt_evaluate(mmbt_model, tokenizer, dataloaders[1])\n",
        "mmbt_major_idxs = list(mmbt_major_result.keys())"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluating: 100%|██████████| 18/18 [00:10<00:00,  1.79it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Z33Vjktj_lu"
      },
      "source": [
        "## False predictions\n",
        "\n",
        "- Key: index in dataset\n",
        "- Value: tuple (prediction, target)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1lk8CgVktgWg",
        "outputId": "3fbbe75c-ff44-487a-fd46-a505d3162880"
      },
      "source": [
        "mmbt_major_result"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{5: (1, array([0])),\n",
              " 27: (0, array([1])),\n",
              " 48: (0, array([1])),\n",
              " 63: (1, array([0])),\n",
              " 107: (1, array([0])),\n",
              " 135: (1, array([0])),\n",
              " 166: (1, array([0])),\n",
              " 193: (0, array([1])),\n",
              " 234: (0, array([1])),\n",
              " 260: (1, array([0])),\n",
              " 277: (0, array([1])),\n",
              " 283: (0, array([1])),\n",
              " 291: (1, array([0])),\n",
              " 303: (1, array([0])),\n",
              " 306: (1, array([0])),\n",
              " 340: (1, array([0])),\n",
              " 369: (1, array([0])),\n",
              " 378: (0, array([1])),\n",
              " 403: (0, array([1])),\n",
              " 407: (0, array([1])),\n",
              " 424: (1, array([0])),\n",
              " 431: (1, array([0])),\n",
              " 448: (1, array([0])),\n",
              " 463: (1, array([0])),\n",
              " 479: (0, array([1])),\n",
              " 547: (1, array([0]))}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tT-gJ1nFtiQX",
        "outputId": "d7598ee0-d96d-4e49-de4f-7a0b023e200a"
      },
      "source": [
        "len(mmbt_major_idxs)"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "26"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oip2gYsEtmLe",
        "outputId": "cc43545d-3d25-4dcd-e5db-a9773593e897"
      },
      "source": [
        "len(major_idxs)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gsFMEIwXkYro"
      },
      "source": [
        "## Investigate if there are cases where MMBT corrects our unimodal model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5obddP3Ntnul"
      },
      "source": [
        "mmbt_major_corrected = []\n",
        "for idx in major_idxs:\n",
        "  if idx not in mmbt_major_idxs:\n",
        "    mmbt_major_corrected.append(idx)"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VSbuNwkmkf3b"
      },
      "source": [
        "## Visualize those cases where MMBT corrected.\n",
        "\n",
        "label = target label\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "id": "SLOYCLAtt0rP",
        "outputId": "b6280b71-af4e-4713-f497-8b951b64d319"
      },
      "source": [
        "df_mmbt_major_corrected = [major.iloc[idx] for idx in mmbt_major_corrected]\n",
        "df_mmbt_major_corrected = pd.DataFrame(df_mmbt_major_corrected)\n",
        "df_mmbt_major_corrected"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>img</th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>185</th>\n",
              "      <td>2508</td>\n",
              "      <td>CXR2448_IM-0983-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>282</th>\n",
              "      <td>683</td>\n",
              "      <td>CXR1392_IM-0251-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>The aortic XXXX, cardiac apex, and stomach are left-sided. Cardiomediastinal silhouette is within normal limits in overall size and appearance. Pulmonary vascular markings are symmetric and within normal limits. The lungs are normally inflated with no focal airspace disease, pleural effusion, or pneumothorax. No acute bony abnormality.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>473</th>\n",
              "      <td>1171</td>\n",
              "      <td>CXR2406_IM-0954-1001.png</td>\n",
              "      <td>1</td>\n",
              "      <td>Both lungs remain clear and expanded. Heart and pulmonary XXXX are normal. No change in the large hiatus hernia.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0  ...                                                                                                                                                                                                                                                                                                                                                text\n",
              "185  2508        ...   Mediastinal contours are normal. No significant change in pneumothorax or right pleural fluid..                                                                                                                                                                                                                                                  \n",
              "282  683         ...   The aortic XXXX, cardiac apex, and stomach are left-sided. Cardiomediastinal silhouette is within normal limits in overall size and appearance. Pulmonary vascular markings are symmetric and within normal limits. The lungs are normally inflated with no focal airspace disease, pleural effusion, or pneumothorax. No acute bony abnormality.\n",
              "473  1171        ...   Both lungs remain clear and expanded. Heart and pulmonary XXXX are normal. No change in the large hiatus hernia.                                                                                                                                                                                                                                 \n",
              "\n",
              "[3 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xpYGcK29uNS8",
        "outputId": "04fa2a80-1769-4176-e436-1c82e03acbc7"
      },
      "source": [
        "len(df_mmbt_major_corrected)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMe4LpkFkEIe"
      },
      "source": [
        "## Check if there are cases of false predictions that overlap between different models."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTD1y92jLtfZ",
        "outputId": "a643b576-5a18-4e61-e99f-26a4612960e5"
      },
      "source": [
        "set(mmbt_base_idxs) & set(mmbt_major_idxs)"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{5, 107, 260, 291, 378}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jkz7QuS-L1Vt",
        "outputId": "2a89bf1f-a66f-4e24-8767-1157206c582e"
      },
      "source": [
        "set(base_idxs) & set(major_idxs)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{185, 291}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Klb6ms_MVIr",
        "outputId": "6d047732-7144-4e99-b76e-a15410e88ae1"
      },
      "source": [
        "set(base_idxs) & set(mmbt_base_idxs)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{5,\n",
              " 10,\n",
              " 15,\n",
              " 19,\n",
              " 22,\n",
              " 91,\n",
              " 98,\n",
              " 99,\n",
              " 100,\n",
              " 106,\n",
              " 107,\n",
              " 108,\n",
              " 113,\n",
              " 140,\n",
              " 156,\n",
              " 204,\n",
              " 219,\n",
              " 224,\n",
              " 229,\n",
              " 242,\n",
              " 260,\n",
              " 271,\n",
              " 273,\n",
              " 291,\n",
              " 296,\n",
              " 297,\n",
              " 313,\n",
              " 410,\n",
              " 421,\n",
              " 434,\n",
              " 437,\n",
              " 443,\n",
              " 458,\n",
              " 485,\n",
              " 487,\n",
              " 488,\n",
              " 498,\n",
              " 520,\n",
              " 530,\n",
              " 531,\n",
              " 533,\n",
              " 542,\n",
              " 564}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KQ95aje8McRB",
        "outputId": "30979250-4272-4c4f-9a05-1c4b703ae5f0"
      },
      "source": [
        "set(major_idxs) & set(mmbt_major_idxs)"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{48, 63, 135, 166, 277, 291, 306, 340, 403, 424, 431, 448, 479, 547}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQtVAd68eUbq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e0d5c79-24ee-43db-e04a-5936069db161"
      },
      "source": [
        "drive.flush_and_unmount()\n",
        "print('All changes made in this colab session should now be visible in Drive.')"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All changes made in this colab session should now be visible in Drive.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}