bakllava.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/santoryudeadman/whisper/blob/main/bakllava.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip ngrok-stable-linux-amd64.zip\n",
        "!./ngrok authtoken 2XMswsT2z3mRwXoEkALduNhNDoA_7tfvtmZ56wGnt7qebcjHd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LE3mzeZFsqYe",
        "outputId": "5b871cb0-90e0-421a-a431-576c0cc2aa4b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-10-28 04:36:58--  https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
            "Resolving bin.equinox.io (bin.equinox.io)... 54.161.241.46, 54.237.133.81, 18.205.222.128, ...\n",
            "Connecting to bin.equinox.io (bin.equinox.io)|54.161.241.46|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 13921656 (13M) [application/octet-stream]\n",
            "Saving to: ‘ngrok-stable-linux-amd64.zip’\n",
            "\n",
            "ngrok-stable-linux- 100%[===================>]  13.28M  21.7MB/s    in 0.6s    \n",
            "\n",
            "2023-10-28 04:36:59 (21.7 MB/s) - ‘ngrok-stable-linux-amd64.zip’ saved [13921656/13921656]\n",
            "\n",
            "Archive:  ngrok-stable-linux-amd64.zip\n",
            "  inflating: ngrok                   \n",
            "Authtoken saved to configuration file: /root/.ngrok2/ngrok.yml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZsfX6ICpjPif",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03064834-b244-442e-b0d5-9ac9650c400d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[33m\r0% [Working]\u001b[0m\r            \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]\n",
            "Get:6 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ Packages [44.8 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB]\n",
            "Hit:8 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,274 kB]\n",
            "Hit:10 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,400 kB]\n",
            "Get:12 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,131 kB]\n",
            "Hit:13 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,009 kB]\n",
            "Hit:15 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Fetched 5,200 kB in 3s (1,976 kB/s)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "20 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "build-essential is already the newest version (12.9ubuntu3).\n",
            "The following additional packages will be installed:\n",
            "  libaria2-0 libc-ares2\n",
            "The following NEW packages will be installed:\n",
            "  aria2 libaria2-0 libc-ares2\n",
            "0 upgraded, 3 newly installed, 0 to remove and 20 not upgraded.\n",
            "Need to get 1,513 kB of archives.\n",
            "After this operation, 5,441 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libc-ares2 amd64 1.18.1-1ubuntu0.22.04.2 [45.0 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libaria2-0 amd64 1.36.0-1 [1,086 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 aria2 amd64 1.36.0-1 [381 kB]\n",
            "Fetched 1,513 kB in 1s (2,000 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 3.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libc-ares2:amd64.\n",
            "(Reading database ... 120874 files and directories currently installed.)\n",
            "Preparing to unpack .../libc-ares2_1.18.1-1ubuntu0.22.04.2_amd64.deb ...\n",
            "Unpacking libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.2) ...\n",
            "Selecting previously unselected package libaria2-0:amd64.\n",
            "Preparing to unpack .../libaria2-0_1.36.0-1_amd64.deb ...\n",
            "Unpacking libaria2-0:amd64 (1.36.0-1) ...\n",
            "Selecting previously unselected package aria2.\n",
            "Preparing to unpack .../aria2_1.36.0-1_amd64.deb ...\n",
            "Unpacking aria2 (1.36.0-1) ...\n",
            "Setting up libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.2) ...\n",
            "Setting up libaria2-0:amd64 (1.36.0-1) ...\n",
            "Setting up aria2 (1.36.0-1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.1) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "\n",
            "10/28 04:37:11 [\u001b[1;32mNOTICE\u001b[0m] Downloading 1 item(s)\n",
            "\n",
            "10/28 04:37:11 [\u001b[1;32mNOTICE\u001b[0m] CUID#7 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27ggml-model-q5_k.gguf%3B+filename%3D%22ggml-model-q5_k.gguf%22%3B&Expires=1698727031&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzAzMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvYzkzZGUxMzc2YmU5YjY5NzdjYzk0ZDI1MmEzZDE2NWQ2MDU5ZTA3YjUyOGRlMGZhNzYyNTM0ZDk1OTliMjdkNj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=B5CDxP1L3h5xkCHqH-kSKWuPwEv%7EBnqX-eHVraZo9fZUGm-K0LG7HYt4qz%7EWd9Isq-IbUdyVs0-gy2HObZ-dd8FSTaW5dmEg8b-YAM8lVzy3UnEmriIdwziAE4r2x7Qxv3jqH8itP3pMkfFD5nxi-yylyZnhdjCygwBOav92eQJiOx8DMurlMJb4vUdV0EZG%7EynfbMThrUCfg-uxRazevMVO5bP2jX2Q94QHdjC41z3wed4z2s9kYlr8KpfHxZlNmiWtKb74v%7EXx7f3DfhqSdE--cfnbo7Uhmg1RUf6eqBw7luqelJF9sMY1H3an8SetT9SBw7h31XpbyloDhHDJeQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\n",
            "10/28 04:37:11 [\u001b[1;32mNOTICE\u001b[0m] CUID#9 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27ggml-model-q5_k.gguf%3B+filename%3D%22ggml-model-q5_k.gguf%22%3B&Expires=1698727031&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzAzMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvYzkzZGUxMzc2YmU5YjY5NzdjYzk0ZDI1MmEzZDE2NWQ2MDU5ZTA3YjUyOGRlMGZhNzYyNTM0ZDk1OTliMjdkNj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=B5CDxP1L3h5xkCHqH-kSKWuPwEv%7EBnqX-eHVraZo9fZUGm-K0LG7HYt4qz%7EWd9Isq-IbUdyVs0-gy2HObZ-dd8FSTaW5dmEg8b-YAM8lVzy3UnEmriIdwziAE4r2x7Qxv3jqH8itP3pMkfFD5nxi-yylyZnhdjCygwBOav92eQJiOx8DMurlMJb4vUdV0EZG%7EynfbMThrUCfg-uxRazevMVO5bP2jX2Q94QHdjC41z3wed4z2s9kYlr8KpfHxZlNmiWtKb74v%7EXx7f3DfhqSdE--cfnbo7Uhmg1RUf6eqBw7luqelJF9sMY1H3an8SetT9SBw7h31XpbyloDhHDJeQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\n",
            "10/28 04:37:11 [\u001b[1;32mNOTICE\u001b[0m] CUID#10 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27ggml-model-q5_k.gguf%3B+filename%3D%22ggml-model-q5_k.gguf%22%3B&Expires=1698727031&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzAzMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvYzkzZGUxMzc2YmU5YjY5NzdjYzk0ZDI1MmEzZDE2NWQ2MDU5ZTA3YjUyOGRlMGZhNzYyNTM0ZDk1OTliMjdkNj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=B5CDxP1L3h5xkCHqH-kSKWuPwEv%7EBnqX-eHVraZo9fZUGm-K0LG7HYt4qz%7EWd9Isq-IbUdyVs0-gy2HObZ-dd8FSTaW5dmEg8b-YAM8lVzy3UnEmriIdwziAE4r2x7Qxv3jqH8itP3pMkfFD5nxi-yylyZnhdjCygwBOav92eQJiOx8DMurlMJb4vUdV0EZG%7EynfbMThrUCfg-uxRazevMVO5bP2jX2Q94QHdjC41z3wed4z2s9kYlr8KpfHxZlNmiWtKb74v%7EXx7f3DfhqSdE--cfnbo7Uhmg1RUf6eqBw7luqelJF9sMY1H3an8SetT9SBw7h31XpbyloDhHDJeQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\n",
            "10/28 04:37:11 [\u001b[1;32mNOTICE\u001b[0m] CUID#11 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/c93de1376be9b6977cc94d252a3d165d6059e07b528de0fa762534d9599b27d6?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27ggml-model-q5_k.gguf%3B+filename%3D%22ggml-model-q5_k.gguf%22%3B&Expires=1698727031&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzAzMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvYzkzZGUxMzc2YmU5YjY5NzdjYzk0ZDI1MmEzZDE2NWQ2MDU5ZTA3YjUyOGRlMGZhNzYyNTM0ZDk1OTliMjdkNj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=B5CDxP1L3h5xkCHqH-kSKWuPwEv%7EBnqX-eHVraZo9fZUGm-K0LG7HYt4qz%7EWd9Isq-IbUdyVs0-gy2HObZ-dd8FSTaW5dmEg8b-YAM8lVzy3UnEmriIdwziAE4r2x7Qxv3jqH8itP3pMkfFD5nxi-yylyZnhdjCygwBOav92eQJiOx8DMurlMJb4vUdV0EZG%7EynfbMThrUCfg-uxRazevMVO5bP2jX2Q94QHdjC41z3wed4z2s9kYlr8KpfHxZlNmiWtKb74v%7EXx7f3DfhqSdE--cfnbo7Uhmg1RUf6eqBw7luqelJF9sMY1H3an8SetT9SBw7h31XpbyloDhHDJeQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\u001b[0m\n",
            "10/28 04:37:38 [\u001b[1;32mNOTICE\u001b[0m] Download complete: /content/ggml-model-q5_k.gguf\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "a72e66|\u001b[1;32mOK\u001b[0m  |   188MiB/s|/content/ggml-model-q5_k.gguf\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "10/28 04:37:38 [\u001b[1;32mNOTICE\u001b[0m] Downloading 1 item(s)\n",
            "\n",
            "10/28 04:37:38 [\u001b[1;32mNOTICE\u001b[0m] CUID#7 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27mmproj-model-f16.gguf%3B+filename%3D%22mmproj-model-f16.gguf%22%3B&Expires=1698727058&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzA1OH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvMmU0NjdlYmE3MTAwMDI4MzllMDk2NmQ1ZTMyOTk0MmJiODM2ZWFiZDRlNzg3YmM3MTNiMDdlZmYxZDhlYTEzYj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=UgSwFlsmUME4ANr%7EtUam%7EjoRO2m4cDifJwuCY4pDlvGD9qekDAL9QICoKEVCeVF0J-E9Nhe1Nf-RcIFZAdR4f6uIdKZILXaSWLK4uabq2hcXbpIQF7S0h6jH8ms1S64v30MfTYVhPZZoXH7QzSnBlb-N3xTUo3zyu9LwbBg5bZxs6jU3FmfKk2FO2eoGai-2OudKc-57kfG9EBZXgK6PejoUMz8AW6ydPPbCJZVJ5Bpd2zQ9wSrj0Io3FiJA6C1notGKS8%7EIkeJYpuEE-rwhXwb0JNnroCX9MUDFCPCOzXQHCQ7NyJ4ZO6z-YHCir1cawOz2XYY3tOQBf35yo00LhQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\n",
            "10/28 04:37:39 [\u001b[1;32mNOTICE\u001b[0m] CUID#9 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27mmproj-model-f16.gguf%3B+filename%3D%22mmproj-model-f16.gguf%22%3B&Expires=1698727058&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzA1OH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvMmU0NjdlYmE3MTAwMDI4MzllMDk2NmQ1ZTMyOTk0MmJiODM2ZWFiZDRlNzg3YmM3MTNiMDdlZmYxZDhlYTEzYj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=UgSwFlsmUME4ANr%7EtUam%7EjoRO2m4cDifJwuCY4pDlvGD9qekDAL9QICoKEVCeVF0J-E9Nhe1Nf-RcIFZAdR4f6uIdKZILXaSWLK4uabq2hcXbpIQF7S0h6jH8ms1S64v30MfTYVhPZZoXH7QzSnBlb-N3xTUo3zyu9LwbBg5bZxs6jU3FmfKk2FO2eoGai-2OudKc-57kfG9EBZXgK6PejoUMz8AW6ydPPbCJZVJ5Bpd2zQ9wSrj0Io3FiJA6C1notGKS8%7EIkeJYpuEE-rwhXwb0JNnroCX9MUDFCPCOzXQHCQ7NyJ4ZO6z-YHCir1cawOz2XYY3tOQBf35yo00LhQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\n",
            "10/28 04:37:39 [\u001b[1;32mNOTICE\u001b[0m] CUID#10 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27mmproj-model-f16.gguf%3B+filename%3D%22mmproj-model-f16.gguf%22%3B&Expires=1698727058&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzA1OH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvMmU0NjdlYmE3MTAwMDI4MzllMDk2NmQ1ZTMyOTk0MmJiODM2ZWFiZDRlNzg3YmM3MTNiMDdlZmYxZDhlYTEzYj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=UgSwFlsmUME4ANr%7EtUam%7EjoRO2m4cDifJwuCY4pDlvGD9qekDAL9QICoKEVCeVF0J-E9Nhe1Nf-RcIFZAdR4f6uIdKZILXaSWLK4uabq2hcXbpIQF7S0h6jH8ms1S64v30MfTYVhPZZoXH7QzSnBlb-N3xTUo3zyu9LwbBg5bZxs6jU3FmfKk2FO2eoGai-2OudKc-57kfG9EBZXgK6PejoUMz8AW6ydPPbCJZVJ5Bpd2zQ9wSrj0Io3FiJA6C1notGKS8%7EIkeJYpuEE-rwhXwb0JNnroCX9MUDFCPCOzXQHCQ7NyJ4ZO6z-YHCir1cawOz2XYY3tOQBf35yo00LhQ__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\n",
            "10/28 04:37:39 [\u001b[1;32mNOTICE\u001b[0m] CUID#11 - Redirecting to https://cdn-lfs-us-1.huggingface.co/repos/a2/57/a257b94bf6eefe8e8df451ecbacea70151490ff16dc2a7cc0ed53f3ed0aa198c/2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27mmproj-model-f16.gguf%3B+filename%3D%22mmproj-model-f16.gguf%22%3B&Expires=1698727059&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5ODcyNzA1OX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2EyLzU3L2EyNTdiOTRiZjZlZWZlOGU4ZGY0NTFlY2JhY2VhNzAxNTE0OTBmZjE2ZGMyYTdjYzBlZDUzZjNlZDBhYTE5OGMvMmU0NjdlYmE3MTAwMDI4MzllMDk2NmQ1ZTMyOTk0MmJiODM2ZWFiZDRlNzg3YmM3MTNiMDdlZmYxZDhlYTEzYj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=dw5gkAhzJSdlEKDh2Ey212dQjsHvfEG1l5NjFEVNOybWfWo3zVvVT1wFrYGaNMlGIxT1qlF9Ejw0HSOwRh3rXUSB1LFonQVXszr8dS8wGtie-K5iA3rYGgKcOe4%7EX0SXoU3Az358b%7EyjCr-qVrewkZnpLNc495TTehFFsmGZcww0YLhYYdwWPfWW2hcLenjERdki4IBd8voZsfSGAlAjLOMuQWnb7P%7E2fabdHwm3RSH3z9oN7O5ZVEiXo%7EAYoMnJH8OJWvPBfouLhXCqUIXhtpONQ7gxZC2xat9IlpkQCB65dmI6XV7PWzTBVV0JydB8gbeEVMYvz-b%7EBUaJO7GDkg__&Key-Pair-Id=KCD77M1F0VK2B\n",
            "\u001b[0m\n",
            "10/28 04:37:41 [\u001b[1;32mNOTICE\u001b[0m] Download complete: /content/mmproj-model-f16.gguf\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "e2da63|\u001b[1;32mOK\u001b[0m  |   259MiB/s|/content/mmproj-model-f16.gguf\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n"
          ]
        }
      ],
      "source": [
        "!sudo apt update -y && sudo apt install aria2 build-essential -y\n",
        "!aria2c -x4 -o ggml-model-q5_k.gguf https://huggingface.co/mys/ggml_bakllava-1/resolve/main/ggml-model-q5_k.gguf\n",
        "!aria2c -x4 -o mmproj-model-f16.gguf https://huggingface.co/mys/ggml_bakllava-1/resolve/main/mmproj-model-f16.gguf\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/ggerganov/llama.cpp && cd llama.cpp && make LLAMA_CUBLAS=1"
      ],
      "metadata": {
        "id": "EVDUuhLAjn5Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f85f0a92-50e0-42ae-f552-72e0b108e570"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'llama.cpp'...\n",
            "remote: Enumerating objects: 11482, done.\u001b[K\n",
            "remote: Counting objects: 100% (2835/2835), done.\u001b[K\n",
            "remote: Compressing objects: 100% (184/184), done.\u001b[K\n",
            "remote: Total 11482 (delta 2737), reused 2674 (delta 2651), pack-reused 8647\u001b[K\n",
            "Receiving objects: 100% (11482/11482), 13.54 MiB | 19.12 MiB/s, done.\n",
            "Resolving deltas: 100% (8003/8003), done.\n",
            "I llama.cpp build info: \n",
            "I UNAME_S:   Linux\n",
            "I UNAME_P:   x86_64\n",
            "I UNAME_M:   x86_64\n",
            "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wdouble-promotion -pthread -march=native -mtune=native \n",
            "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native \n",
            "I NVCCFLAGS: --forward-unknown-to-host-compiler -use_fast_math -arch=native -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DK_QUANTS_PER_ITERATION=2 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread    -Wno-pedantic -Xcompiler \"-Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native \"\n",
            "I LDFLAGS:   -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "I CC:        cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
            "I CXX:       g++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
            "\n",
            "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wdouble-promotion -pthread -march=native -mtune=native    -c ggml.c -o ggml.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -c llama.cpp -o llama.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -c common/common.cpp -o common.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -c common/sampling.cpp -o sampling.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -c common/grammar-parser.cpp -o grammar-parser.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -c common/console.cpp -o console.o\n",
            "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wdouble-promotion -pthread -march=native -mtune=native  -c k_quants.c -o k_quants.o\n",
            "nvcc --forward-unknown-to-host-compiler -use_fast_math -arch=native -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DK_QUANTS_PER_ITERATION=2 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread    -Wno-pedantic -Xcompiler \"-Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native \" -c ggml-cuda.cu -o ggml-cuda.o\n",
            "\u001b[01m\u001b[Kggml-cuda.cu:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kvoid ggml_cuda_op_clamp(const ggml_tensor*, const ggml_tensor*, ggml_tensor*, const float*, const float*, float*, CUstream_st* const&)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kggml-cuda.cu:6613:20:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kdereferencing type-punned pointer will break strict-aliasing rules [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wstrict-aliasing\u0007-Wstrict-aliasing\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            " 6613 |     const float mi\u001b[01;35m\u001b[Kn = ((float *) dst->op_para\u001b[m\u001b[Kms)[0];\n",
            "      |                   \u001b[01;35m\u001b[K~^~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kggml-cuda.cu:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kvoid ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor*, const ggml_tensor*, ggml_tensor*)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kggml-cuda.cu:7269:126:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kcast from type ‘\u001b[01m\u001b[Kvoid**\u001b[m\u001b[K’ to type ‘\u001b[01m\u001b[Kconst void**\u001b[m\u001b[K’ casts away qualifiers [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wcast-qual\u0007-Wcast-qual\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            " 7269 |         CUBLAS_CHECK(\n",
            "      |                                                                                                                              \u001b[01;35m\u001b[K^\u001b[m\u001b[K                                    \n",
            "\u001b[01m\u001b[Kggml-cuda.cu:7269:198:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kcast from type ‘\u001b[01m\u001b[Kvoid**\u001b[m\u001b[K’ to type ‘\u001b[01m\u001b[Kconst void**\u001b[m\u001b[K’ casts away qualifiers [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wcast-qual\u0007-Wcast-qual\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            " 7269 |         CUBLAS_CHECK(\n",
            "      |                                                                                                                                                                                                      \u001b[01;35m\u001b[K^\u001b[m\u001b[K                                    \n",
            "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wdouble-promotion -pthread -march=native -mtune=native    -c ggml-alloc.c -o ggml-alloc.o\n",
            "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wdouble-promotion -pthread -march=native -mtune=native    -c ggml-backend.c -o ggml-backend.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/main/main.cpp ggml.o llama.o common.o sampling.o grammar-parser.o console.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o main -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "\n",
            "====  Run ./main -h for help.  ====\n",
            "\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/quantize/quantize.cpp ggml.o llama.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o quantize -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/quantize-stats/quantize-stats.cpp ggml.o llama.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o quantize-stats -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/perplexity/perplexity.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o perplexity -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/embedding/embedding.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o embedding -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  pocs/vdot/vdot.cpp ggml.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o vdot -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  pocs/vdot/q8dot.cpp ggml.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o q8dot -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -c common/train.cpp -o train.o\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o sampling.o grammar-parser.o train.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o train-text-from-scratch -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o convert-llama2c-to-ggml -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/simple/simple.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o simple -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/batched/batched.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o batched -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/batched-bench/batched-bench.cpp ggml.o llama.o common.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o batched-bench -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o save-load-state -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  -Iexamples/server examples/server/server.cpp examples/llava/clip.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o server -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib   -Wno-cast-qual\n",
            "In copy constructor ‘\u001b[01m\u001b[Ktask_result::task_result(const task_result&)\u001b[m\u001b[K’,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid __gnu_cxx::new_allocator<_Tp>::construct(_Up*, _Args&& ...) [with _Up = task_result; _Args = {const task_result&}; _Tp = task_result]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/ext/new_allocator.h:162:4\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kstatic void std::allocator_traits<std::allocator<_Tp1> >::construct(std::allocator_traits<std::allocator<_Tp1> >::allocator_type&, _Up*, _Args&& ...) [with _Up = task_result; _Args = {const task_result&}; _Tp = task_result]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/alloc_traits.h:516:17\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid std::vector<_Tp, _Alloc>::push_back(const value_type&) [with _Tp = task_result; _Alloc = std::allocator<task_result>]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/stl_vector.h:1192:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid llama_server_context::send_error(int, std::string)\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:1095:32\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:154:8:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kres.task_result::stop\u001b[m\u001b[K’ may be used uninitialized [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wmaybe-uninitialized\u0007-Wmaybe-uninitialized\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            "  154 | struct \u001b[01;35m\u001b[Ktask_result\u001b[m\u001b[K {\n",
            "      |        \u001b[01;35m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:\u001b[m\u001b[K In member function ‘\u001b[01m\u001b[Kvoid llama_server_context::send_error(int, std::string)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:1091:21:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K‘\u001b[01m\u001b[Kres\u001b[m\u001b[K’ declared here\n",
            " 1091 |         task_result \u001b[01;36m\u001b[Kres\u001b[m\u001b[K;\n",
            "      |                     \u001b[01;36m\u001b[K^~~\u001b[m\u001b[K\n",
            "In copy constructor ‘\u001b[01m\u001b[Ktask_server::task_server(const task_server&)\u001b[m\u001b[K’,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid __gnu_cxx::new_allocator<_Tp>::construct(_Up*, _Args&& ...) [with _Up = task_server; _Args = {const task_server&}; _Tp = task_server]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/ext/new_allocator.h:162:4\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kstatic void std::allocator_traits<std::allocator<_Tp1> >::construct(std::allocator_traits<std::allocator<_Tp1> >::allocator_type&, _Up*, _Args&& ...) [with _Up = task_server; _Args = {const task_server&}; _Tp = task_server]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/alloc_traits.h:516:17\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid std::vector<_Tp, _Alloc>::push_back(const value_type&) [with _Tp = task_server; _Alloc = std::allocator<task_server>]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/stl_vector.h:1192:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kint llama_server_context::request_completion(json, bool)\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:1255:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kmain(int, char**)::<lambda(const httplib::Request&, httplib::Response&)>\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:2277:61\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:146:8:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Ktask.task_server::target_id\u001b[m\u001b[K’ may be used uninitialized [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wmaybe-uninitialized\u0007-Wmaybe-uninitialized\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            "  146 | struct \u001b[01;35m\u001b[Ktask_server\u001b[m\u001b[K {\n",
            "      |        \u001b[01;35m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:\u001b[m\u001b[K In lambda function:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:1250:21:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K‘\u001b[01m\u001b[Ktask\u001b[m\u001b[K’ declared here\n",
            " 1250 |         task_server \u001b[01;36m\u001b[Ktask\u001b[m\u001b[K;\n",
            "      |                     \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "In copy constructor ‘\u001b[01m\u001b[Ktask_server::task_server(const task_server&)\u001b[m\u001b[K’,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid __gnu_cxx::new_allocator<_Tp>::construct(_Up*, _Args&& ...) [with _Up = task_server; _Args = {const task_server&}; _Tp = task_server]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/ext/new_allocator.h:162:4\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kstatic void std::allocator_traits<std::allocator<_Tp1> >::construct(std::allocator_traits<std::allocator<_Tp1> >::allocator_type&, _Up*, _Args&& ...) [with _Up = task_server; _Args = {const task_server&}; _Tp = task_server]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/alloc_traits.h:516:17\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid std::vector<_Tp, _Alloc>::push_back(const value_type&) [with _Tp = task_server; _Alloc = std::allocator<task_server>]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/stl_vector.h:1192:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kint llama_server_context::request_completion(json, bool)\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:1255:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kmain(int, char**)::<lambda(const httplib::Request&, httplib::Response&)>\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:2332:61\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:146:8:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Ktask.task_server::target_id\u001b[m\u001b[K’ may be used uninitialized [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wmaybe-uninitialized\u0007-Wmaybe-uninitialized\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            "  146 | struct \u001b[01;35m\u001b[Ktask_server\u001b[m\u001b[K {\n",
            "      |        \u001b[01;35m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:\u001b[m\u001b[K In lambda function:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:1250:21:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K‘\u001b[01m\u001b[Ktask\u001b[m\u001b[K’ declared here\n",
            " 1250 |         task_server \u001b[01;36m\u001b[Ktask\u001b[m\u001b[K;\n",
            "      |                     \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "In copy constructor ‘\u001b[01m\u001b[Ktask_server::task_server(const task_server&)\u001b[m\u001b[K’,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid __gnu_cxx::new_allocator<_Tp>::construct(_Up*, _Args&& ...) [with _Up = task_server; _Args = {const task_server&}; _Tp = task_server]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/ext/new_allocator.h:162:4\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kstatic void std::allocator_traits<std::allocator<_Tp1> >::construct(std::allocator_traits<std::allocator<_Tp1> >::allocator_type&, _Up*, _Args&& ...) [with _Up = task_server; _Args = {const task_server&}; _Tp = task_server]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/alloc_traits.h:516:17\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kvoid std::vector<_Tp, _Alloc>::push_back(const value_type&) [with _Tp = task_server; _Alloc = std::allocator<task_server>]\u001b[m\u001b[K’ at \u001b[01m\u001b[K/usr/include/c++/11/bits/stl_vector.h:1192:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kint llama_server_context::request_completion(json, bool)\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:1255:30\u001b[m\u001b[K,\n",
            "    inlined from ‘\u001b[01m\u001b[Kmain(int, char**)::<lambda(const httplib::Request&, httplib::Response&)>\u001b[m\u001b[K’ at \u001b[01m\u001b[Kexamples/server/server.cpp:2436:61\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:146:8:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Ktask.task_server::target_id\u001b[m\u001b[K’ may be used uninitialized [\u001b[01;35m\u001b[K\u001b]8;;https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wmaybe-uninitialized\u0007-Wmaybe-uninitialized\u001b]8;;\u0007\u001b[m\u001b[K]\n",
            "  146 | struct \u001b[01;35m\u001b[Ktask_server\u001b[m\u001b[K {\n",
            "      |        \u001b[01;35m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:\u001b[m\u001b[K In lambda function:\n",
            "\u001b[01m\u001b[Kexamples/server/server.cpp:1250:21:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K‘\u001b[01m\u001b[Ktask\u001b[m\u001b[K’ declared here\n",
            " 1250 |         task_server \u001b[01;36m\u001b[Ktask\u001b[m\u001b[K;\n",
            "      |                     \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/gguf/gguf.cpp ggml.o llama.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o gguf -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/llama-bench/llama-bench.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o llama-bench -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/llava/llava.cpp examples/llava/clip.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o llava -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib  -Wno-cast-qual\n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o sampling.o grammar-parser.o train.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o baby-llama -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/beam-search/beam-search.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o beam-search -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/speculative/speculative.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o speculative -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/infill/infill.cpp ggml.o llama.o common.o sampling.o grammar-parser.o console.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o infill -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/benchmark/benchmark-matmult.cpp ggml.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o benchmark-matmult -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/parallel/parallel.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o parallel -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/finetune/finetune.cpp ggml.o llama.o common.o sampling.o grammar-parser.o train.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o finetune -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wno-array-bounds -Wno-format-truncation -Wextra-semi -march=native -mtune=native  examples/export-lora/export-lora.cpp ggml.o llama.o common.o sampling.o grammar-parser.o k_quants.o ggml-cuda.o ggml-alloc.o ggml-backend.o -o export-lora -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib \n",
            "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_K_QUANTS -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wdouble-promotion -pthread -march=native -mtune=native  -c tests/test-c.c -o tests/test-c.o\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "config = \"\"\"\n",
        "authtoken: 2XMswsT2z3mRwXoEkALduNhNDoA_7tfvtmZ56wGnt7qebcjHd\n",
        "region: us\n",
        "tunnels:\n",
        "  first_tunnel:\n",
        "    proto: http\n",
        "    addr: 127.0.0.1:1000\n",
        "    hostname: big-poodle-upward.ngrok-free.app\n",
        "  second_tunnel:\n",
        "    proto: http\n",
        "    addr: 127.0.0.1:80\n",
        "\"\"\"\n",
        "\n",
        "with open('/root/.ngrok2/ngrok.yml', 'w') as f:\n",
        "    f.write(config)\n"
      ],
      "metadata": {
        "id": "V-DCFCgKZ3RH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install flask flask-ngrok\n",
        "from flask import Flask\n",
        "from flask_ngrok import run_with_ngrok\n",
        "import subprocess\n",
        "import threading\n",
        "import time\n",
        "from IPython.display import IFrame\n",
        "\n",
        "app = Flask(__name__)\n",
        "run_with_ngrok(app)   #starts ngrok when the app is run\n",
        "\n",
        "@app.route(\"/\")\n",
        "def home():\n",
        "    return \"<h1>Running Flask on Google Colab!</h1>\"\n",
        "\n",
        "# Function to start the server\n",
        "def start_server():\n",
        "    subprocess.Popen([\"./server\", \"--port\", \"1000\", \"-ngl\", \"45\", \"-m\", \"../ggml-model-q5_k.gguf\", \"--mmproj\", \"../mmproj-model-f16.gguf\", \"-t\", \"2\"], cwd=\"/content/llama.cpp/\")\n",
        "\n",
        "# Start the server in a separate thread\n",
        "threading.Thread(target=start_server).start()\n",
        "\n",
        "# Give the server time to start\n",
        "time.sleep(2)\n",
        "\n",
        "# Terminate any existing ngrok processes\n",
        "get_ipython().system_raw('pkill -f ngrok')\n",
        "\n",
        "# Start ngrok with the start command and the --all option:\n",
        "get_ipython().system_raw('./ngrok start --all -config=/content/ngrok.yml > ngrok.log &')\n",
        "\n",
        "# Display the server's content in an iframe\n",
        "IFrame(\"http://localhost:1000\", width=1024, height=768)\n"
      ],
      "metadata": {
        "id": "BHvKZ2MRszfi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "b4934f3e-3cf9-4d46-dc49-7fe338f9038f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: flask in /usr/local/lib/python3.10/dist-packages (2.2.5)\n",
            "Collecting flask-ngrok\n",
            "  Downloading flask_ngrok-0.0.25-py3-none-any.whl (3.1 kB)\n",
            "Requirement already satisfied: Werkzeug>=2.2.2 in /usr/local/lib/python3.10/dist-packages (from flask) (3.0.1)\n",
            "Requirement already satisfied: Jinja2>=3.0 in /usr/local/lib/python3.10/dist-packages (from flask) (3.1.2)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.10/dist-packages (from flask) (2.1.2)\n",
            "Requirement already satisfied: click>=8.0 in /usr/local/lib/python3.10/dist-packages (from flask) (8.1.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from flask-ngrok) (2.31.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from Jinja2>=3.0->flask) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->flask-ngrok) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->flask-ngrok) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->flask-ngrok) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->flask-ngrok) (2023.7.22)\n",
            "Installing collected packages: flask-ngrok\n",
            "Successfully installed flask-ngrok-0.0.25\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.lib.display.IFrame at 0x787f8be2b460>"
            ],
            "text/html": [
              "\n",
              "        <iframe\n",
              "            width=\"1024\"\n",
              "            height=\"768\"\n",
              "            src=\"http://localhost:1000\"\n",
              "            frameborder=\"0\"\n",
              "            allowfullscreen\n",
              "            \n",
              "        ></iframe>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_ipython().system_raw('./ngrok http 1000 -log=stdout > ngrok.log &')\n"
      ],
      "metadata": {
        "id": "wZKE-t5jt25D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# don't worry above is a dummy api key that's been reset , that's what it looks like CHANGE IT WITH YOUR OWN\n",
        "!cat ngrok.log      #look below for the http and https links i.e.  https://d06a-34-87-173-22.ngrok-free.app\n",
        "#Then go on your home computer and open the tunnel to the webserver on this lab and use the model :)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X1VEY86Zt5QF",
        "outputId": "28b08423-b1a4-473a-9d10-679b4aaaa225"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "t=2023-10-28T04:40:04+0000 lvl=info msg=\"no configuration paths supplied\"\n",
            "t=2023-10-28T04:40:04+0000 lvl=info msg=\"using configuration at default config path\" path=/root/.ngrok2/ngrok.yml\n",
            "t=2023-10-28T04:40:04+0000 lvl=info msg=\"open config file\" path=/root/.ngrok2/ngrok.yml err=nil\n",
            "t=2023-10-28T04:40:04+0000 lvl=info msg=\"starting web service\" obj=web addr=127.0.0.1:4040\n",
            "t=2023-10-28T04:40:04+0000 lvl=info msg=\"tunnel session started\" obj=tunnels.session\n",
            "t=2023-10-28T04:40:04+0000 lvl=info msg=\"client session established\" obj=csess id=c0c6ab1df320\n",
            "t=2023-10-28T04:40:05+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=\"command_line (http)\" addr=http://localhost:1000 url=http://d476-34-125-137-51.ngrok-free.app\n",
            "t=2023-10-28T04:40:05+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=command_line addr=http://localhost:1000 url=https://d476-34-125-137-51.ngrok-free.app\n"
          ]
        }
      ]
    }
  ]
}