diff --git a/milvus/.milvus_demo.db.lock b/milvus/.milvus_demo.db.lock new file mode 100644 index 0000000..e69de29 diff --git a/milvus/milvus.ipynb b/milvus/milvus.ipynb new file mode 100644 index 0000000..1e4566a --- /dev/null +++ b/milvus/milvus.ipynb @@ -0,0 +1,480 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting pymilvus\n", + " Using cached pymilvus-2.4.4-py3-none-any.whl (196 kB)\n", + "Collecting pandas>=1.2.4\n", + " Downloading pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl (11.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m16.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hCollecting environs<=9.5.0\n", + " Using cached environs-9.5.0-py2.py3-none-any.whl (12 kB)\n", + "Requirement already satisfied: ujson>=2.0.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus) (5.8.0)\n", + "Requirement already satisfied: protobuf>=3.20.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus) (4.25.3)\n", + "Collecting milvus-lite<2.5.0,>=2.4.0\n", + " Using cached milvus_lite-2.4.8-py3-none-macosx_11_0_arm64.whl (19.8 MB)\n", + "Collecting setuptools>=67\n", + " Using cached setuptools-71.1.0-py3-none-any.whl (2.3 MB)\n", + "Collecting grpcio<=1.63.0,>=1.49.1\n", + " Downloading grpcio-1.63.0-cp310-cp310-macosx_12_0_universal2.whl (10.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dotenv in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from environs<=9.5.0->pymilvus) (1.0.1)\n", + "Collecting marshmallow>=3.0.0\n", + " Using cached marshmallow-3.21.3-py3-none-any.whl (49 kB)\n", + "Requirement already satisfied: tqdm in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from milvus-lite<2.5.0,>=2.4.0->pymilvus) (4.66.4)\n", + "Collecting pytz>=2020.1\n", + " Using cached pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pandas>=1.2.4->pymilvus) (2.9.0.post0)\n", + "Requirement already satisfied: numpy>=1.22.4 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pandas>=1.2.4->pymilvus) (2.0.0)\n", + "Collecting tzdata>=2022.7\n", + " Using cached tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", + "Requirement already satisfied: packaging>=17.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from marshmallow>=3.0.0->environs<=9.5.0->pymilvus) (24.1)\n", + "Requirement already satisfied: six>=1.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.2.4->pymilvus) (1.16.0)\n", + "Installing collected packages: pytz, tzdata, setuptools, milvus-lite, marshmallow, grpcio, pandas, environs, pymilvus\n", + " Attempting uninstall: setuptools\n", + " Found existing installation: setuptools 65.5.0\n", + " Uninstalling setuptools-65.5.0:\n", + " Successfully uninstalled setuptools-65.5.0\n", + " Attempting uninstall: grpcio\n", + " Found existing installation: grpcio 1.65.1\n", + " Uninstalling grpcio-1.65.1:\n", + " Successfully uninstalled grpcio-1.65.1\n", + "Successfully installed environs-9.5.0 grpcio-1.63.0 marshmallow-3.21.3 milvus-lite-2.4.8 pandas-2.2.2 pymilvus-2.4.4 pytz-2024.1 setuptools-71.1.0 tzdata-2024.1\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install -U pymilvus" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from pymilvus import MilvusClient\n", + "\n", + "client = MilvusClient(\"milvus_demo.db\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "if client.has_collection(collection_name=\"demo_collection\"):\n", + " client.drop_collection(collection_name=\"demo_collection\")\n", + "client.create_collection(\n", + " collection_name=\"demo_collection\",\n", + " dimension=768, # The vectors we will use in this demo has 768 dimensions\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pymilvus[model] in /Users/gyliu/pr-agent/lib/python3.10/site-packages (2.4.4)\n", + "Requirement already satisfied: protobuf>=3.20.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (4.25.3)\n", + "Requirement already satisfied: pandas>=1.2.4 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (2.2.2)\n", + "Requirement already satisfied: environs<=9.5.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (9.5.0)\n", + "Requirement already satisfied: grpcio<=1.63.0,>=1.49.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (1.63.0)\n", + "Requirement already satisfied: ujson>=2.0.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (5.8.0)\n", + "Requirement already satisfied: setuptools>=67 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (71.1.0)\n", + "Requirement already satisfied: milvus-lite<2.5.0,>=2.4.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pymilvus[model]) (2.4.8)\n", + "Collecting milvus-model>=0.1.0\n", + " Downloading milvus_model-0.2.3-py3-none-any.whl (29 kB)\n", + "Requirement already satisfied: marshmallow>=3.0.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from environs<=9.5.0->pymilvus[model]) (3.21.3)\n", + "Requirement already satisfied: python-dotenv in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from environs<=9.5.0->pymilvus[model]) (1.0.1)\n", + "Requirement already satisfied: tqdm in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from milvus-lite<2.5.0,>=2.4.0->pymilvus[model]) (4.66.4)\n", + "Collecting transformers>=4.36.0\n", + " Downloading transformers-4.43.2-py3-none-any.whl (9.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.4/9.4 MB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hCollecting onnxruntime\n", + " Downloading onnxruntime-1.18.1-cp310-cp310-macosx_11_0_universal2.whl (15.9 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.9/15.9 MB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hCollecting scipy>=1.10.0\n", + " Downloading scipy-1.14.0-cp310-cp310-macosx_14_0_arm64.whl (23.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.1/23.1 MB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from milvus-model>=0.1.0->pymilvus[model]) (2.0.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pandas>=1.2.4->pymilvus[model]) (2024.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pandas>=1.2.4->pymilvus[model]) (2.9.0.post0)\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pandas>=1.2.4->pymilvus[model]) (2024.1)\n", + "Requirement already satisfied: packaging>=17.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from marshmallow>=3.0.0->environs<=9.5.0->pymilvus[model]) (24.1)\n", + "Requirement already satisfied: six>=1.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.2.4->pymilvus[model]) (1.16.0)\n", + "Requirement already satisfied: filelock in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (3.15.4)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (0.24.0)\n", + "Requirement already satisfied: regex!=2019.12.17 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (2024.5.15)\n", + "Requirement already satisfied: pyyaml>=5.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (6.0.1)\n", + "Requirement already satisfied: tokenizers<0.20,>=0.19 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (0.19.1)\n", + "Requirement already satisfied: requests in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (2.32.3)\n", + "Collecting safetensors>=0.4.1\n", + " Using cached safetensors-0.4.3-cp310-cp310-macosx_11_0_arm64.whl (410 kB)\n", + "Collecting coloredlogs\n", + " Using cached coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n", + "Collecting numpy\n", + " Using cached numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl (14.0 MB)\n", + "Collecting sympy\n", + " Downloading sympy-1.13.1-py3-none-any.whl (6.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hCollecting flatbuffers\n", + " Using cached flatbuffers-24.3.25-py2.py3-none-any.whl (26 kB)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (2024.6.1)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (4.12.2)\n", + "Collecting humanfriendly>=9.1\n", + " Using cached humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from requests->transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from requests->transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from requests->transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (2024.7.4)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from requests->transformers>=4.36.0->milvus-model>=0.1.0->pymilvus[model]) (3.3.2)\n", + "Collecting mpmath<1.4,>=1.1.0\n", + " Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)\n", + "Installing collected packages: mpmath, flatbuffers, sympy, safetensors, numpy, humanfriendly, scipy, coloredlogs, onnxruntime, transformers, milvus-model\n", + " Attempting uninstall: numpy\n", + " Found existing installation: numpy 2.0.0\n", + " Uninstalling numpy-2.0.0:\n", + " Successfully uninstalled numpy-2.0.0\n", + "Successfully installed coloredlogs-15.0.1 flatbuffers-24.3.25 humanfriendly-10.0 milvus-model-0.2.3 mpmath-1.3.0 numpy-1.26.4 onnxruntime-1.18.1 safetensors-0.4.3 scipy-1.14.0 sympy-1.13.1 transformers-4.43.2\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install \"pymilvus[model]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting jupyter\n", + " Downloading jupyter-1.0.0-py2.py3-none-any.whl (2.7 kB)\n", + "Collecting jupyter-console\n", + " Downloading jupyter_console-6.6.3-py3-none-any.whl (24 kB)\n", + "Collecting notebook\n", + " Downloading notebook-7.2.1-py3-none-any.whl (5.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.0/5.0 MB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hCollecting nbconvert\n", + " Downloading nbconvert-7.16.4-py3-none-any.whl (257 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m257.4/257.4 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting qtconsole\n", + " Downloading qtconsole-5.5.2-py3-none-any.whl (123 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.4/123.4 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: ipykernel in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter) (6.29.5)\n", + "Collecting ipywidgets\n", + " Downloading ipywidgets-8.1.3-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.4/139.4 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: traitlets>=5.4.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (5.14.3)\n", + "Requirement already satisfied: pyzmq>=24 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (26.0.3)\n", + "Requirement already satisfied: packaging in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (24.1)\n", + "Requirement already satisfied: psutil in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (6.0.0)\n", + "Requirement already satisfied: comm>=0.1.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (0.2.2)\n", + "Requirement already satisfied: nest-asyncio in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (1.6.0)\n", + "Requirement already satisfied: matplotlib-inline>=0.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (0.1.7)\n", + "Requirement already satisfied: ipython>=7.23.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (8.26.0)\n", + "Requirement already satisfied: tornado>=6.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (6.4.1)\n", + "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (5.7.2)\n", + "Requirement already satisfied: debugpy>=1.6.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (1.8.2)\n", + "Requirement already satisfied: jupyter-client>=6.1.12 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (8.6.2)\n", + "Requirement already satisfied: appnope in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipykernel->jupyter) (0.1.4)\n", + "Collecting jupyterlab-widgets~=3.0.11\n", + " Downloading jupyterlab_widgets-3.0.11-py3-none-any.whl (214 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.4/214.4 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting widgetsnbextension~=4.0.11\n", + " Downloading widgetsnbextension-4.0.11-py3-none-any.whl (2.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pygments in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter-console->jupyter) (2.18.0)\n", + "Requirement already satisfied: prompt-toolkit>=3.0.30 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter-console->jupyter) (3.0.47)\n", + "Collecting defusedxml\n", + " Downloading defusedxml-0.7.1-py2.py3-none-any.whl (25 kB)\n", + "Collecting jupyterlab-pygments\n", + " Downloading jupyterlab_pygments-0.3.0-py3-none-any.whl (15 kB)\n", + "Collecting tinycss2\n", + " Downloading tinycss2-1.3.0-py3-none-any.whl (22 kB)\n", + "Collecting mistune<4,>=2.0.3\n", + " Downloading mistune-3.0.2-py3-none-any.whl (47 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.0/48.0 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: markupsafe>=2.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from nbconvert->jupyter) (2.1.5)\n", + "Collecting pandocfilters>=1.4.1\n", + " Downloading pandocfilters-1.5.1-py2.py3-none-any.whl (8.7 kB)\n", + "Collecting nbformat>=5.7\n", + " Downloading nbformat-5.10.4-py3-none-any.whl (78 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.5/78.5 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting beautifulsoup4\n", + " Using cached beautifulsoup4-4.12.3-py3-none-any.whl (147 kB)\n", + "Collecting bleach!=5.0.0\n", + " Downloading bleach-6.1.0-py3-none-any.whl (162 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.8/162.8 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: jinja2>=3.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from nbconvert->jupyter) (3.1.2)\n", + "Collecting nbclient>=0.5.0\n", + " Downloading nbclient-0.10.0-py3-none-any.whl (25 kB)\n", + "Collecting jupyter-server<3,>=2.4.0\n", + " Downloading jupyter_server-2.14.2-py3-none-any.whl (383 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m383.6/383.6 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting notebook-shim<0.3,>=0.2\n", + " Downloading notebook_shim-0.2.4-py3-none-any.whl (13 kB)\n", + "Collecting jupyterlab-server<3,>=2.27.1\n", + " Downloading jupyterlab_server-2.27.3-py3-none-any.whl (59 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.7/59.7 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jupyterlab<4.3,>=4.2.0\n", + " Downloading jupyterlab-4.2.4-py3-none-any.whl (11.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hCollecting qtpy>=2.4.0\n", + " Downloading QtPy-2.4.1-py3-none-any.whl (93 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.5/93.5 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: six>=1.9.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from bleach!=5.0.0->nbconvert->jupyter) (1.16.0)\n", + "Collecting webencodings\n", + " Downloading webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n", + "Requirement already satisfied: pexpect>4.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=7.23.1->ipykernel->jupyter) (4.9.0)\n", + "Requirement already satisfied: exceptiongroup in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=7.23.1->ipykernel->jupyter) (1.2.2)\n", + "Requirement already satisfied: typing-extensions>=4.6 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=7.23.1->ipykernel->jupyter) (4.12.2)\n", + "Requirement already satisfied: jedi>=0.16 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=7.23.1->ipykernel->jupyter) (0.19.1)\n", + "Requirement already satisfied: stack-data in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=7.23.1->ipykernel->jupyter) (0.6.3)\n", + "Requirement already satisfied: decorator in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=7.23.1->ipykernel->jupyter) (5.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter-client>=6.1.12->ipykernel->jupyter) (2.9.0.post0)\n", + "Requirement already satisfied: platformdirs>=2.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel->jupyter) (4.2.2)\n", + "Collecting jupyter-server-terminals>=0.4.4\n", + " Downloading jupyter_server_terminals-0.5.3-py3-none-any.whl (13 kB)\n", + "Collecting terminado>=0.8.3\n", + " Downloading terminado-0.18.1-py3-none-any.whl (14 kB)\n", + "Collecting prometheus-client>=0.9\n", + " Using cached prometheus_client-0.20.0-py3-none-any.whl (54 kB)\n", + "Collecting send2trash>=1.8.2\n", + " Downloading Send2Trash-1.8.3-py3-none-any.whl (18 kB)\n", + "Collecting websocket-client>=1.7\n", + " Using cached websocket_client-1.8.0-py3-none-any.whl (58 kB)\n", + "Collecting jupyter-events>=0.9.0\n", + " Downloading jupyter_events-0.10.0-py3-none-any.whl (18 kB)\n", + "Collecting overrides>=5.0\n", + " Using cached overrides-7.7.0-py3-none-any.whl (17 kB)\n", + "Requirement already satisfied: anyio>=3.1.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->notebook->jupyter) (4.4.0)\n", + "Collecting argon2-cffi>=21.1\n", + " Downloading argon2_cffi-23.1.0-py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: httpx>=0.25.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyterlab<4.3,>=4.2.0->notebook->jupyter) (0.27.0)\n", + "Requirement already satisfied: tomli>=1.2.2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyterlab<4.3,>=4.2.0->notebook->jupyter) (2.0.1)\n", + "Collecting jupyter-lsp>=2.0.0\n", + " Downloading jupyter_lsp-2.2.5-py3-none-any.whl (69 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.1/69.1 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting async-lru>=1.0.0\n", + " Downloading async_lru-2.0.4-py3-none-any.whl (6.1 kB)\n", + "Requirement already satisfied: setuptools>=40.1.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyterlab<4.3,>=4.2.0->notebook->jupyter) (71.1.0)\n", + "Collecting jsonschema>=4.18.0\n", + " Using cached jsonschema-4.23.0-py3-none-any.whl (88 kB)\n", + "Collecting json5>=0.9.0\n", + " Downloading json5-0.9.25-py3-none-any.whl (30 kB)\n", + "Requirement already satisfied: requests>=2.31 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.27.1->notebook->jupyter) (2.32.3)\n", + "Collecting babel>=2.10\n", + " Using cached Babel-2.15.0-py3-none-any.whl (9.6 MB)\n", + "Collecting fastjsonschema>=2.15\n", + " Downloading fastjsonschema-2.20.0-py3-none-any.whl (23 kB)\n", + "Requirement already satisfied: wcwidth in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from prompt-toolkit>=3.0.30->jupyter-console->jupyter) (0.2.13)\n", + "Collecting soupsieve>1.2\n", + " Using cached soupsieve-2.5-py3-none-any.whl (36 kB)\n", + "Requirement already satisfied: idna>=2.8 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->notebook->jupyter) (3.7)\n", + "Requirement already satisfied: sniffio>=1.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->notebook->jupyter) (1.3.1)\n", + "Collecting argon2-cffi-bindings\n", + " Downloading argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl (53 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: httpcore==1.* in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from httpx>=0.25.0->jupyterlab<4.3,>=4.2.0->notebook->jupyter) (1.0.5)\n", + "Requirement already satisfied: certifi in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from httpx>=0.25.0->jupyterlab<4.3,>=4.2.0->notebook->jupyter) (2024.7.4)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from httpcore==1.*->httpx>=0.25.0->jupyterlab<4.3,>=4.2.0->notebook->jupyter) (0.14.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyter) (0.8.4)\n", + "Collecting rpds-py>=0.7.1\n", + " Downloading rpds_py-0.19.1-cp310-cp310-macosx_11_0_arm64.whl (310 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.4/310.4 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: attrs>=22.2.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->notebook->jupyter) (23.2.0)\n", + "Collecting referencing>=0.28.4\n", + " Using cached referencing-0.35.1-py3-none-any.whl (26 kB)\n", + "Collecting jsonschema-specifications>=2023.03.6\n", + " Using cached jsonschema_specifications-2023.12.1-py3-none-any.whl (18 kB)\n", + "Requirement already satisfied: pyyaml>=5.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->notebook->jupyter) (6.0.1)\n", + "Collecting rfc3986-validator>=0.1.1\n", + " Downloading rfc3986_validator-0.1.1-py2.py3-none-any.whl (4.2 kB)\n", + "Collecting python-json-logger>=2.0.4\n", + " Downloading python_json_logger-2.0.7-py3-none-any.whl (8.1 kB)\n", + "Collecting rfc3339-validator\n", + " Downloading rfc3339_validator-0.1.4-py2.py3-none-any.whl (3.5 kB)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel->jupyter) (0.7.0)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.27.1->notebook->jupyter) (2.0.7)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.27.1->notebook->jupyter) (3.3.2)\n", + "Requirement already satisfied: pure-eval in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter) (0.2.3)\n", + "Requirement already satisfied: executing>=1.2.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter) (2.0.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyter) (2.4.1)\n", + "Collecting jsonpointer>1.13\n", + " Using cached jsonpointer-3.0.0-py2.py3-none-any.whl (7.6 kB)\n", + "Collecting fqdn\n", + " Downloading fqdn-1.5.1-py3-none-any.whl (9.1 kB)\n", + "Collecting webcolors>=24.6.0\n", + " Downloading webcolors-24.6.0-py3-none-any.whl (14 kB)\n", + "Collecting uri-template\n", + " Downloading uri_template-1.3.0-py3-none-any.whl (11 kB)\n", + "Collecting isoduration\n", + " Downloading isoduration-20.11.0-py3-none-any.whl (11 kB)\n", + "Requirement already satisfied: cffi>=1.0.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter) (1.16.0)\n", + "Requirement already satisfied: pycparser in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->notebook->jupyter) (2.22)\n", + "Collecting arrow>=0.15.0\n", + " Downloading arrow-1.3.0-py3-none-any.whl (66 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.4/66.4 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting types-python-dateutil>=2.8.10\n", + " Downloading types_python_dateutil-2.9.0.20240316-py3-none-any.whl (9.7 kB)\n", + "Installing collected packages: webencodings, fastjsonschema, widgetsnbextension, websocket-client, webcolors, uri-template, types-python-dateutil, tinycss2, terminado, soupsieve, send2trash, rpds-py, rfc3986-validator, rfc3339-validator, qtpy, python-json-logger, prometheus-client, pandocfilters, overrides, mistune, jupyterlab-widgets, jupyterlab-pygments, jsonpointer, json5, fqdn, defusedxml, bleach, babel, async-lru, referencing, jupyter-server-terminals, beautifulsoup4, arrow, argon2-cffi-bindings, jsonschema-specifications, isoduration, argon2-cffi, jsonschema, ipywidgets, qtconsole, nbformat, jupyter-console, nbclient, jupyter-events, nbconvert, jupyter-server, notebook-shim, jupyterlab-server, jupyter-lsp, jupyterlab, notebook, jupyter\n", + "Successfully installed argon2-cffi-23.1.0 argon2-cffi-bindings-21.2.0 arrow-1.3.0 async-lru-2.0.4 babel-2.15.0 beautifulsoup4-4.12.3 bleach-6.1.0 defusedxml-0.7.1 fastjsonschema-2.20.0 fqdn-1.5.1 ipywidgets-8.1.3 isoduration-20.11.0 json5-0.9.25 jsonpointer-3.0.0 jsonschema-4.23.0 jsonschema-specifications-2023.12.1 jupyter-1.0.0 jupyter-console-6.6.3 jupyter-events-0.10.0 jupyter-lsp-2.2.5 jupyter-server-2.14.2 jupyter-server-terminals-0.5.3 jupyterlab-4.2.4 jupyterlab-pygments-0.3.0 jupyterlab-server-2.27.3 jupyterlab-widgets-3.0.11 mistune-3.0.2 nbclient-0.10.0 nbconvert-7.16.4 nbformat-5.10.4 notebook-7.2.1 notebook-shim-0.2.4 overrides-7.7.0 pandocfilters-1.5.1 prometheus-client-0.20.0 python-json-logger-2.0.7 qtconsole-5.5.2 qtpy-2.4.1 referencing-0.35.1 rfc3339-validator-0.1.4 rfc3986-validator-0.1.1 rpds-py-0.19.1 send2trash-1.8.3 soupsieve-2.5 terminado-0.18.1 tinycss2-1.3.0 types-python-dateutil-2.9.0.20240316 uri-template-1.3.0 webcolors-24.6.0 webencodings-0.5.1 websocket-client-1.8.0 widgetsnbextension-4.0.11\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: ipywidgets in /Users/gyliu/pr-agent/lib/python3.10/site-packages (8.1.3)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0.11 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipywidgets) (3.0.11)\n", + "Requirement already satisfied: comm>=0.1.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipywidgets) (0.2.2)\n", + "Requirement already satisfied: ipython>=6.1.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipywidgets) (8.26.0)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.11 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipywidgets) (4.0.11)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipywidgets) (5.14.3)\n", + "Requirement already satisfied: exceptiongroup in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (1.2.2)\n", + "Requirement already satisfied: pygments>=2.4.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (2.18.0)\n", + "Requirement already satisfied: jedi>=0.16 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\n", + "Requirement already satisfied: stack-data in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: typing-extensions>=4.6 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (4.12.2)\n", + "Requirement already satisfied: decorator in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n", + "Requirement already satisfied: matplotlib-inline in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.7)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.47)\n", + "Requirement already satisfied: pexpect>4.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.4)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.13)\n", + "Requirement already satisfied: executing>=1.2.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)\n", + "Requirement already satisfied: pure-eval in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.3)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)\n", + "Requirement already satisfied: six>=1.12.0 in /Users/gyliu/pr-agent/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install --upgrade jupyter\n", + "!pip install --upgrade ipywidgets" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 12\u001b[0m\n\u001b[1;32m 4\u001b[0m embedding_fn \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mDefaultEmbeddingFunction()\n\u001b[1;32m 6\u001b[0m docs \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 7\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mArtificial intelligence was founded as an academic discipline in 1956.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 8\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAlan Turing was the first person to conduct substantial research in AI.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 9\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBorn in Maida Vale, London, Turing was raised in southern England.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 10\u001b[0m ]\n\u001b[0;32m---> 12\u001b[0m vectors \u001b[38;5;241m=\u001b[39m \u001b[43membedding_fn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode_documents\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDim:\u001b[39m\u001b[38;5;124m\"\u001b[39m, embedding_fn\u001b[38;5;241m.\u001b[39mdim, vectors[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;66;03m# Dim: 768 (768,)\u001b[39;00m\n\u001b[1;32m 15\u001b[0m data \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 16\u001b[0m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: i, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvector\u001b[39m\u001b[38;5;124m\"\u001b[39m: vectors[i], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m: docs[i], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msubject\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhistory\u001b[39m\u001b[38;5;124m\"\u001b[39m}\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(vectors))\n\u001b[1;32m 18\u001b[0m ]\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/milvus_model/dense/onnx.py:29\u001b[0m, in \u001b[0;36mOnnxEmbeddingFunction.encode_documents\u001b[0;34m(self, documents)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mencode_documents\u001b[39m(\u001b[38;5;28mself\u001b[39m, documents: List[\u001b[38;5;28mstr\u001b[39m]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[np\u001b[38;5;241m.\u001b[39marray]:\n\u001b[0;32m---> 29\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_encode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/milvus_model/dense/onnx.py:32\u001b[0m, in \u001b[0;36mOnnxEmbeddingFunction._encode\u001b[0;34m(self, texts)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_encode\u001b[39m(\u001b[38;5;28mself\u001b[39m, texts: List[\u001b[38;5;28mstr\u001b[39m]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[np\u001b[38;5;241m.\u001b[39marray]:\n\u001b[0;32m---> 32\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_to_embedding(text) \u001b[38;5;28;01mfor\u001b[39;00m text \u001b[38;5;129;01min\u001b[39;00m texts]\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/milvus_model/dense/onnx.py:32\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_encode\u001b[39m(\u001b[38;5;28mself\u001b[39m, texts: List[\u001b[38;5;28mstr\u001b[39m]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[np\u001b[38;5;241m.\u001b[39marray]:\n\u001b[0;32m---> 32\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_to_embedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m text \u001b[38;5;129;01min\u001b[39;00m texts]\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/milvus_model/dense/onnx.py:45\u001b[0m, in \u001b[0;36mOnnxEmbeddingFunction._to_embedding\u001b[0;34m(self, data, **_)\u001b[0m\n\u001b[1;32m 43\u001b[0m ort_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mort_session\u001b[38;5;241m.\u001b[39mrun(\u001b[38;5;28;01mNone\u001b[39;00m, ort_inputs)\n\u001b[1;32m 44\u001b[0m ort_feat \u001b[38;5;241m=\u001b[39m ort_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m---> 45\u001b[0m emb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_post_proc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mort_feat\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mort_inputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattention_mask\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 46\u001b[0m emb \u001b[38;5;241m=\u001b[39m emb\u001b[38;5;241m.\u001b[39mflatten()\n\u001b[1;32m 47\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m emb \u001b[38;5;241m/\u001b[39m np\u001b[38;5;241m.\u001b[39mlinalg\u001b[38;5;241m.\u001b[39mnorm(emb)\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/milvus_model/dense/onnx.py:55\u001b[0m, in \u001b[0;36mOnnxEmbeddingFunction._post_proc\u001b[0;34m(self, token_embeddings, attention_mask)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_post_proc\u001b[39m(\u001b[38;5;28mself\u001b[39m, token_embeddings, attention_mask):\n\u001b[1;32m 50\u001b[0m input_mask_expanded \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 51\u001b[0m np\u001b[38;5;241m.\u001b[39mexpand_dims(attention_mask, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 52\u001b[0m \u001b[38;5;241m.\u001b[39mrepeat(token_embeddings\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m], \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 53\u001b[0m \u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mfloat\u001b[39m)\n\u001b[1;32m 54\u001b[0m )\n\u001b[0;32m---> 55\u001b[0m sentence_embs \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39msum(\u001b[43mtoken_embeddings\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[43minput_mask_expanded\u001b[49m, \u001b[38;5;241m1\u001b[39m) \u001b[38;5;241m/\u001b[39m np\u001b[38;5;241m.\u001b[39mmaximum(\n\u001b[1;32m 56\u001b[0m input_mask_expanded\u001b[38;5;241m.\u001b[39msum(\u001b[38;5;241m1\u001b[39m), \u001b[38;5;241m1e-9\u001b[39m\n\u001b[1;32m 57\u001b[0m )\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m sentence_embs\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/numpy/core/_internal.py:855\u001b[0m, in \u001b[0;36marray_ufunc_errmsg_formatter\u001b[0;34m(dummy, ufunc, method, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21marray_ufunc_errmsg_formatter\u001b[39m(dummy, ufunc, method, \u001b[38;5;241m*\u001b[39minputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 854\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Format the error message for when __array_ufunc__ gives up. \"\"\"\u001b[39;00m\n\u001b[0;32m--> 855\u001b[0m args_string \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin([\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(arg) \u001b[38;5;28;01mfor\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m inputs] \u001b[38;5;241m+\u001b[39m\n\u001b[1;32m 856\u001b[0m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m=\u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(k, v)\n\u001b[1;32m 857\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mitems()])\n\u001b[1;32m 858\u001b[0m args \u001b[38;5;241m=\u001b[39m inputs \u001b[38;5;241m+\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mout\u001b[39m\u001b[38;5;124m'\u001b[39m, ())\n\u001b[1;32m 859\u001b[0m types_string \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mrepr\u001b[39m(\u001b[38;5;28mtype\u001b[39m(arg)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m args)\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/numpy/core/_internal.py:855\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21marray_ufunc_errmsg_formatter\u001b[39m(dummy, ufunc, method, \u001b[38;5;241m*\u001b[39minputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 854\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Format the error message for when __array_ufunc__ gives up. \"\"\"\u001b[39;00m\n\u001b[0;32m--> 855\u001b[0m args_string \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin([\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;132;43;01m{!r}\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m inputs] \u001b[38;5;241m+\u001b[39m\n\u001b[1;32m 856\u001b[0m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m=\u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(k, v)\n\u001b[1;32m 857\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mitems()])\n\u001b[1;32m 858\u001b[0m args \u001b[38;5;241m=\u001b[39m inputs \u001b[38;5;241m+\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mout\u001b[39m\u001b[38;5;124m'\u001b[39m, ())\n\u001b[1;32m 859\u001b[0m types_string \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mrepr\u001b[39m(\u001b[38;5;28mtype\u001b[39m(arg)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m args)\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/numpy/core/arrayprint.py:1499\u001b[0m, in \u001b[0;36m_array_repr_implementation\u001b[0;34m(arr, max_line_width, precision, suppress_small, array2string)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1497\u001b[0m class_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124marray\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m-> 1499\u001b[0m skipdtype \u001b[38;5;241m=\u001b[39m \u001b[43mdtype_is_implied\u001b[49m\u001b[43m(\u001b[49m\u001b[43marr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mand\u001b[39;00m arr\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 1501\u001b[0m prefix \u001b[38;5;241m=\u001b[39m class_name \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1502\u001b[0m suffix \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m skipdtype \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32m~/pr-agent/lib/python3.10/site-packages/numpy/core/arrayprint.py:1440\u001b[0m, in \u001b[0;36mdtype_is_implied\u001b[0;34m(dtype)\u001b[0m\n\u001b[1;32m 1415\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdtype_is_implied\u001b[39m(dtype):\n\u001b[1;32m 1416\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1417\u001b[0m \u001b[38;5;124;03m Determine if the given dtype is implied by the representation of its values.\u001b[39;00m\n\u001b[1;32m 1418\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1438\u001b[0m \u001b[38;5;124;03m array([1, 2, 3], dtype=int8)\u001b[39;00m\n\u001b[1;32m 1439\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1440\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _format_options[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlegacy\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m113\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m dtype\u001b[38;5;241m.\u001b[39mtype \u001b[38;5;241m==\u001b[39m bool_:\n\u001b[1;32m 1442\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n", + "\u001b[0;31mTypeError\u001b[0m: " + ] + } + ], + "source": [ + "from pymilvus import model\n", + "\n", + "\n", + "embedding_fn = model.DefaultEmbeddingFunction()\n", + "\n", + "docs = [\n", + " \"Artificial intelligence was founded as an academic discipline in 1956.\",\n", + " \"Alan Turing was the first person to conduct substantial research in AI.\",\n", + " \"Born in Maida Vale, London, Turing was raised in southern England.\",\n", + "]\n", + "\n", + "vectors = embedding_fn.encode_documents(docs)\n", + "print(\"Dim:\", embedding_fn.dim, vectors[0].shape) # Dim: 768 (768,)\n", + "\n", + "data = [\n", + " {\"id\": i, \"vector\": vectors[i], \"text\": docs[i], \"subject\": \"history\"}\n", + " for i in range(len(vectors))\n", + "]\n", + "\n", + "print(\"Data has\", len(data), \"entities, each with fields: \", data[0].keys())\n", + "print(\"Vector dim:\", len(data[0][\"vector\"]))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pr-agent", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/milvus/milvus_demo.db b/milvus/milvus_demo.db new file mode 100644 index 0000000..1b26311 Binary files /dev/null and b/milvus/milvus_demo.db differ